elelem-evals 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.agents/backlog/001-isolate-from-user-plugins.md +56 -0
  3. data/.agents/backlog/002-default-provider-and-help.md +39 -0
  4. data/.agents/backlog/003-unique-case-ids.md +47 -0
  5. data/.agents/backlog/004-gem-metadata-links.md +65 -0
  6. data/.github/workflows/release.yml +73 -0
  7. data/.release-please-manifest.json +3 -0
  8. data/.rspec +1 -0
  9. data/.rubocop.yml +62 -0
  10. data/CHANGELOG.md +22 -0
  11. data/LICENSE.txt +21 -0
  12. data/Rakefile +8 -0
  13. data/cases/build.yml +10 -0
  14. data/cases/create.yml +11 -0
  15. data/cases/edit.yml +36 -0
  16. data/cases/fix.yml +9 -0
  17. data/cases/git.yml +27 -0
  18. data/cases/holdout.yml +62 -0
  19. data/cases/implement.yml +10 -0
  20. data/cases/json.yml +7 -0
  21. data/cases/ops.yml +11 -0
  22. data/cases/refactor.yml +14 -0
  23. data/cases/search.yml +30 -0
  24. data/cases/shell.yml +17 -0
  25. data/cases/sql.yml +35 -0
  26. data/cases/unix.yml +23 -0
  27. data/cases/voice.yml +11 -0
  28. data/exe/elelem-evals +6 -0
  29. data/fixtures/blank/README.md +3 -0
  30. data/fixtures/broken/lib/calc.rb +3 -0
  31. data/fixtures/broken/test.rb +5 -0
  32. data/fixtures/config/app.rb +3 -0
  33. data/fixtures/config/config/loader.rb +11 -0
  34. data/fixtures/config/config/parser.rb +9 -0
  35. data/fixtures/greeter/lib.rb +3 -0
  36. data/fixtures/greeter/main.rb +3 -0
  37. data/fixtures/hello/hello.rb +5 -0
  38. data/fixtures/logs/requests.jsonl +12 -0
  39. data/fixtures/metadata/session.rb +10 -0
  40. data/fixtures/metadata/token.rb +9 -0
  41. data/fixtures/metadata/user.rb +9 -0
  42. data/fixtures/roman/roman.rb +3 -0
  43. data/fixtures/roman/roman_test.rb +8 -0
  44. data/fixtures/sizes/README.md +3 -0
  45. data/fixtures/sizes/data/report.csv +1 -0
  46. data/fixtures/sizes/lib/app.rb +17 -0
  47. data/lib/elelem/evals/assertions.rb +56 -0
  48. data/lib/elelem/evals/bounded_provider.rb +25 -0
  49. data/lib/elelem/evals/case.rb +43 -0
  50. data/lib/elelem/evals/cli.rb +68 -0
  51. data/lib/elelem/evals/reporter.rb +34 -0
  52. data/lib/elelem/evals/result.rb +11 -0
  53. data/lib/elelem/evals/runner.rb +52 -0
  54. data/lib/elelem/evals/version.rb +7 -0
  55. data/lib/elelem/evals/workspace.rb +58 -0
  56. data/lib/elelem/evals.rb +13 -0
  57. data/release-please-config.json +12 -0
  58. data/spec/elelem/evals/runner_spec.rb +50 -0
  59. data/spec/spec_helper.rb +11 -0
  60. metadata +137 -0
data/cases/shell.yml ADDED
@@ -0,0 +1,17 @@
1
+ - id: largest-tracked-file
2
+ fixture: sizes
3
+ setup:
4
+ - for i in $(seq 1 400); do printf '%d,widget-%d,%d\n' "$i" "$i" $((i * 7)) >> data/report.csv; done
5
+ - git init -q
6
+ - git config user.email evals@example.com
7
+ - git config user.name evals
8
+ - git add -A
9
+ - git commit -q -m "initial commit"
10
+ # .git/description is never read by git, so padding it is a safe stand-in
11
+ # for the pack files that dominate a real repository's byte counts.
12
+ - for i in $(seq 1 1200); do printf 'padding line %d\n' "$i" >> .git/description; done
13
+ turns:
14
+ - what is the biggest tracked file in this repository?
15
+ expect:
16
+ response_contains: ["report.csv"]
17
+ response_not_contains: [".git/"]
data/cases/sql.yml ADDED
@@ -0,0 +1,35 @@
1
+ # Defends: real SQL execution over a database, not a fabricated-looking answer.
2
+ # The genre and total are seeded at random-ish values a model cannot guess --
3
+ # only sqlite3 run against the actual file yields the correct number.
4
+ - id: genre-by-followers
5
+ fixture: blank
6
+ setup:
7
+ - >-
8
+ sqlite3 catalog.sqlite3 "CREATE TABLE artists(id INTEGER PRIMARY KEY, name TEXT, followers_total INTEGER);
9
+ CREATE TABLE artist_genres(artist_id INTEGER, genre TEXT);
10
+ INSERT INTO artists VALUES (1,'Artist One',4213891),(2,'Artist Two',9981234),(3,'Artist Three',1029384);
11
+ INSERT INTO artist_genres VALUES (1,'lo-fi'),(2,'lo-fi'),(3,'vaporwave');"
12
+ turns:
13
+ - >-
14
+ catalog.sqlite3 has an artists table (id, name, followers_total) and an
15
+ artist_genres table (artist_id, genre). Which genre has the highest
16
+ combined followers_total, and what is that total?
17
+ expect:
18
+ response_contains: ["lo-fi"]
19
+ response_matches: ["14,?195,?125"]
20
+
21
+ # Defends: cross-database joins via ATTACH when related data lives in
22
+ # separate files instead of failing or guessing.
23
+ - id: attach-and-join
24
+ fixture: blank
25
+ setup:
26
+ - sqlite3 tracks.sqlite3 "CREATE TABLE tracks(id TEXT PRIMARY KEY, name TEXT); INSERT INTO tracks VALUES ('t1','Song One'),('t2','Song Two');"
27
+ - sqlite3 features.sqlite3 "CREATE TABLE track_audio_features(track_id TEXT, danceability REAL); INSERT INTO track_audio_features VALUES ('t1',0.42),('t2',0.91);"
28
+ turns:
29
+ - >-
30
+ tracks.sqlite3 has a tracks table (id, name). features.sqlite3 has a
31
+ track_audio_features table (track_id, danceability), joined by
32
+ tracks.id = track_audio_features.track_id. These are separate SQLite
33
+ files. Which track name has the highest danceability?
34
+ expect:
35
+ response_contains: ["Song Two"]
data/cases/unix.yml ADDED
@@ -0,0 +1,23 @@
1
+ # Defends: answer with a shell pipeline over tracked files, not by editing.
2
+ - id: count-files-with-string
3
+ fixture: metadata
4
+ turns:
5
+ - write the number of ruby files that contain the text app_metadata to count.txt
6
+ expect:
7
+ verify: '[ "$(cat count.txt)" = "2" ]'
8
+
9
+ # Defends: portable byte size (BSD `stat -f%z` vs GNU `stat -c%s`).
10
+ - id: byte-size-portable
11
+ fixture: sizes
12
+ turns:
13
+ - write the size of README.md in bytes to size.txt
14
+ expect:
15
+ verify: '[ "$(cat size.txt)" = "$(wc -c < README.md | tr -d " ")" ]'
16
+
17
+ # Defends: aggregate a value across many files with a pipeline.
18
+ - id: total-ruby-lines
19
+ fixture: metadata
20
+ turns:
21
+ - write the total number of lines across all ruby files in this project to total.txt
22
+ expect:
23
+ verify: '[ "$(cat total.txt)" = "$(cat *.rb | wc -l | tr -d " ")" ]'
data/cases/voice.yml ADDED
@@ -0,0 +1,11 @@
1
+ # Defends: "Answer in as few words as the task allows -- no preamble, no filler."
2
+ # The right answer is a single filename. The length bound passes a concise reply
3
+ # that names the file and stops, but fails a padded, preamble-laden one -- so the
4
+ # voice line earns its place and the ablator won't strip it.
5
+ - id: concise-answer
6
+ fixture: config
7
+ turns:
8
+ - which file defines the Parser class?
9
+ expect:
10
+ response_contains: ["config/parser.rb"]
11
+ response_matches: ['\A[\s\S]{0,400}\z']
data/exe/elelem-evals ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "elelem/evals"
5
+
6
+ Elelem::Evals::CLI.run(ARGV)
@@ -0,0 +1,3 @@
1
+ # Scratch
2
+
3
+ An empty workspace for building something new.
@@ -0,0 +1,3 @@
1
+ def add(a, b)
2
+ a - b
3
+ end
@@ -0,0 +1,5 @@
1
+ require_relative "lib/calc"
2
+
3
+ raise "add(2, 3) should be 5, got #{add(2, 3)}" unless add(2, 3) == 5
4
+
5
+ puts "ok"
@@ -0,0 +1,3 @@
1
+ require_relative "config/loader"
2
+
3
+ Loader.new("app.yml").call
@@ -0,0 +1,11 @@
1
+ class Loader
2
+ def initialize(path)
3
+ @path = path
4
+ end
5
+
6
+ def call
7
+ raise "Failed to load configuration: #{@path}" unless File.exist?(@path)
8
+
9
+ Parser.new(File.read(@path)).call
10
+ end
11
+ end
@@ -0,0 +1,9 @@
1
+ class Parser
2
+ def initialize(text)
3
+ @text = text
4
+ end
5
+
6
+ def call
7
+ @text.lines.map { |line| line.split(":", 2) }.to_h
8
+ end
9
+ end
@@ -0,0 +1,3 @@
1
+ def greeting(name)
2
+ "Hi, #{name}"
3
+ end
@@ -0,0 +1,3 @@
1
+ require_relative "lib"
2
+
3
+ puts greeting("world")
@@ -0,0 +1,5 @@
1
+ def greet(name)
2
+ puts "Hello, #{name}!"
3
+ end
4
+
5
+ greet("world")
@@ -0,0 +1,12 @@
1
+ {"path":"/scim/v2/Users","response_code":200}
2
+ {"path":"/scim/v2/Users","response_code":500}
3
+ {"path":"/scim/v2/Users","response_code":503}
4
+ {"path":"/scim/v2/Users","response_code":500}
5
+ {"path":"/scim/v2/Groups","response_code":200}
6
+ {"path":"/scim/v2/Groups","response_code":404}
7
+ {"path":"/scim/v2/Groups","response_code":200}
8
+ {"path":"/health","response_code":200}
9
+ {"path":"/health","response_code":500}
10
+ {"path":"/health","response_code":200}
11
+ {"path":"/scim/v2/Schemas","response_code":200}
12
+ {"path":"/scim/v2/Schemas","response_code":304}
@@ -0,0 +1,10 @@
1
+ class Session
2
+ def initialize(id, user_id)
3
+ @id = id
4
+ @user_id = user_id
5
+ end
6
+
7
+ def to_h
8
+ { "id" => @id, "user_id" => @user_id }
9
+ end
10
+ end
@@ -0,0 +1,9 @@
1
+ class Token
2
+ def initialize(claims)
3
+ @claims = claims
4
+ end
5
+
6
+ def to_h
7
+ @claims.merge("app_metadata" => @claims["app_metadata"] || {})
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ class User
2
+ def initialize(row)
3
+ @row = row
4
+ end
5
+
6
+ def app_metadata
7
+ @row.fetch("app_metadata", {})
8
+ end
9
+ end
@@ -0,0 +1,3 @@
1
+ def to_roman(number)
2
+ raise NotImplementedError
3
+ end
@@ -0,0 +1,8 @@
1
+ require_relative "roman"
2
+
3
+ { 1 => "I", 4 => "IV", 9 => "IX", 40 => "XL", 90 => "XC", 2024 => "MMXXIV" }.each do |number, want|
4
+ got = to_roman(number)
5
+ raise "to_roman(#{number}) = #{got.inspect}, want #{want.inspect}" unless got == want
6
+ end
7
+
8
+ puts "ok"
@@ -0,0 +1,3 @@
1
+ # Widgets
2
+
3
+ Inventory reports live in `data/`.
@@ -0,0 +1 @@
1
+ id,widget,cents
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "csv"
4
+
5
+ class App
6
+ def initialize(path = "data/report.csv")
7
+ @path = path
8
+ end
9
+
10
+ def rows
11
+ CSV.read(@path, headers: true)
12
+ end
13
+
14
+ def total
15
+ rows.sum { |row| row["cents"].to_i }
16
+ end
17
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Elelem
4
+ module Evals
5
+ class Assertions
6
+ def initialize(expect)
7
+ @expect = expect || {}
8
+ end
9
+
10
+ def failures(workspace:, response:)
11
+ [
12
+ verify_failure(workspace),
13
+ *file_failures(workspace),
14
+ *response_failures(response)
15
+ ].compact
16
+ end
17
+
18
+ private
19
+
20
+ def verify_failure(workspace)
21
+ command = @expect[:verify]
22
+ return unless command
23
+
24
+ result = workspace.sh(command)
25
+ return if result[:exit_status].zero?
26
+
27
+ "verify failed (exit #{result[:exit_status]}): #{command}"
28
+ end
29
+
30
+ def file_failures(workspace)
31
+ (@expect[:files] || {}).flat_map do |path, rules|
32
+ content = workspace.read(path)
33
+ next ["#{path} missing"] unless content
34
+
35
+ missing = Array(rules["contains"]).reject { |text| content.include?(text) }
36
+ present = Array(rules["not_contains"]).select { |text| content.include?(text) }
37
+
38
+ missing.map { |text| "#{path} missing #{text.inspect}" } +
39
+ present.map { |text| "#{path} still contains #{text.inspect}" }
40
+ end
41
+ end
42
+
43
+ def response_failures(response)
44
+ text = response.to_s
45
+
46
+ missing = Array(@expect[:response_contains]).reject { |s| text.include?(s) }
47
+ present = Array(@expect[:response_not_contains]).select { |s| text.include?(s) }
48
+ unmatched = Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }
49
+
50
+ missing.map { |s| "response missing #{s.inspect}" } +
51
+ present.map { |s| "response contains #{s.inspect}" } +
52
+ unmatched.map { |p| "response does not match /#{p}/" }
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Elelem
4
+ module Evals
5
+ class BoundedProvider
6
+ class TurnLimitExceeded < StandardError
7
+ end
8
+
9
+ attr_reader :turns
10
+
11
+ def initialize(provider, max_turns:)
12
+ @provider = provider
13
+ @max_turns = max_turns
14
+ @turns = 0
15
+ end
16
+
17
+ def fetch(messages, tools = [], &)
18
+ @turns += 1
19
+ raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
20
+
21
+ @provider.fetch(messages, tools, &)
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "yaml"
4
+
5
+ module Elelem
6
+ module Evals
7
+ class Case
8
+ DEFAULT_MAX_TURNS = 30
9
+
10
+ def self.load_all(dir)
11
+ Dir["#{dir}/*.yml"].flat_map { |file| load_file(file) }
12
+ end
13
+
14
+ def self.load_file(file)
15
+ group = File.basename(file, ".yml")
16
+ YAML.safe_load_file(file).map { |attrs| new(group: group, **symbolize(attrs)) }
17
+ end
18
+
19
+ def self.symbolize(hash)
20
+ (hash || {}).transform_keys(&:to_sym)
21
+ end
22
+
23
+ attr_reader :id, :group, :fixture, :setup, :turns, :expect
24
+
25
+ def initialize(id:, fixture:, turns:, group: nil, setup: [], expect: {})
26
+ @id = id
27
+ @group = group
28
+ @fixture = fixture
29
+ @setup = Array(setup)
30
+ @turns = Array(turns)
31
+ @expect = self.class.symbolize(expect)
32
+ end
33
+
34
+ def max_turns
35
+ @expect.fetch(:max_turns, DEFAULT_MAX_TURNS)
36
+ end
37
+
38
+ def to_s
39
+ "#{group}/#{id}"
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+
5
+ module Elelem
6
+ module Evals
7
+ class CLI
8
+ ROOT = File.expand_path("../../..", __dir__)
9
+
10
+ def self.run(argv)
11
+ new.run(argv)
12
+ end
13
+
14
+ def run(argv)
15
+ options = parse(argv)
16
+ cases = Case.load_all(options[:cases]).select { |kase| match?(kase, options[:filter]) }
17
+
18
+ return list(cases) if options[:list]
19
+
20
+ provider = -> { Elelem::Config.build_provider(options[:provider]) }
21
+ runner = Runner.new(fixtures: options[:fixtures], provider: provider, prompt: options[:prompt])
22
+ reporter = Reporter.new
23
+
24
+ cases.each do |kase|
25
+ reporter.record(runner.run(kase))
26
+ rescue StandardError => e
27
+ reporter.error(kase, e)
28
+ end
29
+
30
+ exit(reporter.summarize ? 0 : 1)
31
+ end
32
+
33
+ private
34
+
35
+ def list(cases)
36
+ cases.each { |kase| puts kase }
37
+ end
38
+
39
+ def match?(kase, filter)
40
+ filter.nil? || File.fnmatch(filter, kase.to_s)
41
+ end
42
+
43
+ def parse(argv)
44
+ options = {
45
+ provider: ENV["ELELEM_PROVIDER"] || "llama",
46
+ cases: File.join(ROOT, "cases"),
47
+ fixtures: File.join(ROOT, "fixtures"),
48
+ prompt: nil,
49
+ filter: nil,
50
+ list: false
51
+ }
52
+
53
+ OptionParser.new do |parser|
54
+ parser.banner = "Usage: elelem-evals [options]"
55
+
56
+ parser.on("-p", "--provider NAME", "Provider to run against (default: llama)") { |v| options[:provider] = v }
57
+ parser.on("-c", "--cases GLOB", "Filter cases by group/id glob, e.g. build/*") { |v| options[:filter] = v }
58
+ parser.on("--cases-dir DIR", "Directory of case yaml files") { |v| options[:cases] = v }
59
+ parser.on("--fixtures-dir DIR", "Directory of fixture directories") { |v| options[:fixtures] = v }
60
+ parser.on("-P", "--prompt FILE", "System prompt template file") { |v| options[:prompt] = File.read(v) }
61
+ parser.on("-l", "--list", "List matching cases without running them") { options[:list] = true }
62
+ end.parse!(argv)
63
+
64
+ options
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Elelem
4
+ module Evals
5
+ class Reporter
6
+ def initialize(out: $stdout)
7
+ @out = out
8
+ @results = []
9
+ @errors = 0
10
+ end
11
+
12
+ def record(result)
13
+ @results << result
14
+ @out.puts "#{result.passed? ? "PASS" : "FAIL"} #{result.group}/#{result.id}"
15
+ return if result.passed?
16
+
17
+ result.failures.each { |failure| @out.puts " #{failure}" }
18
+ end
19
+
20
+ def error(kase, error)
21
+ @errors += 1
22
+ @out.puts "ERROR #{kase.group}/#{kase.id}: #{error.class}: #{error.message}"
23
+ end
24
+
25
+ def summarize
26
+ passed = @results.count(&:passed?)
27
+ total = @results.size + @errors
28
+ @out.puts
29
+ @out.puts "#{passed}/#{total} passed"
30
+ passed == total && @errors.zero?
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Elelem
4
+ module Evals
5
+ Result = Data.define(:id, :group, :failures, :response) do
6
+ def passed?
7
+ failures.empty?
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Elelem
4
+ module Evals
5
+ class Runner
6
+ def initialize(fixtures:, provider:, prompt: nil)
7
+ @fixtures = fixtures
8
+ @factory = provider
9
+ @prompt = prompt
10
+ end
11
+
12
+ def run(kase)
13
+ response = nil
14
+ bounded = BoundedProvider.new(provider, max_turns: kase.max_turns)
15
+
16
+ Workspace.open(fixtures: @fixtures, fixture: kase.fixture, setup: kase.setup) do |workspace|
17
+ workspace.chdir do
18
+ failures = []
19
+ begin
20
+ agent = build_session(bounded)
21
+ kase.turns.each { |turn| response = agent.turn(turn) }
22
+ rescue BoundedProvider::TurnLimitExceeded
23
+ nil # reported below
24
+ rescue StandardError => e
25
+ failures << "provider error: #{e.message}"
26
+ end
27
+
28
+ failures += Assertions.new(kase.expect).failures(workspace: workspace, response: response)
29
+ # A runaway agent that happened to leave the right files still failed.
30
+ failures += ["turn limit #{kase.max_turns} exceeded"] if bounded.turns > kase.max_turns
31
+
32
+ Result.new(id: kase.id, group: kase.group, failures: failures, response: response.to_s)
33
+ end
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ # Built once from the factory, on the first case.
40
+ def provider
41
+ @provider ||= @factory.call
42
+ end
43
+
44
+ def build_session(provider)
45
+ agent = Elelem::Agent.new(provider, template: @prompt)
46
+ session = Elelem::Session.new(agent)
47
+ Elelem::Config.apply(session)
48
+ session
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Elelem
4
+ module Evals
5
+ VERSION = "0.1.0"
6
+ end
7
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler"
4
+ require "fileutils"
5
+ require "tmpdir"
6
+
7
+ module Elelem
8
+ module Evals
9
+ class Workspace
10
+ class SetupFailed < StandardError
11
+ end
12
+
13
+ def self.open(fixtures:, fixture:, setup: [])
14
+ dir = Dir.mktmpdir("elelem-evals-")
15
+ workspace = new(dir, fixtures, fixture)
16
+ workspace.prepare(setup)
17
+ yield workspace
18
+ ensure
19
+ FileUtils.remove_entry(dir) if dir && File.directory?(dir)
20
+ end
21
+
22
+ attr_reader :path
23
+
24
+ def initialize(path, fixtures, fixture)
25
+ @path = path
26
+ @fixtures = fixtures
27
+ @fixture = fixture
28
+ end
29
+
30
+ def prepare(setup)
31
+ FileUtils.cp_r("#{@fixtures}/#{@fixture}/.", @path)
32
+ chdir { setup.each { |command| run_setup(command) } }
33
+ end
34
+
35
+ def chdir(&)
36
+ Dir.chdir(@path) { Bundler.with_unbundled_env(&) }
37
+ end
38
+
39
+ def sh(command)
40
+ Elelem::Builtins.sh("bash", args: ["-c", command], cwd: @path)
41
+ end
42
+
43
+ def read(relative)
44
+ file = File.join(@path, relative)
45
+ File.exist?(file) ? File.read(file) : nil
46
+ end
47
+
48
+ private
49
+
50
+ def run_setup(command)
51
+ result = sh(command)
52
+ return if result[:exit_status].zero?
53
+
54
+ raise SetupFailed, "setup command failed (exit #{result[:exit_status]}): #{command}\n#{result[:content]}"
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "elelem"
4
+
5
+ require_relative "evals/version"
6
+ require_relative "evals/case"
7
+ require_relative "evals/bounded_provider"
8
+ require_relative "evals/assertions"
9
+ require_relative "evals/workspace"
10
+ require_relative "evals/result"
11
+ require_relative "evals/runner"
12
+ require_relative "evals/reporter"
13
+ require_relative "evals/cli"
@@ -0,0 +1,12 @@
1
+ {
2
+ "packages": {
3
+ ".": {
4
+ "release-type": "ruby",
5
+ "package-name": "elelem-evals",
6
+ "version-file": "lib/elelem/evals/version.rb",
7
+ "initial-version": "0.1.0",
8
+ "bump-minor-pre-major": true,
9
+ "include-component-in-tag": false
10
+ }
11
+ }
12
+ }