elelem-evals 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.agents/backlog/001-isolate-from-user-plugins.md +56 -0
- data/.agents/backlog/002-default-provider-and-help.md +39 -0
- data/.agents/backlog/003-unique-case-ids.md +47 -0
- data/.agents/backlog/004-gem-metadata-links.md +65 -0
- data/.github/workflows/release.yml +73 -0
- data/.release-please-manifest.json +3 -0
- data/.rspec +1 -0
- data/.rubocop.yml +62 -0
- data/CHANGELOG.md +22 -0
- data/LICENSE.txt +21 -0
- data/Rakefile +8 -0
- data/cases/build.yml +10 -0
- data/cases/create.yml +11 -0
- data/cases/edit.yml +36 -0
- data/cases/fix.yml +9 -0
- data/cases/git.yml +27 -0
- data/cases/holdout.yml +62 -0
- data/cases/implement.yml +10 -0
- data/cases/json.yml +7 -0
- data/cases/ops.yml +11 -0
- data/cases/refactor.yml +14 -0
- data/cases/search.yml +30 -0
- data/cases/shell.yml +17 -0
- data/cases/sql.yml +35 -0
- data/cases/unix.yml +23 -0
- data/cases/voice.yml +11 -0
- data/exe/elelem-evals +6 -0
- data/fixtures/blank/README.md +3 -0
- data/fixtures/broken/lib/calc.rb +3 -0
- data/fixtures/broken/test.rb +5 -0
- data/fixtures/config/app.rb +3 -0
- data/fixtures/config/config/loader.rb +11 -0
- data/fixtures/config/config/parser.rb +9 -0
- data/fixtures/greeter/lib.rb +3 -0
- data/fixtures/greeter/main.rb +3 -0
- data/fixtures/hello/hello.rb +5 -0
- data/fixtures/logs/requests.jsonl +12 -0
- data/fixtures/metadata/session.rb +10 -0
- data/fixtures/metadata/token.rb +9 -0
- data/fixtures/metadata/user.rb +9 -0
- data/fixtures/roman/roman.rb +3 -0
- data/fixtures/roman/roman_test.rb +8 -0
- data/fixtures/sizes/README.md +3 -0
- data/fixtures/sizes/data/report.csv +1 -0
- data/fixtures/sizes/lib/app.rb +17 -0
- data/lib/elelem/evals/assertions.rb +56 -0
- data/lib/elelem/evals/bounded_provider.rb +25 -0
- data/lib/elelem/evals/case.rb +43 -0
- data/lib/elelem/evals/cli.rb +68 -0
- data/lib/elelem/evals/reporter.rb +34 -0
- data/lib/elelem/evals/result.rb +11 -0
- data/lib/elelem/evals/runner.rb +52 -0
- data/lib/elelem/evals/version.rb +7 -0
- data/lib/elelem/evals/workspace.rb +58 -0
- data/lib/elelem/evals.rb +13 -0
- data/release-please-config.json +12 -0
- data/spec/elelem/evals/runner_spec.rb +50 -0
- data/spec/spec_helper.rb +11 -0
- metadata +137 -0
data/cases/shell.yml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
- id: largest-tracked-file
|
|
2
|
+
fixture: sizes
|
|
3
|
+
setup:
|
|
4
|
+
- for i in $(seq 1 400); do printf '%d,widget-%d,%d\n' "$i" "$i" $((i * 7)) >> data/report.csv; done
|
|
5
|
+
- git init -q
|
|
6
|
+
- git config user.email evals@example.com
|
|
7
|
+
- git config user.name evals
|
|
8
|
+
- git add -A
|
|
9
|
+
- git commit -q -m "initial commit"
|
|
10
|
+
# .git/description is never read by git, so padding it is a safe stand-in
|
|
11
|
+
# for the pack files that dominate a real repository's byte counts.
|
|
12
|
+
- for i in $(seq 1 1200); do printf 'padding line %d\n' "$i" >> .git/description; done
|
|
13
|
+
turns:
|
|
14
|
+
- what is the biggest tracked file in this repository?
|
|
15
|
+
expect:
|
|
16
|
+
response_contains: ["report.csv"]
|
|
17
|
+
response_not_contains: [".git/"]
|
data/cases/sql.yml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Defends: real SQL execution over a database, not a fabricated-looking answer.
|
|
2
|
+
# The genre and total are seeded at random-ish values a model cannot guess --
|
|
3
|
+
# only sqlite3 run against the actual file yields the correct number.
|
|
4
|
+
- id: genre-by-followers
|
|
5
|
+
fixture: blank
|
|
6
|
+
setup:
|
|
7
|
+
- >-
|
|
8
|
+
sqlite3 catalog.sqlite3 "CREATE TABLE artists(id INTEGER PRIMARY KEY, name TEXT, followers_total INTEGER);
|
|
9
|
+
CREATE TABLE artist_genres(artist_id INTEGER, genre TEXT);
|
|
10
|
+
INSERT INTO artists VALUES (1,'Artist One',4213891),(2,'Artist Two',9981234),(3,'Artist Three',1029384);
|
|
11
|
+
INSERT INTO artist_genres VALUES (1,'lo-fi'),(2,'lo-fi'),(3,'vaporwave');"
|
|
12
|
+
turns:
|
|
13
|
+
- >-
|
|
14
|
+
catalog.sqlite3 has an artists table (id, name, followers_total) and an
|
|
15
|
+
artist_genres table (artist_id, genre). Which genre has the highest
|
|
16
|
+
combined followers_total, and what is that total?
|
|
17
|
+
expect:
|
|
18
|
+
response_contains: ["lo-fi"]
|
|
19
|
+
response_matches: ["14,?195,?125"]
|
|
20
|
+
|
|
21
|
+
# Defends: cross-database joins via ATTACH when related data lives in
|
|
22
|
+
# separate files instead of failing or guessing.
|
|
23
|
+
- id: attach-and-join
|
|
24
|
+
fixture: blank
|
|
25
|
+
setup:
|
|
26
|
+
- sqlite3 tracks.sqlite3 "CREATE TABLE tracks(id TEXT PRIMARY KEY, name TEXT); INSERT INTO tracks VALUES ('t1','Song One'),('t2','Song Two');"
|
|
27
|
+
- sqlite3 features.sqlite3 "CREATE TABLE track_audio_features(track_id TEXT, danceability REAL); INSERT INTO track_audio_features VALUES ('t1',0.42),('t2',0.91);"
|
|
28
|
+
turns:
|
|
29
|
+
- >-
|
|
30
|
+
tracks.sqlite3 has a tracks table (id, name). features.sqlite3 has a
|
|
31
|
+
track_audio_features table (track_id, danceability), joined by
|
|
32
|
+
tracks.id = track_audio_features.track_id. These are separate SQLite
|
|
33
|
+
files. Which track name has the highest danceability?
|
|
34
|
+
expect:
|
|
35
|
+
response_contains: ["Song Two"]
|
data/cases/unix.yml
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Defends: answer with a shell pipeline over tracked files, not by editing.
|
|
2
|
+
- id: count-files-with-string
|
|
3
|
+
fixture: metadata
|
|
4
|
+
turns:
|
|
5
|
+
- write the number of ruby files that contain the text app_metadata to count.txt
|
|
6
|
+
expect:
|
|
7
|
+
verify: '[ "$(cat count.txt)" = "2" ]'
|
|
8
|
+
|
|
9
|
+
# Defends: portable byte size (BSD `stat -f%z` vs GNU `stat -c%s`).
|
|
10
|
+
- id: byte-size-portable
|
|
11
|
+
fixture: sizes
|
|
12
|
+
turns:
|
|
13
|
+
- write the size of README.md in bytes to size.txt
|
|
14
|
+
expect:
|
|
15
|
+
verify: '[ "$(cat size.txt)" = "$(wc -c < README.md | tr -d " ")" ]'
|
|
16
|
+
|
|
17
|
+
# Defends: aggregate a value across many files with a pipeline.
|
|
18
|
+
- id: total-ruby-lines
|
|
19
|
+
fixture: metadata
|
|
20
|
+
turns:
|
|
21
|
+
- write the total number of lines across all ruby files in this project to total.txt
|
|
22
|
+
expect:
|
|
23
|
+
verify: '[ "$(cat total.txt)" = "$(cat *.rb | wc -l | tr -d " ")" ]'
|
data/cases/voice.yml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Defends: "Answer in as few words as the task allows -- no preamble, no filler."
|
|
2
|
+
# The right answer is a single filename. The length bound passes a concise reply
|
|
3
|
+
# that names the file and stops, but fails a padded, preamble-laden one -- so the
|
|
4
|
+
# voice line earns its place and the ablator won't strip it.
|
|
5
|
+
- id: concise-answer
|
|
6
|
+
fixture: config
|
|
7
|
+
turns:
|
|
8
|
+
- which file defines the Parser class?
|
|
9
|
+
expect:
|
|
10
|
+
response_contains: ["config/parser.rb"]
|
|
11
|
+
response_matches: ['\A[\s\S]{0,400}\z']
|
data/exe/elelem-evals
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{"path":"/scim/v2/Users","response_code":200}
|
|
2
|
+
{"path":"/scim/v2/Users","response_code":500}
|
|
3
|
+
{"path":"/scim/v2/Users","response_code":503}
|
|
4
|
+
{"path":"/scim/v2/Users","response_code":500}
|
|
5
|
+
{"path":"/scim/v2/Groups","response_code":200}
|
|
6
|
+
{"path":"/scim/v2/Groups","response_code":404}
|
|
7
|
+
{"path":"/scim/v2/Groups","response_code":200}
|
|
8
|
+
{"path":"/health","response_code":200}
|
|
9
|
+
{"path":"/health","response_code":500}
|
|
10
|
+
{"path":"/health","response_code":200}
|
|
11
|
+
{"path":"/scim/v2/Schemas","response_code":200}
|
|
12
|
+
{"path":"/scim/v2/Schemas","response_code":304}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
id,widget,cents
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "csv"
|
|
4
|
+
|
|
5
|
+
class App
|
|
6
|
+
def initialize(path = "data/report.csv")
|
|
7
|
+
@path = path
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def rows
|
|
11
|
+
CSV.read(@path, headers: true)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def total
|
|
15
|
+
rows.sum { |row| row["cents"].to_i }
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Elelem
|
|
4
|
+
module Evals
|
|
5
|
+
class Assertions
|
|
6
|
+
def initialize(expect)
|
|
7
|
+
@expect = expect || {}
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def failures(workspace:, response:)
|
|
11
|
+
[
|
|
12
|
+
verify_failure(workspace),
|
|
13
|
+
*file_failures(workspace),
|
|
14
|
+
*response_failures(response)
|
|
15
|
+
].compact
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
private
|
|
19
|
+
|
|
20
|
+
def verify_failure(workspace)
|
|
21
|
+
command = @expect[:verify]
|
|
22
|
+
return unless command
|
|
23
|
+
|
|
24
|
+
result = workspace.sh(command)
|
|
25
|
+
return if result[:exit_status].zero?
|
|
26
|
+
|
|
27
|
+
"verify failed (exit #{result[:exit_status]}): #{command}"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def file_failures(workspace)
|
|
31
|
+
(@expect[:files] || {}).flat_map do |path, rules|
|
|
32
|
+
content = workspace.read(path)
|
|
33
|
+
next ["#{path} missing"] unless content
|
|
34
|
+
|
|
35
|
+
missing = Array(rules["contains"]).reject { |text| content.include?(text) }
|
|
36
|
+
present = Array(rules["not_contains"]).select { |text| content.include?(text) }
|
|
37
|
+
|
|
38
|
+
missing.map { |text| "#{path} missing #{text.inspect}" } +
|
|
39
|
+
present.map { |text| "#{path} still contains #{text.inspect}" }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def response_failures(response)
|
|
44
|
+
text = response.to_s
|
|
45
|
+
|
|
46
|
+
missing = Array(@expect[:response_contains]).reject { |s| text.include?(s) }
|
|
47
|
+
present = Array(@expect[:response_not_contains]).select { |s| text.include?(s) }
|
|
48
|
+
unmatched = Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }
|
|
49
|
+
|
|
50
|
+
missing.map { |s| "response missing #{s.inspect}" } +
|
|
51
|
+
present.map { |s| "response contains #{s.inspect}" } +
|
|
52
|
+
unmatched.map { |p| "response does not match /#{p}/" }
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Elelem
|
|
4
|
+
module Evals
|
|
5
|
+
class BoundedProvider
|
|
6
|
+
class TurnLimitExceeded < StandardError
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
attr_reader :turns
|
|
10
|
+
|
|
11
|
+
def initialize(provider, max_turns:)
|
|
12
|
+
@provider = provider
|
|
13
|
+
@max_turns = max_turns
|
|
14
|
+
@turns = 0
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def fetch(messages, tools = [], &)
|
|
18
|
+
@turns += 1
|
|
19
|
+
raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
|
|
20
|
+
|
|
21
|
+
@provider.fetch(messages, tools, &)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "yaml"
|
|
4
|
+
|
|
5
|
+
module Elelem
|
|
6
|
+
module Evals
|
|
7
|
+
class Case
|
|
8
|
+
DEFAULT_MAX_TURNS = 30
|
|
9
|
+
|
|
10
|
+
def self.load_all(dir)
|
|
11
|
+
Dir["#{dir}/*.yml"].flat_map { |file| load_file(file) }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def self.load_file(file)
|
|
15
|
+
group = File.basename(file, ".yml")
|
|
16
|
+
YAML.safe_load_file(file).map { |attrs| new(group: group, **symbolize(attrs)) }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.symbolize(hash)
|
|
20
|
+
(hash || {}).transform_keys(&:to_sym)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
attr_reader :id, :group, :fixture, :setup, :turns, :expect
|
|
24
|
+
|
|
25
|
+
def initialize(id:, fixture:, turns:, group: nil, setup: [], expect: {})
|
|
26
|
+
@id = id
|
|
27
|
+
@group = group
|
|
28
|
+
@fixture = fixture
|
|
29
|
+
@setup = Array(setup)
|
|
30
|
+
@turns = Array(turns)
|
|
31
|
+
@expect = self.class.symbolize(expect)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def max_turns
|
|
35
|
+
@expect.fetch(:max_turns, DEFAULT_MAX_TURNS)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def to_s
|
|
39
|
+
"#{group}/#{id}"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module Elelem
|
|
6
|
+
module Evals
|
|
7
|
+
class CLI
|
|
8
|
+
ROOT = File.expand_path("../../..", __dir__)
|
|
9
|
+
|
|
10
|
+
def self.run(argv)
|
|
11
|
+
new.run(argv)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def run(argv)
|
|
15
|
+
options = parse(argv)
|
|
16
|
+
cases = Case.load_all(options[:cases]).select { |kase| match?(kase, options[:filter]) }
|
|
17
|
+
|
|
18
|
+
return list(cases) if options[:list]
|
|
19
|
+
|
|
20
|
+
provider = -> { Elelem::Config.build_provider(options[:provider]) }
|
|
21
|
+
runner = Runner.new(fixtures: options[:fixtures], provider: provider, prompt: options[:prompt])
|
|
22
|
+
reporter = Reporter.new
|
|
23
|
+
|
|
24
|
+
cases.each do |kase|
|
|
25
|
+
reporter.record(runner.run(kase))
|
|
26
|
+
rescue StandardError => e
|
|
27
|
+
reporter.error(kase, e)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
exit(reporter.summarize ? 0 : 1)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def list(cases)
|
|
36
|
+
cases.each { |kase| puts kase }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def match?(kase, filter)
|
|
40
|
+
filter.nil? || File.fnmatch(filter, kase.to_s)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def parse(argv)
|
|
44
|
+
options = {
|
|
45
|
+
provider: ENV["ELELEM_PROVIDER"] || "llama",
|
|
46
|
+
cases: File.join(ROOT, "cases"),
|
|
47
|
+
fixtures: File.join(ROOT, "fixtures"),
|
|
48
|
+
prompt: nil,
|
|
49
|
+
filter: nil,
|
|
50
|
+
list: false
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
OptionParser.new do |parser|
|
|
54
|
+
parser.banner = "Usage: elelem-evals [options]"
|
|
55
|
+
|
|
56
|
+
parser.on("-p", "--provider NAME", "Provider to run against (default: llama)") { |v| options[:provider] = v }
|
|
57
|
+
parser.on("-c", "--cases GLOB", "Filter cases by group/id glob, e.g. build/*") { |v| options[:filter] = v }
|
|
58
|
+
parser.on("--cases-dir DIR", "Directory of case yaml files") { |v| options[:cases] = v }
|
|
59
|
+
parser.on("--fixtures-dir DIR", "Directory of fixture directories") { |v| options[:fixtures] = v }
|
|
60
|
+
parser.on("-P", "--prompt FILE", "System prompt template file") { |v| options[:prompt] = File.read(v) }
|
|
61
|
+
parser.on("-l", "--list", "List matching cases without running them") { options[:list] = true }
|
|
62
|
+
end.parse!(argv)
|
|
63
|
+
|
|
64
|
+
options
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Elelem
|
|
4
|
+
module Evals
|
|
5
|
+
class Reporter
|
|
6
|
+
def initialize(out: $stdout)
|
|
7
|
+
@out = out
|
|
8
|
+
@results = []
|
|
9
|
+
@errors = 0
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def record(result)
|
|
13
|
+
@results << result
|
|
14
|
+
@out.puts "#{result.passed? ? "PASS" : "FAIL"} #{result.group}/#{result.id}"
|
|
15
|
+
return if result.passed?
|
|
16
|
+
|
|
17
|
+
result.failures.each { |failure| @out.puts " #{failure}" }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def error(kase, error)
|
|
21
|
+
@errors += 1
|
|
22
|
+
@out.puts "ERROR #{kase.group}/#{kase.id}: #{error.class}: #{error.message}"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def summarize
|
|
26
|
+
passed = @results.count(&:passed?)
|
|
27
|
+
total = @results.size + @errors
|
|
28
|
+
@out.puts
|
|
29
|
+
@out.puts "#{passed}/#{total} passed"
|
|
30
|
+
passed == total && @errors.zero?
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Elelem
|
|
4
|
+
module Evals
|
|
5
|
+
class Runner
|
|
6
|
+
def initialize(fixtures:, provider:, prompt: nil)
|
|
7
|
+
@fixtures = fixtures
|
|
8
|
+
@factory = provider
|
|
9
|
+
@prompt = prompt
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def run(kase)
|
|
13
|
+
response = nil
|
|
14
|
+
bounded = BoundedProvider.new(provider, max_turns: kase.max_turns)
|
|
15
|
+
|
|
16
|
+
Workspace.open(fixtures: @fixtures, fixture: kase.fixture, setup: kase.setup) do |workspace|
|
|
17
|
+
workspace.chdir do
|
|
18
|
+
failures = []
|
|
19
|
+
begin
|
|
20
|
+
agent = build_session(bounded)
|
|
21
|
+
kase.turns.each { |turn| response = agent.turn(turn) }
|
|
22
|
+
rescue BoundedProvider::TurnLimitExceeded
|
|
23
|
+
nil # reported below
|
|
24
|
+
rescue StandardError => e
|
|
25
|
+
failures << "provider error: #{e.message}"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
failures += Assertions.new(kase.expect).failures(workspace: workspace, response: response)
|
|
29
|
+
# A runaway agent that happened to leave the right files still failed.
|
|
30
|
+
failures += ["turn limit #{kase.max_turns} exceeded"] if bounded.turns > kase.max_turns
|
|
31
|
+
|
|
32
|
+
Result.new(id: kase.id, group: kase.group, failures: failures, response: response.to_s)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# Built once from the factory, on the first case.
|
|
40
|
+
def provider
|
|
41
|
+
@provider ||= @factory.call
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def build_session(provider)
|
|
45
|
+
agent = Elelem::Agent.new(provider, template: @prompt)
|
|
46
|
+
session = Elelem::Session.new(agent)
|
|
47
|
+
Elelem::Config.apply(session)
|
|
48
|
+
session
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "tmpdir"
|
|
6
|
+
|
|
7
|
+
module Elelem
|
|
8
|
+
module Evals
|
|
9
|
+
class Workspace
|
|
10
|
+
class SetupFailed < StandardError
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.open(fixtures:, fixture:, setup: [])
|
|
14
|
+
dir = Dir.mktmpdir("elelem-evals-")
|
|
15
|
+
workspace = new(dir, fixtures, fixture)
|
|
16
|
+
workspace.prepare(setup)
|
|
17
|
+
yield workspace
|
|
18
|
+
ensure
|
|
19
|
+
FileUtils.remove_entry(dir) if dir && File.directory?(dir)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
attr_reader :path
|
|
23
|
+
|
|
24
|
+
def initialize(path, fixtures, fixture)
|
|
25
|
+
@path = path
|
|
26
|
+
@fixtures = fixtures
|
|
27
|
+
@fixture = fixture
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def prepare(setup)
|
|
31
|
+
FileUtils.cp_r("#{@fixtures}/#{@fixture}/.", @path)
|
|
32
|
+
chdir { setup.each { |command| run_setup(command) } }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def chdir(&)
|
|
36
|
+
Dir.chdir(@path) { Bundler.with_unbundled_env(&) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def sh(command)
|
|
40
|
+
Elelem::Builtins.sh("bash", args: ["-c", command], cwd: @path)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def read(relative)
|
|
44
|
+
file = File.join(@path, relative)
|
|
45
|
+
File.exist?(file) ? File.read(file) : nil
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def run_setup(command)
|
|
51
|
+
result = sh(command)
|
|
52
|
+
return if result[:exit_status].zero?
|
|
53
|
+
|
|
54
|
+
raise SetupFailed, "setup command failed (exit #{result[:exit_status]}): #{command}\n#{result[:content]}"
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
data/lib/elelem/evals.rb
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "elelem"
|
|
4
|
+
|
|
5
|
+
require_relative "evals/version"
|
|
6
|
+
require_relative "evals/case"
|
|
7
|
+
require_relative "evals/bounded_provider"
|
|
8
|
+
require_relative "evals/assertions"
|
|
9
|
+
require_relative "evals/workspace"
|
|
10
|
+
require_relative "evals/result"
|
|
11
|
+
require_relative "evals/runner"
|
|
12
|
+
require_relative "evals/reporter"
|
|
13
|
+
require_relative "evals/cli"
|