elelem-evals 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.agents/backlog/001-isolate-from-user-plugins.md +56 -0
  3. data/.agents/backlog/002-default-provider-and-help.md +39 -0
  4. data/.agents/backlog/003-unique-case-ids.md +47 -0
  5. data/.agents/backlog/004-gem-metadata-links.md +65 -0
  6. data/.github/workflows/release.yml +73 -0
  7. data/.release-please-manifest.json +3 -0
  8. data/.rspec +1 -0
  9. data/.rubocop.yml +62 -0
  10. data/CHANGELOG.md +22 -0
  11. data/LICENSE.txt +21 -0
  12. data/Rakefile +8 -0
  13. data/cases/build.yml +10 -0
  14. data/cases/create.yml +11 -0
  15. data/cases/edit.yml +36 -0
  16. data/cases/fix.yml +9 -0
  17. data/cases/git.yml +27 -0
  18. data/cases/holdout.yml +62 -0
  19. data/cases/implement.yml +10 -0
  20. data/cases/json.yml +7 -0
  21. data/cases/ops.yml +11 -0
  22. data/cases/refactor.yml +14 -0
  23. data/cases/search.yml +30 -0
  24. data/cases/shell.yml +17 -0
  25. data/cases/sql.yml +35 -0
  26. data/cases/unix.yml +23 -0
  27. data/cases/voice.yml +11 -0
  28. data/exe/elelem-evals +6 -0
  29. data/fixtures/blank/README.md +3 -0
  30. data/fixtures/broken/lib/calc.rb +3 -0
  31. data/fixtures/broken/test.rb +5 -0
  32. data/fixtures/config/app.rb +3 -0
  33. data/fixtures/config/config/loader.rb +11 -0
  34. data/fixtures/config/config/parser.rb +9 -0
  35. data/fixtures/greeter/lib.rb +3 -0
  36. data/fixtures/greeter/main.rb +3 -0
  37. data/fixtures/hello/hello.rb +5 -0
  38. data/fixtures/logs/requests.jsonl +12 -0
  39. data/fixtures/metadata/session.rb +10 -0
  40. data/fixtures/metadata/token.rb +9 -0
  41. data/fixtures/metadata/user.rb +9 -0
  42. data/fixtures/roman/roman.rb +3 -0
  43. data/fixtures/roman/roman_test.rb +8 -0
  44. data/fixtures/sizes/README.md +3 -0
  45. data/fixtures/sizes/data/report.csv +1 -0
  46. data/fixtures/sizes/lib/app.rb +17 -0
  47. data/lib/elelem/evals/assertions.rb +56 -0
  48. data/lib/elelem/evals/bounded_provider.rb +25 -0
  49. data/lib/elelem/evals/case.rb +43 -0
  50. data/lib/elelem/evals/cli.rb +68 -0
  51. data/lib/elelem/evals/reporter.rb +34 -0
  52. data/lib/elelem/evals/result.rb +11 -0
  53. data/lib/elelem/evals/runner.rb +52 -0
  54. data/lib/elelem/evals/version.rb +7 -0
  55. data/lib/elelem/evals/workspace.rb +58 -0
  56. data/lib/elelem/evals.rb +13 -0
  57. data/release-please-config.json +12 -0
  58. data/spec/elelem/evals/runner_spec.rb +50 -0
  59. data/spec/spec_helper.rb +11 -0
  60. metadata +137 -0
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tmpdir"
4
+
5
+ RSpec.describe Elelem::Evals::Runner do
6
+ around { |example| Dir.mktmpdir { |dir| @fixtures = dir and example.run } }
7
+
8
+ # Runs the turn's text as one shell command, then reports, like a model would.
9
+ let(:scripted) do
10
+ Class.new do
11
+ def fetch(messages, _tools)
12
+ return yield(type: "saying", text: "done") if messages.last[:role] == "tool"
13
+
14
+ prompt = messages.rfind { |message| message[:role] == "user" }[:content]
15
+ yield(type: "doing", id: "call-#{messages.length}", name: "execute",
16
+ arguments: { "command" => prompt.delete_prefix("execute ") })
17
+ end
18
+ end
19
+ end
20
+ let(:built) { [] }
21
+ let(:factory) { -> { built << :provider and scripted.new } }
22
+ let(:runner) { described_class.new(fixtures: @fixtures, provider: factory) }
23
+
24
+ def kase(**attrs)
25
+ Elelem::Evals::Case.new(id: "c", fixture: nil, turns: ["execute ls"], **attrs)
26
+ end
27
+
28
+ it "builds the provider from its factory once, and runs the case with it" do
29
+ 2.times { runner.run(kase) }
30
+
31
+ expect(built).to eq([:provider])
32
+ end
33
+
34
+ it "fails a case that goes past max_turns, even when its checks pass" do
35
+ result = runner.run(kase(expect: { "max_turns" => 1 }))
36
+
37
+ expect(result.failures).to eq(["turn limit 1 exceeded"])
38
+ end
39
+
40
+ it "fails a case whose provider raises, and keeps running" do
41
+ broken = Class.new { def fetch(*) = raise("provider down") }
42
+ runner = described_class.new(fixtures: @fixtures, provider: -> { broken.new })
43
+
44
+ expect(runner.run(kase).failures).to eq(["provider error: provider down"])
45
+ end
46
+
47
+ it "passes a case within max_turns" do
48
+ expect(runner.run(kase(expect: { "max_turns" => 5 }))).to be_passed
49
+ end
50
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../lib/elelem/evals"
4
+
5
+ RSpec.configure do |config|
6
+ config.disable_monkey_patching!
7
+
8
+ config.expect_with :rspec do |c|
9
+ c.syntax = :expect
10
+ end
11
+ end
metadata ADDED
@@ -0,0 +1,137 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: elelem-evals
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - mo khan
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: elelem
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0.13'
19
+ - - "<"
20
+ - !ruby/object:Gem::Version
21
+ version: '1'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ requirements:
26
+ - - ">="
27
+ - !ruby/object:Gem::Version
28
+ version: '0.13'
29
+ - - "<"
30
+ - !ruby/object:Gem::Version
31
+ version: '1'
32
+ - !ruby/object:Gem::Dependency
33
+ name: yaml
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - "~>"
37
+ - !ruby/object:Gem::Version
38
+ version: '0.4'
39
+ type: :runtime
40
+ prerelease: false
41
+ version_requirements: !ruby/object:Gem::Requirement
42
+ requirements:
43
+ - - "~>"
44
+ - !ruby/object:Gem::Version
45
+ version: '0.4'
46
+ description: Runs scenario-based evals against any elelem provider.
47
+ email:
48
+ - mo@mokhan.ca
49
+ executables:
50
+ - elelem-evals
51
+ extensions: []
52
+ extra_rdoc_files: []
53
+ files:
54
+ - ".agents/backlog/001-isolate-from-user-plugins.md"
55
+ - ".agents/backlog/002-default-provider-and-help.md"
56
+ - ".agents/backlog/003-unique-case-ids.md"
57
+ - ".agents/backlog/004-gem-metadata-links.md"
58
+ - ".github/workflows/release.yml"
59
+ - ".release-please-manifest.json"
60
+ - ".rspec"
61
+ - ".rubocop.yml"
62
+ - CHANGELOG.md
63
+ - LICENSE.txt
64
+ - Rakefile
65
+ - cases/build.yml
66
+ - cases/create.yml
67
+ - cases/edit.yml
68
+ - cases/fix.yml
69
+ - cases/git.yml
70
+ - cases/holdout.yml
71
+ - cases/implement.yml
72
+ - cases/json.yml
73
+ - cases/ops.yml
74
+ - cases/refactor.yml
75
+ - cases/search.yml
76
+ - cases/shell.yml
77
+ - cases/sql.yml
78
+ - cases/unix.yml
79
+ - cases/voice.yml
80
+ - exe/elelem-evals
81
+ - fixtures/blank/README.md
82
+ - fixtures/broken/lib/calc.rb
83
+ - fixtures/broken/test.rb
84
+ - fixtures/config/app.rb
85
+ - fixtures/config/config/loader.rb
86
+ - fixtures/config/config/parser.rb
87
+ - fixtures/greeter/lib.rb
88
+ - fixtures/greeter/main.rb
89
+ - fixtures/hello/hello.rb
90
+ - fixtures/logs/requests.jsonl
91
+ - fixtures/metadata/session.rb
92
+ - fixtures/metadata/token.rb
93
+ - fixtures/metadata/user.rb
94
+ - fixtures/roman/roman.rb
95
+ - fixtures/roman/roman_test.rb
96
+ - fixtures/sizes/README.md
97
+ - fixtures/sizes/data/report.csv
98
+ - fixtures/sizes/lib/app.rb
99
+ - lib/elelem/evals.rb
100
+ - lib/elelem/evals/assertions.rb
101
+ - lib/elelem/evals/bounded_provider.rb
102
+ - lib/elelem/evals/case.rb
103
+ - lib/elelem/evals/cli.rb
104
+ - lib/elelem/evals/reporter.rb
105
+ - lib/elelem/evals/result.rb
106
+ - lib/elelem/evals/runner.rb
107
+ - lib/elelem/evals/version.rb
108
+ - lib/elelem/evals/workspace.rb
109
+ - release-please-config.json
110
+ - spec/elelem/evals/runner_spec.rb
111
+ - spec/spec_helper.rb
112
+ homepage: https://elelem.sh
113
+ licenses:
114
+ - MIT
115
+ metadata:
116
+ allowed_push_host: https://rubygems.org
117
+ homepage_uri: https://elelem.sh
118
+ source_code_uri: https://github.com/elelem-sh/evals
119
+ rubygems_mfa_required: 'true'
120
+ rdoc_options: []
121
+ require_paths:
122
+ - lib
123
+ required_ruby_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ version: 4.0.0
128
+ required_rubygems_version: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ requirements: []
134
+ rubygems_version: 4.0.6
135
+ specification_version: 4
136
+ summary: Eval harness for elelem providers.
137
+ test_files: []