elelem-evals 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.agents/backlog/001-isolate-from-user-plugins.md +56 -0
- data/.agents/backlog/002-default-provider-and-help.md +39 -0
- data/.agents/backlog/003-unique-case-ids.md +47 -0
- data/.agents/backlog/004-gem-metadata-links.md +65 -0
- data/.github/workflows/release.yml +73 -0
- data/.release-please-manifest.json +3 -0
- data/.rspec +1 -0
- data/.rubocop.yml +62 -0
- data/CHANGELOG.md +22 -0
- data/LICENSE.txt +21 -0
- data/Rakefile +8 -0
- data/cases/build.yml +10 -0
- data/cases/create.yml +11 -0
- data/cases/edit.yml +36 -0
- data/cases/fix.yml +9 -0
- data/cases/git.yml +27 -0
- data/cases/holdout.yml +62 -0
- data/cases/implement.yml +10 -0
- data/cases/json.yml +7 -0
- data/cases/ops.yml +11 -0
- data/cases/refactor.yml +14 -0
- data/cases/search.yml +30 -0
- data/cases/shell.yml +17 -0
- data/cases/sql.yml +35 -0
- data/cases/unix.yml +23 -0
- data/cases/voice.yml +11 -0
- data/exe/elelem-evals +6 -0
- data/fixtures/blank/README.md +3 -0
- data/fixtures/broken/lib/calc.rb +3 -0
- data/fixtures/broken/test.rb +5 -0
- data/fixtures/config/app.rb +3 -0
- data/fixtures/config/config/loader.rb +11 -0
- data/fixtures/config/config/parser.rb +9 -0
- data/fixtures/greeter/lib.rb +3 -0
- data/fixtures/greeter/main.rb +3 -0
- data/fixtures/hello/hello.rb +5 -0
- data/fixtures/logs/requests.jsonl +12 -0
- data/fixtures/metadata/session.rb +10 -0
- data/fixtures/metadata/token.rb +9 -0
- data/fixtures/metadata/user.rb +9 -0
- data/fixtures/roman/roman.rb +3 -0
- data/fixtures/roman/roman_test.rb +8 -0
- data/fixtures/sizes/README.md +3 -0
- data/fixtures/sizes/data/report.csv +1 -0
- data/fixtures/sizes/lib/app.rb +17 -0
- data/lib/elelem/evals/assertions.rb +56 -0
- data/lib/elelem/evals/bounded_provider.rb +25 -0
- data/lib/elelem/evals/case.rb +43 -0
- data/lib/elelem/evals/cli.rb +68 -0
- data/lib/elelem/evals/reporter.rb +34 -0
- data/lib/elelem/evals/result.rb +11 -0
- data/lib/elelem/evals/runner.rb +52 -0
- data/lib/elelem/evals/version.rb +7 -0
- data/lib/elelem/evals/workspace.rb +58 -0
- data/lib/elelem/evals.rb +13 -0
- data/release-please-config.json +12 -0
- data/spec/elelem/evals/runner_spec.rb +50 -0
- data/spec/spec_helper.rb +11 -0
- metadata +137 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
|
|
5
|
+
RSpec.describe Elelem::Evals::Runner do
|
|
6
|
+
around { |example| Dir.mktmpdir { |dir| @fixtures = dir and example.run } }
|
|
7
|
+
|
|
8
|
+
# Runs the turn's text as one shell command, then reports, like a model would.
|
|
9
|
+
let(:scripted) do
|
|
10
|
+
Class.new do
|
|
11
|
+
def fetch(messages, _tools)
|
|
12
|
+
return yield(type: "saying", text: "done") if messages.last[:role] == "tool"
|
|
13
|
+
|
|
14
|
+
prompt = messages.rfind { |message| message[:role] == "user" }[:content]
|
|
15
|
+
yield(type: "doing", id: "call-#{messages.length}", name: "execute",
|
|
16
|
+
arguments: { "command" => prompt.delete_prefix("execute ") })
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
let(:built) { [] }
|
|
21
|
+
let(:factory) { -> { built << :provider and scripted.new } }
|
|
22
|
+
let(:runner) { described_class.new(fixtures: @fixtures, provider: factory) }
|
|
23
|
+
|
|
24
|
+
def kase(**attrs)
|
|
25
|
+
Elelem::Evals::Case.new(id: "c", fixture: nil, turns: ["execute ls"], **attrs)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
it "builds the provider from its factory once, and runs the case with it" do
|
|
29
|
+
2.times { runner.run(kase) }
|
|
30
|
+
|
|
31
|
+
expect(built).to eq([:provider])
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
it "fails a case that goes past max_turns, even when its checks pass" do
|
|
35
|
+
result = runner.run(kase(expect: { "max_turns" => 1 }))
|
|
36
|
+
|
|
37
|
+
expect(result.failures).to eq(["turn limit 1 exceeded"])
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
it "fails a case whose provider raises, and keeps running" do
|
|
41
|
+
broken = Class.new { def fetch(*) = raise("provider down") }
|
|
42
|
+
runner = described_class.new(fixtures: @fixtures, provider: -> { broken.new })
|
|
43
|
+
|
|
44
|
+
expect(runner.run(kase).failures).to eq(["provider error: provider down"])
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
it "passes a case within max_turns" do
|
|
48
|
+
expect(runner.run(kase(expect: { "max_turns" => 5 }))).to be_passed
|
|
49
|
+
end
|
|
50
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: elelem-evals
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- mo khan
|
|
8
|
+
bindir: exe
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: elelem
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '0.13'
|
|
19
|
+
- - "<"
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: '1'
|
|
22
|
+
type: :runtime
|
|
23
|
+
prerelease: false
|
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
25
|
+
requirements:
|
|
26
|
+
- - ">="
|
|
27
|
+
- !ruby/object:Gem::Version
|
|
28
|
+
version: '0.13'
|
|
29
|
+
- - "<"
|
|
30
|
+
- !ruby/object:Gem::Version
|
|
31
|
+
version: '1'
|
|
32
|
+
- !ruby/object:Gem::Dependency
|
|
33
|
+
name: yaml
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
35
|
+
requirements:
|
|
36
|
+
- - "~>"
|
|
37
|
+
- !ruby/object:Gem::Version
|
|
38
|
+
version: '0.4'
|
|
39
|
+
type: :runtime
|
|
40
|
+
prerelease: false
|
|
41
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
42
|
+
requirements:
|
|
43
|
+
- - "~>"
|
|
44
|
+
- !ruby/object:Gem::Version
|
|
45
|
+
version: '0.4'
|
|
46
|
+
description: Runs scenario-based evals against any elelem provider.
|
|
47
|
+
email:
|
|
48
|
+
- mo@mokhan.ca
|
|
49
|
+
executables:
|
|
50
|
+
- elelem-evals
|
|
51
|
+
extensions: []
|
|
52
|
+
extra_rdoc_files: []
|
|
53
|
+
files:
|
|
54
|
+
- ".agents/backlog/001-isolate-from-user-plugins.md"
|
|
55
|
+
- ".agents/backlog/002-default-provider-and-help.md"
|
|
56
|
+
- ".agents/backlog/003-unique-case-ids.md"
|
|
57
|
+
- ".agents/backlog/004-gem-metadata-links.md"
|
|
58
|
+
- ".github/workflows/release.yml"
|
|
59
|
+
- ".release-please-manifest.json"
|
|
60
|
+
- ".rspec"
|
|
61
|
+
- ".rubocop.yml"
|
|
62
|
+
- CHANGELOG.md
|
|
63
|
+
- LICENSE.txt
|
|
64
|
+
- Rakefile
|
|
65
|
+
- cases/build.yml
|
|
66
|
+
- cases/create.yml
|
|
67
|
+
- cases/edit.yml
|
|
68
|
+
- cases/fix.yml
|
|
69
|
+
- cases/git.yml
|
|
70
|
+
- cases/holdout.yml
|
|
71
|
+
- cases/implement.yml
|
|
72
|
+
- cases/json.yml
|
|
73
|
+
- cases/ops.yml
|
|
74
|
+
- cases/refactor.yml
|
|
75
|
+
- cases/search.yml
|
|
76
|
+
- cases/shell.yml
|
|
77
|
+
- cases/sql.yml
|
|
78
|
+
- cases/unix.yml
|
|
79
|
+
- cases/voice.yml
|
|
80
|
+
- exe/elelem-evals
|
|
81
|
+
- fixtures/blank/README.md
|
|
82
|
+
- fixtures/broken/lib/calc.rb
|
|
83
|
+
- fixtures/broken/test.rb
|
|
84
|
+
- fixtures/config/app.rb
|
|
85
|
+
- fixtures/config/config/loader.rb
|
|
86
|
+
- fixtures/config/config/parser.rb
|
|
87
|
+
- fixtures/greeter/lib.rb
|
|
88
|
+
- fixtures/greeter/main.rb
|
|
89
|
+
- fixtures/hello/hello.rb
|
|
90
|
+
- fixtures/logs/requests.jsonl
|
|
91
|
+
- fixtures/metadata/session.rb
|
|
92
|
+
- fixtures/metadata/token.rb
|
|
93
|
+
- fixtures/metadata/user.rb
|
|
94
|
+
- fixtures/roman/roman.rb
|
|
95
|
+
- fixtures/roman/roman_test.rb
|
|
96
|
+
- fixtures/sizes/README.md
|
|
97
|
+
- fixtures/sizes/data/report.csv
|
|
98
|
+
- fixtures/sizes/lib/app.rb
|
|
99
|
+
- lib/elelem/evals.rb
|
|
100
|
+
- lib/elelem/evals/assertions.rb
|
|
101
|
+
- lib/elelem/evals/bounded_provider.rb
|
|
102
|
+
- lib/elelem/evals/case.rb
|
|
103
|
+
- lib/elelem/evals/cli.rb
|
|
104
|
+
- lib/elelem/evals/reporter.rb
|
|
105
|
+
- lib/elelem/evals/result.rb
|
|
106
|
+
- lib/elelem/evals/runner.rb
|
|
107
|
+
- lib/elelem/evals/version.rb
|
|
108
|
+
- lib/elelem/evals/workspace.rb
|
|
109
|
+
- release-please-config.json
|
|
110
|
+
- spec/elelem/evals/runner_spec.rb
|
|
111
|
+
- spec/spec_helper.rb
|
|
112
|
+
homepage: https://elelem.sh
|
|
113
|
+
licenses:
|
|
114
|
+
- MIT
|
|
115
|
+
metadata:
|
|
116
|
+
allowed_push_host: https://rubygems.org
|
|
117
|
+
homepage_uri: https://elelem.sh
|
|
118
|
+
source_code_uri: https://github.com/elelem-sh/evals
|
|
119
|
+
rubygems_mfa_required: 'true'
|
|
120
|
+
rdoc_options: []
|
|
121
|
+
require_paths:
|
|
122
|
+
- lib
|
|
123
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
124
|
+
requirements:
|
|
125
|
+
- - ">="
|
|
126
|
+
- !ruby/object:Gem::Version
|
|
127
|
+
version: 4.0.0
|
|
128
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
129
|
+
requirements:
|
|
130
|
+
- - ">="
|
|
131
|
+
- !ruby/object:Gem::Version
|
|
132
|
+
version: '0'
|
|
133
|
+
requirements: []
|
|
134
|
+
rubygems_version: 4.0.6
|
|
135
|
+
specification_version: 4
|
|
136
|
+
summary: Eval harness for elelem providers.
|
|
137
|
+
test_files: []
|