elelem-evals 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.agents/backlog/001-isolate-from-user-plugins.md +56 -0
  3. data/.agents/backlog/002-default-provider-and-help.md +39 -0
  4. data/.agents/backlog/003-unique-case-ids.md +47 -0
  5. data/.agents/backlog/004-gem-metadata-links.md +65 -0
  6. data/.github/workflows/release.yml +73 -0
  7. data/.release-please-manifest.json +3 -0
  8. data/.rspec +1 -0
  9. data/.rubocop.yml +62 -0
  10. data/CHANGELOG.md +22 -0
  11. data/LICENSE.txt +21 -0
  12. data/Rakefile +8 -0
  13. data/cases/build.yml +10 -0
  14. data/cases/create.yml +11 -0
  15. data/cases/edit.yml +36 -0
  16. data/cases/fix.yml +9 -0
  17. data/cases/git.yml +27 -0
  18. data/cases/holdout.yml +62 -0
  19. data/cases/implement.yml +10 -0
  20. data/cases/json.yml +7 -0
  21. data/cases/ops.yml +11 -0
  22. data/cases/refactor.yml +14 -0
  23. data/cases/search.yml +30 -0
  24. data/cases/shell.yml +17 -0
  25. data/cases/sql.yml +35 -0
  26. data/cases/unix.yml +23 -0
  27. data/cases/voice.yml +11 -0
  28. data/exe/elelem-evals +6 -0
  29. data/fixtures/blank/README.md +3 -0
  30. data/fixtures/broken/lib/calc.rb +3 -0
  31. data/fixtures/broken/test.rb +5 -0
  32. data/fixtures/config/app.rb +3 -0
  33. data/fixtures/config/config/loader.rb +11 -0
  34. data/fixtures/config/config/parser.rb +9 -0
  35. data/fixtures/greeter/lib.rb +3 -0
  36. data/fixtures/greeter/main.rb +3 -0
  37. data/fixtures/hello/hello.rb +5 -0
  38. data/fixtures/logs/requests.jsonl +12 -0
  39. data/fixtures/metadata/session.rb +10 -0
  40. data/fixtures/metadata/token.rb +9 -0
  41. data/fixtures/metadata/user.rb +9 -0
  42. data/fixtures/roman/roman.rb +3 -0
  43. data/fixtures/roman/roman_test.rb +8 -0
  44. data/fixtures/sizes/README.md +3 -0
  45. data/fixtures/sizes/data/report.csv +1 -0
  46. data/fixtures/sizes/lib/app.rb +17 -0
  47. data/lib/elelem/evals/assertions.rb +56 -0
  48. data/lib/elelem/evals/bounded_provider.rb +25 -0
  49. data/lib/elelem/evals/case.rb +43 -0
  50. data/lib/elelem/evals/cli.rb +68 -0
  51. data/lib/elelem/evals/reporter.rb +34 -0
  52. data/lib/elelem/evals/result.rb +11 -0
  53. data/lib/elelem/evals/runner.rb +52 -0
  54. data/lib/elelem/evals/version.rb +7 -0
  55. data/lib/elelem/evals/workspace.rb +58 -0
  56. data/lib/elelem/evals.rb +13 -0
  57. data/release-please-config.json +12 -0
  58. data/spec/elelem/evals/runner_spec.rb +50 -0
  59. data/spec/spec_helper.rb +11 -0
  60. metadata +137 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 40e245d5b4f226f34ecd00d6cc0a86f81f79c08155d48de3beed133afbb2c664
4
+ data.tar.gz: 40a35b663bc6d8d8697eb6d57fd4b019a62954164ade01b06731b6f3ad61f14a
5
+ SHA512:
6
+ metadata.gz: 99977014077d828621bc0d4893a1b13d2a74040957b38d3a5f005f49d74c2df962406345568a5119982bc43426773284260196f26f96d2477216dab2d21a18e0
7
+ data.tar.gz: 0432fcdc50f1eaab289490e0148300764168cd0eab9621f1659b913030c78ac0af36cb7ec70e959745841bd2f945eaf1908fe0ff011202a2042b5d2c889f95fb
@@ -0,0 +1,56 @@
1
+ As a `maintainer comparing eval results across machines`, I `want
2
+ elelem-evals to load only the plugins the run names`, so that `results
3
+ do not change with what is in someone's ~/.agents/plugins`.
4
+
5
+ # SYNOPSIS
6
+
7
+ Load only explicit plugins during an eval run, and document that the
8
+ agent is not sandboxed.
9
+
10
+ # DESCRIPTION
11
+
12
+ The runner builds the provider with `Elelem::Config.build_provider`
13
+ and applies setups with `Elelem::Config.apply` (cli.rb:20,
14
+ runner.rb:47). Both call core's plugin loader, which loads every `.rb`
15
+ in core's plugins directory, `~/.agents/plugins` and `./.agents/plugins`
16
+ (elelem lib/elelem/plugins.rb:5-9). The provider is built before the
17
+ case's workspace is entered, so the user's own plugins and the plugins
18
+ of whatever directory the run starts in all join the agent. Two people
19
+ running the same cases against the same model can get different tools,
20
+ prompts and permission hooks, and so different results.
21
+
22
+ The agent also runs with full shell tools, unsandboxed, in a temporary
23
+ copy of the fixture (workspace.rb). A model can run any command as the
24
+ user. That is acceptable for local use but must be stated.
25
+
26
+ Proposed:
27
+
28
+ * Build the eval session with a plugin loader whose load paths are
29
+ only core's built-in plugins plus plugins named on the command line
30
+ (for example `--plugin elelem/tools`).
31
+ * Print the loaded plugins at the start of a run, so a report says
32
+ what it measured.
33
+ * Document the sandboxing caveat in the README (none exists yet) and in
34
+ `--help`.
35
+
36
+ # SEE ALSO
37
+
38
+ * [ ] lib/elelem/evals/cli.rb:20 - build_provider
39
+ * [ ] lib/elelem/evals/runner.rb:44-49 - build_session, Config.apply
40
+ * [ ] lib/elelem/evals/workspace.rb - temporary fixture copy
41
+ * [ ] elelem lib/elelem/plugins.rb:5-9 - plugin load paths
42
+ * [ ] Formerly elelem-sh/elelem.sh#36
43
+
44
+ # Tasks
45
+
46
+ * [ ] TBD (filled in design mode)
47
+
48
+ # Acceptance Criteria
49
+
50
+ * [ ] A plugin in `~/.agents/plugins` is not loaded during an eval run
51
+ * [ ] A plugin in `./.agents/plugins` of the starting directory is not
52
+ loaded
53
+ * [ ] A plugin named on the command line is loaded
54
+ * [ ] The run prints the plugins it loaded
55
+ * [ ] The README and `--help` state that the agent runs unsandboxed
56
+ * [ ] Specs cover plugin selection
@@ -0,0 +1,39 @@
1
+ As a `user running elelem-evals`, I `want the --provider help to show
2
+ the provider that will really be used`, so that `I know what I am
3
+ measuring before a long run starts`.
4
+
5
+ # SYNOPSIS
6
+
7
+ Show the effective default provider in `--help`, and reconsider the
8
+ `llama` default.
9
+
10
+ # DESCRIPTION
11
+
12
+ The default provider is `ENV["ELELEM_PROVIDER"] || "llama"`
13
+ (cli.rb:45), but the help text is fixed: "Provider to run against
14
+ (default: llama)" (cli.rb:56). With `ELELEM_PROVIDER=ollama` set, the
15
+ help still says llama. elelem-server already builds its help from the
16
+ effective value (elelem-server exe/elelem-server:13).
17
+
18
+ `llama` as the built-in default also assumes elelem-llama is installed
19
+ and a model is available. Decide in design whether to keep it, or to
20
+ require `-p` or `ELELEM_PROVIDER` and fail with a clear message.
21
+
22
+ # SEE ALSO
23
+
24
+ * [ ] lib/elelem/evals/cli.rb:45 - default provider
25
+ * [ ] lib/elelem/evals/cli.rb:56 - fixed help text
26
+ * [ ] elelem-server exe/elelem-server:13 - help from the real default
27
+ * [ ] Formerly elelem-sh/elelem.sh#36
28
+
29
+ # Tasks
30
+
31
+ * [ ] TBD (filled in design mode)
32
+
33
+ # Acceptance Criteria
34
+
35
+ * [ ] `ELELEM_PROVIDER=ollama elelem-evals --help` shows ollama as the
36
+ default
37
+ * [ ] Without `ELELEM_PROVIDER`, the help shows the built-in default
38
+ (or says a provider is required, per the design decision)
39
+ * [ ] A spec covers the help text with and without the variable
@@ -0,0 +1,47 @@
1
+ As a `maintainer reading eval reports`, I `want every case id to be
2
+ unique`, so that `a result line points at exactly one case`.
3
+
4
+ # SYNOPSIS
5
+
6
+ Rename the duplicate `count-ruby-files` case and reject duplicate ids
7
+ on load.
8
+
9
+ # DESCRIPTION
10
+
11
+ The id `count-ruby-files` appears in cases/search.yml:10 and in
12
+ cases/holdout.yml:50. Results are reported by group and id, and `-c`
13
+ filters on them, so the two are easy to confuse; any tooling that keys
14
+ on the id alone merges them.
15
+
16
+ Also in the case files: comments refer to an "improver"
17
+ (holdout.yml:1, :14, :27) and an "ablator" (voice.yml:4) that are not
18
+ in this repository. Either reword them or say where those tools live.
19
+
20
+ Proposed:
21
+
22
+ * Rename one of the cases (the holdout copy is the natural choice,
23
+ for example `holdout-count-ruby-files`).
24
+ * Make `Case.load_all` raise when two cases share an id.
25
+ * Fix or remove the comments.
26
+
27
+ # SEE ALSO
28
+
29
+ * [ ] cases/search.yml:10 - count-ruby-files
30
+ * [ ] cases/holdout.yml:50 - count-ruby-files
31
+ * [ ] cases/holdout.yml:1 - "improver"
32
+ * [ ] cases/voice.yml:4 - "ablator"
33
+ * [ ] lib/elelem/evals/case.rb - load_all
34
+ * [ ] Formerly elelem-sh/elelem.sh#36
35
+
36
+ # Tasks
37
+
38
+ * [ ] TBD (filled in design mode)
39
+
40
+ # Acceptance Criteria
41
+
42
+ * [ ] No two cases under cases/ share an id
43
+ * [ ] Loading cases with a duplicate id fails with a message naming
44
+ the id and both files
45
+ * [ ] No comment refers to a tool that is not in the repository or
46
+ linked
47
+ * [ ] A spec covers the duplicate check
@@ -0,0 +1,65 @@
1
+ As a `person who finds elelem-evals on RubyGems`, I `want the
2
+ gem page to link to the plugin's documentation, backlog and
3
+ changelog`, so that `I can learn how to use it and see known
4
+ problems without GitHub Issues`.
5
+
6
+ # SYNOPSIS
7
+
8
+ Point the gemspec homepage at elelem.sh and add the standard links.
9
+
10
+ # DESCRIPTION
11
+
12
+ RubyGems shows `homepage_uri` and the other `metadata` links on every
13
+ gem page. Today `elelem-evals.gemspec` sets `spec.homepage`
14
+ to the site root (https://elelem.sh) and sets only `homepage_uri`,
15
+ `source_code_uri`, `allowed_push_host` and
16
+ `rubygems_mfa_required`.
17
+ There is no documentation, bug tracker or changelog link.
18
+
19
+ Proposed change:
20
+
21
+ ```ruby
22
+ spec.homepage = "https://elelem.sh/plugins/evals/"
23
+ spec.metadata["homepage_uri"] = spec.homepage
24
+ spec.metadata["source_code_uri"] = "https://github.com/elelem-sh/evals"
25
+ spec.metadata["documentation_uri"] = spec.homepage
26
+ spec.metadata["bug_tracker_uri"] = "https://github.com/elelem-sh/evals/tree/main/.agents/backlog"
27
+ spec.metadata["changelog_uri"] = "https://github.com/elelem-sh/evals/blob/main/CHANGELOG.md"
28
+ ```
29
+
30
+ The original proposal pointed `bug_tracker_uri` at GitHub Issues on
31
+ elelem-sh/elelem.sh, filtered by label. That no longer fits: the
32
+ project is moving its backlog out of GitHub Issues and into
33
+ `.agents/backlog` story files in each repository, so the bug tracker
34
+ link points at this repository's backlog directory instead.
35
+
36
+ /man/evals/ redirects to /plugins/evals/ on elelem.sh, so a
37
+ separate documentation_uri adds nothing. Point both at the plugin page.
38
+
39
+ There is no CHANGELOG.md on main yet. release-please writes one
40
+ on the first release, so ship the link with or after that release.
41
+
42
+ Keep `allowed_push_host` and `rubygems_mfa_required` as they are.
43
+ RubyGems only shows the new links after the next release.
44
+
45
+ # SEE ALSO
46
+
47
+ * [ ] elelem-evals.gemspec:13 - spec.homepage
48
+ * [ ] elelem-evals.gemspec:18 - source_code_uri
49
+ * [ ] .agents/backlog/ - where bug_tracker_uri will point
50
+ * [ ] Formerly elelem-sh/elelem.sh#41
51
+
52
+ # Tasks
53
+
54
+ * [ ] TBD (filled in design mode)
55
+
56
+ # Acceptance Criteria
57
+
58
+ * [ ] `spec.homepage` and `homepage_uri` are https://elelem.sh/plugins/evals/
59
+ * [ ] `bug_tracker_uri` points at this repository's `.agents/backlog`,
60
+ not at GitHub Issues
61
+ * [ ] `changelog_uri` and `source_code_uri` point at this repository
62
+ * [ ] `allowed_push_host` and `rubygems_mfa_required` are unchanged
63
+ * [ ] `gem build` succeeds and `gem spec` on the built gem shows every
64
+ link
65
+ * [ ] Every link resolves (no 404) at release time
@@ -0,0 +1,73 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ workflow_dispatch:
7
+
8
+ permissions: {}
9
+
10
+ concurrency:
11
+ group: release
12
+ cancel-in-progress: false
13
+
14
+ jobs:
15
+ test:
16
+ runs-on: ubuntu-latest
17
+ permissions:
18
+ contents: read
19
+ steps:
20
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
21
+ with:
22
+ persist-credentials: false
23
+ - uses: ruby/setup-ruby@3ff19f5e2baf30647122352b96108b1fbe250c64 # v1.299.0
24
+ with:
25
+ ruby-version: "4.0"
26
+ bundler-cache: true
27
+ - run: bin/test
28
+ - run: bin/lint
29
+
30
+ release-please:
31
+ needs: test
32
+ runs-on: ubuntu-latest
33
+ permissions:
34
+ contents: write
35
+ issues: write
36
+ pull-requests: write
37
+ outputs:
38
+ release_created: ${{ steps.release.outputs.release_created }}
39
+ steps:
40
+ - id: release
41
+ uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0
42
+ with:
43
+ config-file: release-please-config.json
44
+ manifest-file: .release-please-manifest.json
45
+
46
+ publish:
47
+ needs: release-please
48
+ if: needs.release-please.outputs.release_created == 'true' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')
49
+ runs-on: ubuntu-latest
50
+ permissions:
51
+ contents: write
52
+ id-token: write
53
+ steps:
54
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
55
+ with:
56
+ persist-credentials: false
57
+ - uses: ruby/setup-ruby@3ff19f5e2baf30647122352b96108b1fbe250c64 # v1.299.0
58
+ with:
59
+ ruby-version: "4.0"
60
+ bundler-cache: true
61
+ - shell: bash
62
+ run: |
63
+ set -euo pipefail
64
+ jwt=$(curl -sS --fail-with-body -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=rubygems.org" | jq -r .value)
65
+ echo "::add-mask::$jwt"
66
+ key=$(jq -n --arg jwt "$jwt" '{jwt: $jwt, owner: "66898"}' | curl -sS --fail-with-body -H "content-type: application/json" -d @- https://rubygems.org/api/v1/oidc/trusted_publisher/exchange_token | jq -r .rubygems_api_key)
67
+ test -n "$key" -a "$key" != null
68
+ echo "::add-mask::$key"
69
+ printf 'GEM_HOST_API_KEY=%s\nBUNDLE_GEM__PUSH_KEY=%s\nRUBYGEMS_API_KEY=%s\n' "$key" "$key" "$key" >> "$GITHUB_ENV"
70
+ - uses: rubygems/release-gem@7f9650160c1a4e7989fdc9855807bdbd421d8b6b # v1.4.1
71
+ with:
72
+ await-release: false
73
+ setup-trusted-publisher: false
@@ -0,0 +1,3 @@
1
+ {
2
+ ".": "0.1.0"
3
+ }
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --require spec_helper
data/.rubocop.yml ADDED
@@ -0,0 +1,62 @@
1
+ AllCops:
2
+ NewCops: enable
3
+ TargetRubyVersion: 4.0
4
+ SuggestExtensions: false
5
+ Exclude:
6
+ - "pkg/**/*"
7
+ - "bin/**/*"
8
+ - "vendor/**/*"
9
+ - "fixtures/**/*"
10
+
11
+ Metrics:
12
+ Enabled: false
13
+
14
+ Naming/PredicateMethod:
15
+ AllowedMethods:
16
+ - call
17
+ - run
18
+ - summarize
19
+ - prompt
20
+
21
+ Naming/MethodParameterName:
22
+ AllowedNames:
23
+ - me
24
+ - id
25
+ - as
26
+ - io
27
+
28
+ Style/GlobalVars:
29
+ Exclude:
30
+ - "spec/**/*"
31
+
32
+ Lint/EmptyBlock:
33
+ Exclude:
34
+ - "spec/**/*"
35
+
36
+ Lint/UselessDefaultValueArgument:
37
+ Enabled: false
38
+
39
+ Lint/RedundantRequireStatement:
40
+ Enabled: false
41
+
42
+ Lint/UnusedBlockArgument:
43
+ IgnoreEmptyBlocks: true
44
+ AllowUnusedKeywordArguments: true
45
+
46
+ Style/Documentation:
47
+ Enabled: false
48
+
49
+ Style/FormatStringToken:
50
+ Enabled: false
51
+
52
+ Style/FrozenStringLiteralComment:
53
+ Enabled: true
54
+
55
+ Style/StringLiterals:
56
+ EnforcedStyle: double_quotes
57
+
58
+ Style/StringLiteralsInInterpolation:
59
+ EnforcedStyle: double_quotes
60
+
61
+ Layout/LineLength:
62
+ Max: 120
data/CHANGELOG.md ADDED
@@ -0,0 +1,22 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0 (2026-11-16)
4
+
5
+
6
+ ### Features
7
+
8
+ * scaffold elelem-evals gem ([9249d19](https://github.com/elelem-sh/evals/commit/9249d19d8eb346b0d76cc1eecacb20a94d2b7345))
9
+
10
+
11
+ ### Bug Fixes
12
+
13
+ * a raising provider fails its case instead of the whole run ([fc5bcb2](https://github.com/elelem-sh/evals/commit/fc5bcb2f89ededb63a71aa94c5b140f2d9b8aa4b))
14
+ * adapt to elelem core's Config/Registry and Output/Input APIs ([923735d](https://github.com/elelem-sh/evals/commit/923735d26bd2ddbf0e735594929e590ce962a604))
15
+ * an eval case that goes past max_turns fails ([fd3d395](https://github.com/elelem-sh/evals/commit/fd3d395adb9a60d6d1be1eeb8291295295c18c13))
16
+ * build a Session before Config.apply ([38bb6ba](https://github.com/elelem-sh/evals/commit/38bb6ba43a024503cf05c1e06179740bd3d6fe28))
17
+ * depend on elelem-builtins for Workspace#sh ([39c5b6f](https://github.com/elelem-sh/evals/commit/39c5b6fa59b02dae68d251c869781060be20649f))
18
+ * finish client->provider rename and reuse provider across cases ([9f6f961](https://github.com/elelem-sh/evals/commit/9f6f961959ec3ca8e918e1a59cfc4aea04b05325))
19
+ * require bundler explicitly in Workspace ([f3192d7](https://github.com/elelem-sh/evals/commit/f3192d708b4686150b1b8c779bb08d252b873772))
20
+ * require elelem ~> 0.12, the floor builtins/skills actually need ([39cbe0c](https://github.com/elelem-sh/evals/commit/39cbe0cc43b18938f0417d17b549d6b9b49388f1))
21
+ * the eval runner calls its provider factory, so cases can run (elelem-sh/elelem.sh[#35](https://github.com/elelem-sh/evals/issues/35)) ([7968e3f](https://github.com/elelem-sh/evals/commit/7968e3fb289e4fbc432446ae0374cfde35bd1a60))
22
+ * use Registry#build_client's new name (was #build) ([e90e9a0](https://github.com/elelem-sh/evals/commit/e90e9a09a8e8af5f045f1212f1a36349777a88c2))
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2026 mo khan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/Rakefile ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler/gem_tasks"
4
+ require "rspec/core/rake_task"
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task default: %i[spec]
data/cases/build.yml ADDED
@@ -0,0 +1,10 @@
1
+ # Defends: run the tests, interpret the failure, and DON'T edit when only asked to diagnose.
2
+ - id: diagnose-failing-test
3
+ fixture: broken
4
+ turns:
5
+ - run test.rb and explain what is broken, but do not change any files
6
+ expect:
7
+ response_contains: ["add"]
8
+ files:
9
+ lib/calc.rb:
10
+ contains: ["a - b"]
data/cases/create.yml ADDED
@@ -0,0 +1,11 @@
1
+ # Defends: scaffold a new program from scratch and run it.
2
+ - id: fizzbuzz-from-scratch
3
+ fixture: blank
4
+ turns:
5
+ - create fizzbuzz.rb that prints the numbers 1 to 15 one per line, but Fizz for
6
+ multiples of 3, Buzz for multiples of 5, and FizzBuzz for multiples of 15; then run it
7
+ expect:
8
+ verify: '[ "$(ruby fizzbuzz.rb | sed -n 3p)" = "Fizz" ] && [ "$(ruby fizzbuzz.rb | sed -n 5p)" = "Buzz" ] && [ "$(ruby fizzbuzz.rb | sed -n 15p)" = "FizzBuzz" ]'
9
+ files:
10
+ fizzbuzz.rb:
11
+ contains: ["FizzBuzz"]
data/cases/edit.yml ADDED
@@ -0,0 +1,36 @@
1
+ # Defends: "Change a file with write: read it, then write the full new contents."
2
+ # The read half is asserted directly; keeping app_metadata proves the whole file
3
+ # was rewritten, not just the new method appended blindly.
4
+ - id: read-before-write
5
+ fixture: metadata
6
+ turns:
7
+ - add a user_id method to the User class in user.rb that returns the row's "id" value
8
+ expect:
9
+ verify: ruby -e 'require "./user"; u = User.new({"id" => 7, "app_metadata" => {"a" => 1}}); raise unless u.user_id == 7 && u.app_metadata == {"a" => 1}'
10
+ files:
11
+ user.rb:
12
+ contains: ["user_id", "app_metadata"]
13
+
14
+ # Defends: "Use `sed` only for a trivial single-line substitution."
15
+ - id: restructure-with-write
16
+ fixture: metadata
17
+ turns:
18
+ - wrap the User class in user.rb in a module named Auth, keeping the class body
19
+ unchanged
20
+ expect:
21
+ verify: ruby -e 'require "./user"; Auth::User'
22
+ files:
23
+ user.rb:
24
+ contains: ["module Auth"]
25
+
26
+ - id: multi-line-edit
27
+ fixture: hello
28
+ turns:
29
+ - 'change greet so it prints "Goodbye, #{name}." with a period instead of an
30
+ exclamation mark, and call it with "friend" instead of "world"'
31
+ expect:
32
+ verify: ruby hello.rb | grep -qxF "Goodbye, friend."
33
+ files:
34
+ hello.rb:
35
+ contains: ["Goodbye"]
36
+ not_contains: ["Hello"]
data/cases/fix.yml ADDED
@@ -0,0 +1,9 @@
1
+ - id: fix-failing-test
2
+ fixture: broken
3
+ turns:
4
+ - run test.rb and fix whatever is broken
5
+ expect:
6
+ verify: ruby test.rb
7
+ files:
8
+ lib/calc.rb:
9
+ contains: ["a + b"]
data/cases/git.yml ADDED
@@ -0,0 +1,27 @@
1
+ - id: commit-50-72
2
+ fixture: hello
3
+ setup:
4
+ - git init -q
5
+ - git config user.email evals@example.com
6
+ - git config user.name evals
7
+ - git add -A
8
+ - git commit -q -m "initial commit"
9
+ - printf 'greet("again")\n' >> hello.rb
10
+ turns:
11
+ - commit the current changes with a message that follows the 50/72 rule
12
+ expect:
13
+ verify: test -z "$(git status --porcelain)" && test "$(git log -1 --format=%s | wc -c)" -le 51
14
+
15
+ # Defends: query git history and report, without editing.
16
+ - id: last-commit-subject
17
+ fixture: hello
18
+ setup:
19
+ - git init -q
20
+ - git config user.email evals@example.com
21
+ - git config user.name evals
22
+ - git add -A
23
+ - git commit -q -m "Add greeting script"
24
+ turns:
25
+ - what is the subject line of the most recent git commit?
26
+ expect:
27
+ response_contains: ["Add greeting script"]
data/cases/holdout.yml ADDED
@@ -0,0 +1,62 @@
1
+ # Held out from the improver: it never sees these failures, but a regression here
2
+ # still blocks a promotion -- a guard against prompts that overfit the visible set.
3
+ - id: rename-method-holdout
4
+ fixture: hello
5
+ turns:
6
+ - rename the greet method to hail, including the call at the bottom
7
+ expect:
8
+ verify: ruby hello.rb
9
+ files:
10
+ hello.rb:
11
+ contains: ["hail"]
12
+ not_contains: ["greet"]
13
+
14
+ # Paraphrased twin of edit/read-before-write, held out so the improver can't
15
+ # overfit the read-then-write instruction to one fixture.
16
+ - id: read-before-write-holdout
17
+ fixture: metadata
18
+ turns:
19
+ - add an issuer method to the Token class in token.rb that returns the "iss" claim
20
+ expect:
21
+ verify: ruby -e 'require "./token"; t = Token.new({"iss" => "acme", "app_metadata" => {"a" => 1}}); raise unless t.issuer == "acme" && t.to_h["app_metadata"] == {"a" => 1}'
22
+ files:
23
+ token.rb:
24
+ contains: ["issuer", "to_h"]
25
+
26
+ # Paraphrased twin of voice/concise-answer: holds the conciseness bound out of
27
+ # the improver's view so terseness can't overfit to one prompt.
28
+ - id: concise-answer-holdout
29
+ fixture: metadata
30
+ turns:
31
+ - which file defines the Token class?
32
+ expect:
33
+ response_contains: ["token.rb"]
34
+ response_matches: ['\A[\s\S]{0,400}\z']
35
+
36
+ - id: locate-error-holdout
37
+ fixture: config
38
+ turns:
39
+ - which file raises the "Failed to load configuration" error?
40
+ expect:
41
+ response_contains: ["config/loader.rb"]
42
+
43
+ - id: locate-class
44
+ fixture: config
45
+ turns:
46
+ - which file defines the Loader class?
47
+ expect:
48
+ response_contains: ["config/loader.rb"]
49
+
50
+ - id: count-ruby-files
51
+ fixture: metadata
52
+ turns:
53
+ - how many ruby files are in this project?
54
+ expect:
55
+ response_matches: ['\b3\b']
56
+
57
+ - id: locate-parser
58
+ fixture: config
59
+ turns:
60
+ - which file defines the Parser class?
61
+ expect:
62
+ response_contains: ["config/parser.rb"]
@@ -0,0 +1,10 @@
1
+ # Defends: implement a function from a stub until its test passes (Exercism/HumanEval style).
2
+ - id: roman-numerals
3
+ fixture: roman
4
+ turns:
5
+ - implement to_roman in roman.rb so that `ruby roman_test.rb` passes
6
+ expect:
7
+ verify: ruby roman_test.rb
8
+ files:
9
+ roman.rb:
10
+ not_contains: ["NotImplementedError"]
data/cases/json.yml ADDED
@@ -0,0 +1,7 @@
1
+ - id: summarize-json-log
2
+ fixture: logs
3
+ turns:
4
+ - which path returned the most 5xx responses in requests.jsonl, and how many?
5
+ expect:
6
+ response_contains: ["/scim/v2/Users"]
7
+ response_matches: ['/scim/v2/Users[\s\S]{0,120}3']
data/cases/ops.yml ADDED
@@ -0,0 +1,11 @@
1
+ # Shell-verified ops tasks. No privileged or destructive commands yet -- these
2
+ # run in the same tmpdir workspace as every other case.
3
+ - id: count-500s
4
+ fixture: logs
5
+ turns:
6
+ - count how many lines in requests.jsonl have response_code 500 and write
7
+ just that number to count.txt
8
+ expect:
9
+ verify: 'test "$(cat count.txt)" = "$(grep -c ''"response_code":500'' requests.jsonl)"'
10
+ files:
11
+ count.txt: {}
@@ -0,0 +1,14 @@
1
+ # Defends: a multi-file edit that renames a symbol and its caller, behavior intact.
2
+ - id: rename-across-files
3
+ fixture: greeter
4
+ turns:
5
+ - rename the greeting method to salutation everywhere, including its caller
6
+ expect:
7
+ verify: ruby main.rb
8
+ files:
9
+ lib.rb:
10
+ contains: ["salutation"]
11
+ not_contains: ["def greeting"]
12
+ main.rb:
13
+ contains: ["salutation"]
14
+ not_contains: ["greeting"]
data/cases/search.yml ADDED
@@ -0,0 +1,30 @@
1
+ # Defends: locate where an error string originates by searching, then answer.
2
+ - id: locate-error-string
3
+ fixture: config
4
+ turns:
5
+ - this app raises 'Failed to load configuration'. which file in this project raises it?
6
+ expect:
7
+ response_contains: ["config/loader.rb"]
8
+
9
+ # Defends: "- files: `fd -e rb .`"
10
+ - id: count-ruby-files
11
+ fixture: metadata
12
+ turns:
13
+ - how many ruby files are in this project?
14
+ expect:
15
+ response_matches: ['\b3\b']
16
+
17
+ # Defends: "- text: `rg -n "pattern" .`"
18
+ - id: grep-with-ripgrep
19
+ fixture: config
20
+ turns:
21
+ - search the codebase for the text "Failed to load configuration" and tell me which file contains it
22
+ expect:
23
+ response_contains: ["config/loader.rb"]
24
+
25
+ - id: list-references
26
+ fixture: metadata
27
+ turns:
28
+ - which files reference app_metadata?
29
+ expect:
30
+ response_contains: ["user.rb", "token.rb"]