plai_testr 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +16 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +176 -0
  5. data/bin/plai_testr +5 -0
  6. data/docs/frame_queue.md +123 -0
  7. data/docs/referee.md +59 -0
  8. data/lib/plai_testr/agents/ollama_client.rb +177 -0
  9. data/lib/plai_testr/agents/player_agent.rb +216 -0
  10. data/lib/plai_testr/agents/referee_agent.rb +56 -0
  11. data/lib/plai_testr/botrun.rb +60 -0
  12. data/lib/plai_testr/cli.rb +190 -0
  13. data/lib/plai_testr/config.rb +39 -0
  14. data/lib/plai_testr/deck.rb +11 -0
  15. data/lib/plai_testr/errors.rb +9 -0
  16. data/lib/plai_testr/event_log.rb +24 -0
  17. data/lib/plai_testr/game.rb +50 -0
  18. data/lib/plai_testr/generator.rb +50 -0
  19. data/lib/plai_testr/model_settings.rb +25 -0
  20. data/lib/plai_testr/names.rb +7 -0
  21. data/lib/plai_testr/personas.rb +97 -0
  22. data/lib/plai_testr/preflight.rb +70 -0
  23. data/lib/plai_testr/reporter/botrun_report.rb +141 -0
  24. data/lib/plai_testr/reporter/model_comparison.rb +104 -0
  25. data/lib/plai_testr/reporter/report_builder.rb +154 -0
  26. data/lib/plai_testr/reporter/think_log.rb +32 -0
  27. data/lib/plai_testr/rules_loader.rb +40 -0
  28. data/lib/plai_testr/session/frame_queue.rb +47 -0
  29. data/lib/plai_testr/session/manager.rb +77 -0
  30. data/lib/plai_testr/session/runner.rb +555 -0
  31. data/lib/plai_testr/state/base.rb +13 -0
  32. data/lib/plai_testr/state/game_state.rb +102 -0
  33. data/lib/plai_testr/state/player_state.rb +61 -0
  34. data/lib/plai_testr/version.rb +3 -0
  35. data/lib/plai_testr.rb +61 -0
  36. data/templates/app/.gitignore +3 -0
  37. data/templates/app/.rspec +3 -0
  38. data/templates/app/AGENTS.md +158 -0
  39. data/templates/app/CLAUDE.md +1 -0
  40. data/templates/app/Gemfile.erb +7 -0
  41. data/templates/app/README.md.erb +91 -0
  42. data/templates/app/config/plai_testr.rb +15 -0
  43. data/templates/app/spec/spec_helper.rb +25 -0
  44. data/templates/examples/gin_rummy/NOTES.md +12 -0
  45. data/templates/examples/gin_rummy/data/bots.yml +18 -0
  46. data/templates/examples/gin_rummy/data/personas.yml +26 -0
  47. data/templates/examples/gin_rummy/data/rules.json +76 -0
  48. data/templates/examples/gin_rummy/data/rules.md +44 -0
  49. data/templates/examples/gin_rummy/lib/gin_rummy/bot.rb +34 -0
  50. data/templates/examples/gin_rummy/lib/gin_rummy/game.rb +8 -0
  51. data/templates/examples/gin_rummy/lib/gin_rummy/melds.rb +51 -0
  52. data/templates/examples/gin_rummy/lib/gin_rummy/session/runner.rb +197 -0
  53. data/templates/examples/gin_rummy/lib/gin_rummy/state/game_state.rb +19 -0
  54. data/templates/examples/gin_rummy/lib/gin_rummy.rb +9 -0
  55. data/templates/examples/gin_rummy/spec/melds_spec.rb +26 -0
  56. data/templates/examples/gin_rummy/spec/session/runner_spec.rb +105 -0
  57. data/templates/examples/hearts/data/bots.yml +22 -0
  58. data/templates/examples/hearts/data/personas.yml +27 -0
  59. data/templates/examples/hearts/data/rules.json +73 -0
  60. data/templates/examples/hearts/data/rules.md +30 -0
  61. data/templates/examples/hearts/lib/hearts/bot.rb +53 -0
  62. data/templates/examples/hearts/lib/hearts/game.rb +16 -0
  63. data/templates/examples/hearts/lib/hearts/reporter/botrun_report.rb +29 -0
  64. data/templates/examples/hearts/lib/hearts/session/runner.rb +255 -0
  65. data/templates/examples/hearts/lib/hearts/state/game_state.rb +19 -0
  66. data/templates/examples/hearts/lib/hearts.rb +11 -0
  67. data/templates/examples/hearts/spec/session/runner_spec.rb +122 -0
  68. data/templates/game/data/bots.yml +13 -0
  69. data/templates/game/data/personas.yml +16 -0
  70. data/templates/game/data/rules.json.erb +16 -0
  71. data/templates/game/data/rules.md.erb +11 -0
  72. data/templates/game/lib/__name__/bot.rb.erb +7 -0
  73. data/templates/game/lib/__name__/session/runner.rb.erb +36 -0
  74. data/templates/game/lib/__name__/state/game_state.rb.erb +11 -0
  75. data/templates/game/lib/__name__.rb.erb +10 -0
  76. data/templates/game/spec/session/runner_spec.rb.erb +29 -0
  77. metadata +153 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: c248d4cb1b694d6260effa15b48ef4805638dfc19691f622defc0564c5334328
4
+ data.tar.gz: 32b542793975ea15359ca7b6d6def786656fd362b72d490fcd810a20fac98a26
5
+ SHA512:
6
+ metadata.gz: 0f21750ca5f57820932d029dcc5cd5461c9633568246dd1e7578155fc2445ac7eb90486c64c02063596b86b69b606a28ee98b26a4924e45e85ab9992f950a85b
7
+ data.tar.gz: c6ef28a4491f87a94ccfcbd8591af5cd2ecac0b5f1722b8a174cbce01e4a91ffce628a45ac8199a8004fbfd58526058e518f1716977add17f0744f26eea2a4e2
data/CHANGELOG.md ADDED
@@ -0,0 +1,16 @@
1
+ # Changelog
2
+
3
+ ## 0.1.1 (2026-10-23)
4
+
5
+ - Author name in the gemspec and license.
6
+
7
+ ## 0.1.0 (2026-10-23, yanked)
8
+
9
+ First public release.
10
+
11
+ - `plai_testr new` creates a playtesting app with two example games (Hearts, Gin Rummy); `plai_testr generate game` adds a game skeleton.
12
+ - `play` runs one model game with preflight checks; `botrun` plays many bot-only games and reports balance; `resume`, `compare`, `list` and `preflight` round it out.
13
+ - Games list their legal moves as a numbered menu; the model picks one. Bot rules can play obvious moves (hybrid mode) or every move (bot mode).
14
+ - Personas (`personas.yml`) and bot parameters (`bots.yml`) per game.
15
+ - Checkpoints after every decision; resume refuses sessions whose game has changed.
16
+ - Configurable decision limit and model settings in `config/plai_testr.rb`.
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2026 Aaron H
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,176 @@
1
+ # PlAI Testr
2
+
3
+ Automated tabletop game playtesting with local LLM players via [Ollama](https://ollama.com).
4
+
5
+ Your game lists the legal moves each turn as a numbered menu; a model (or a scripted bot) picks one; PlAI Testr applies it, checkpoints, and writes a markdown report. Bots alone can play thousands of games in minutes for balance numbers; model players add reasoning and end-of-game feedback.
6
+
7
+ ## Requirements
8
+
9
+ - Ruby 3.4+
10
+ - [Ollama](https://ollama.com), local or on your network (not needed for bot-only runs)
11
+
12
+ ## Quick start
13
+
14
+ ```sh
15
+ gem install plai_testr # or in a Gemfile: gem 'plai_testr', github: 'armahillo/plai_testr'
16
+ plai_testr new my_playtests # an app with two example games: Hearts and Gin Rummy
17
+ cd my_playtests
18
+ bundle install
19
+
20
+ bundle exec plai_testr hearts botrun --games 100 # bots only, no model: a report in seconds
21
+ bundle exec plai_testr gin_rummy play # one model game (pulls qwen3:8b if missing)
22
+ ```
23
+
24
+ Reports land in `games/<game>/reports/`. Add your own game with `bundle exec plai_testr generate game my_game`, then fill in the hooks below. The generated app's `AGENTS.md` walks an AI coding assistant through building a harness, so you can have your own agent do it.
25
+
26
+ ## Commands
27
+
28
+ ```
29
+ plai_testr <game> play [--seed N] [--model M] [--think] [--mode llm|hybrid|bot] [--players N] [--personas A,B] [--referee M]
30
+ plai_testr <game> botrun [--games N | --seeds 1,2,3] [--players N] [--personas all|A,B]
31
+ plai_testr <game> resume <session_id>
32
+ plai_testr <game> preflight
33
+ plai_testr <game> compare [--since ID]
34
+ plai_testr <game> list
35
+ plai_testr new <dir>
36
+ plai_testr generate game <name>
37
+ ```
38
+
39
+ - `play` runs one game. Without `--seed` it picks one from 1–100 and prints it. Before playing, preflight checks Ollama, pulls missing models, warms them up, measures the first prompt against the context size, and plays 5 bot games to catch menu bugs. `--skip-preflight` skips all of that.
40
+ - `--mode hybrid` lets the bot play moves it marks *obvious*; `--mode bot` uses no model.
41
+ - `botrun` plays many bot-only games in memory and writes one report: endings, wins by persona and seat with 95% ranges, game length, bot rules used, warnings and crashes.
42
+ - `--quiet` shows a progress line and the result; `--verbose` adds raw model output (`play`) or every game (`botrun`).
43
+ - Ctrl+C (or SIGTERM) saves and stops after the current move; a botrun finishes its game and reports. A second Ctrl+C quits at once.
44
+ - Errors print one line. `PLAI_TESTR_DEBUG=1` shows the backtrace; mistakes in game code (`MoveError`) always do.
45
+
46
+ ## Writing a game
47
+
48
+ `generate game` creates `games/<name>/` with everything below. The examples in [`templates/examples/`](templates/examples) are complete games to copy from.
49
+
50
+ ```
51
+ games/<name>/
52
+ lib/<name>.rb Requires the game and calls PlaiTestr.register
53
+ lib/<name>/bot.rb Bot rules (bot_choice), included in the runner
54
+ lib/<name>/session/ Runner (rules engine)
55
+ lib/<name>/state/ GameState / PlayerState subclasses
56
+ data/rules.json Name, version, player count, card deck, numbers the engine uses
57
+ data/rules.md Rules text the model players read
58
+ data/personas.yml Optional: strategy and play styles for model players
59
+ data/bots.yml Optional: parameters the bot rules read
60
+ spec/
61
+ ```
62
+
63
+ ### Hooks
64
+
65
+ A game subclasses `PlaiTestr::Session::Runner`. These hooks are its public API ([`runner.rb`](lib/plai_testr/session/runner.rb)):
66
+
67
+ | Hook | Required | Purpose |
68
+ |---|---|---|
69
+ | `setup` | yes | Build the starting state. Not called on resume. |
70
+ | `legal_moves(player)` | yes | The menu: an array of `option(...)` entries |
71
+ | `apply_action_effects(player, action)` | yes | Apply the chosen move's action to `@game_state` |
72
+ | `next_player` | | Who acts next (default: alternate) |
73
+ | `pick_range(player)` | | `[min, max]` to pick several options at once |
74
+ | `game_over?` | | Default: `status['game_over']` |
75
+ | `turn_phase(player)` | | Short step label for the console and report |
76
+ | `rules_text_for`, `view_text_for`, `prompt_note_for` | | Prompt text: rules, the player's view, a one-off note |
77
+ | `recent_events_for(player)` | | Recent events from the event log (default: the last 8 this player may see) |
78
+ | `persona_for(player)` | | Strategy text for the system prompt (default: built from `personas.yml`) |
79
+ | `display_name(player)` | | Name in the console and reports (default: the persona's key, title-cased, else the player id) |
80
+ | `bot_choice(player, moves, pick: nil)` | | Bot rules: `nil` or `{ 'choice' => n, 'rule' => name, 'obvious' => bool }` (`'choices'` on pick turns) |
81
+ | `think_about?(player)` | | With `--think`: whether this turn gets thinking (default: every turn) |
82
+ | `debrief_questions`, `debrief_view_for` | | End-of-game feedback questions |
83
+ | `report_sections`, `report_debrief?` | | Extra report sections; false to render the debrief yourself |
84
+ | `botrun_stats` | | Game-specific numbers for one botrun game |
85
+
86
+ Helpers for hooks: `option`, `menu(player)` (the menu as the runner sees it), `data`, `log(text, to: nil, turn: decision)`, `event_log`, `persona(player)`, `persona_name(player)`, `bot_strategy_for(player)`, `@rng`, `@rules`, `@game_state`, `out`. Specs drive a runner with `setup`, `play_turn`, `play_out` and the hooks above.
87
+
88
+ ### Moves
89
+
90
+ ```ruby
91
+ def legal_moves(player)
92
+ hand(player).map { |card| option("Play #{card['name']}", public: "played #{card['name']}", play: { value: card['uid'] }) }
93
+ end
94
+
95
+ def apply_action_effects(player, action)
96
+ case action
97
+ in { play: { value: uid } } then play(player, uid)
98
+ in { pass: { value: uids } } then pass(player, uids)
99
+ end
100
+ end
101
+ ```
102
+
103
+ `label` (the first argument) is what the acting player sees; `public:` is what others see; `log:` is a shorter console line. An action has one name; `value:` is the main parameter and others sit beside it. Moves and actions have symbol keys, so `apply_action_effects` can pattern-match them; a hand-written move with string keys is converted. On a pick turn the chosen values arrive together: `{ pass: { value: [42, 17, 9] } }`. Every menu is checked before it is used: a misspelled key or a malformed action raises `PlaiTestr::MoveError` naming the hook, player, decision and option. A menu with one option is applied without asking anyone.
104
+
105
+ ### State
106
+
107
+ `@game_state` ([`game_state.rb`](lib/plai_testr/state/game_state.rb)) holds `turn` (decisions so far), `players`, `status` and `data`; each `player_state(name)` ([`player_state.rb`](lib/plai_testr/state/player_state.rb)) holds `score`, `hand` and `data`.
108
+
109
+ - `status` belongs to the runner: set `game_over`, `winner` and `end_reason` when the game ends, with `@game_state.apply_changes('status' => {...})`.
110
+ - `data` is the game's own state, as plain JSON-safe values so checkpoints work without extra code.
111
+ - Subclass `GameState` to name your keys (`def step = @data['step']`) and to hide information per player with `data_for(observer)`; subclass `PlayerState` and override `serialize_for` / `public_data` to choose what opponents see. The examples show both.
112
+
113
+ ### Personas and bots
114
+
115
+ `data/personas.yml` gives model players a shared `basic_strategy` and play styles under `personas`; `data/bots.yml` gives the bot rules parameters, per persona, read with `bot_strategy_for(player)`. Both files are optional and the generated copies explain their format. Each player gets a persona from `--personas a,b` or at random per seed; `botrun --personas all` cycles every pairing.
116
+
117
+ ### Randomness and limits
118
+
119
+ - Use `@rng` for all randomness. It is reset for each decision from the seed and the decision number, so the same seed and the same choices always give the same game, interrupted or not.
120
+ - When the model never gives a valid answer, the bot's choice is played instead (or a random legal move from a separate generator), and the turn is flagged.
121
+ - Every game stops at a decision limit (`end_reason: 'decision_limit_reached'`, no winner) so a buggy game cannot run forever. Set `max_decisions:` in `PlaiTestr.register`; otherwise the app backstop applies.
122
+
123
+ ### Nested decisions and rulings
124
+
125
+ - [FrameQueue](docs/frame_queue.md): attacks and responses, triggered effects, decisions inside decisions.
126
+ - [Referee](docs/referee.md): yes/no rulings on free-text conditions by a second model.
127
+
128
+ ## Configuration
129
+
130
+ `config/plai_testr.rb` in the app (generated, all commented out):
131
+
132
+ ```ruby
133
+ PlaiTestr.configure do |config|
134
+ config.max_decisions = 100_000 # backstop for games without their own limit
135
+ config.player_model = 'qwen3:8b'
136
+ config.ollama_host = 'localhost'
137
+ config.ollama_port = 11434
138
+ config.model_settings = { context_size: 8192 }
139
+ end
140
+ ```
141
+
142
+ A game can override model settings in `PlaiTestr.register(..., model_settings: { ... })`.
143
+
144
+ | Model setting | Default | Used for |
145
+ |---|---|---|
146
+ | `context_size` | 8192 | Menu turns without `--think`; the referee |
147
+ | `long_context_size` | 16384 | Sessions with `--think`; the debrief |
148
+ | `menu_budget` | 512 | Output tokens for a menu answer |
149
+ | `think_budget` | 8192 | Output tokens for a thinking turn |
150
+ | `debrief_budget` | 1024 | Output tokens for the debrief |
151
+ | `referee_budget` | 2048 | Output tokens for a ruling |
152
+ | `temperature` | 0.6 | Player sampling |
153
+
154
+ Values outside the hard limits are clamped with a warning: context sizes between 2048 and the model's own maximum, temperature 0–2, budgets above 0 and below the context size. The values used are recorded in each report. Environment variables set the defaults: `PLAI_TESTR_PLAYER_MODEL`, `PLAI_TESTR_REFEREE_MODEL`, `PLAI_TESTR_OLLAMA_HOST`, `PLAI_TESTR_OLLAMA_PORT`.
155
+
156
+ ## Session output
157
+
158
+ - `sessions/<id>/state.json`: the checkpoint, replaced atomically after every decision
159
+ - `sessions/<id>/transcript.jsonl`: one JSON object per decision, with the model's reasoning. After a crash the last line may be cut off; skip lines that do not parse.
160
+ - `sessions/<id>/think.log`: every model reply with Ollama's timings
161
+ - `reports/<id>.md`: the session report
162
+
163
+ `resume` continues an interrupted session exactly, or refuses: if the game's data, code or the gem version changed since the session started, start a new one with `play`.
164
+
165
+ ## Development
166
+
167
+ ```sh
168
+ bundle install
169
+ bundle exec rspec
170
+ ```
171
+
172
+ `lib/plai_testr/` holds the gem: [`session/runner.rb`](lib/plai_testr/session/runner.rb) (the decision loop and hooks), [`agents/`](lib/plai_testr/agents) (Ollama client, player and referee prompts), [`reporter/`](lib/plai_testr/reporter), [`state/`](lib/plai_testr/state), [`cli.rb`](lib/plai_testr/cli.rb), [`preflight.rb`](lib/plai_testr/preflight.rb), [`botrun.rb`](lib/plai_testr/botrun.rb). `templates/` holds what `new` and `generate game` copy, including the example games and their specs.
173
+
174
+ ## License
175
+
176
+ MIT. See [LICENSE.txt](LICENSE.txt).
data/bin/plai_testr ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'plai_testr'
4
+
5
+ PlaiTestr::CLI.start(ARGV)
@@ -0,0 +1,123 @@
1
+ # Nested decisions with FrameQueue
2
+
3
+ Most turns are one decision: the active player picks from a menu. Some games need decisions *inside* other decisions: an attack the defender may block, a card that triggers an opponent's response, an effect that asks someone else to choose. `PlaiTestr::Session::FrameQueue` handles these.
4
+
5
+ ## The idea
6
+
7
+ A **frame** is one pending step: a hash with a `type`, the `player` who decides it (if anyone), and any data the step needs. Frames live on a stack in game state, so a checkpoint saved in the middle of an attack resumes in the middle of that attack.
8
+
9
+ - The frame on top of the stack is the current decision. `next_player` and `legal_moves` read it.
10
+ - When the player picks a move, `resolve_frame(params)` pops the frame and calls `on_<type>(frame, params)`.
11
+ - Handlers push new frames for whatever happens next. **The last frame pushed resolves first.**
12
+ - Frames marked `auto: true` need no decision: they are popped and resolved with `on_<type>(frame)` as soon as they reach the top.
13
+ - When the stack is empty, `schedule_frames` pushes the next normal turn.
14
+
15
+ ## What you implement
16
+
17
+ | Method | Purpose |
18
+ |---|---|
19
+ | `frame_queue` | The array that holds the frames, kept in `@game_state.data` |
20
+ | `schedule_frames` | Push the next turn's frame(s) when the stack is empty |
21
+ | `on_<type>(frame, params)` | Resolve a decision frame |
22
+ | `on_<type>(frame)` | Resolve an `auto` frame |
23
+ | `halt_frames?` | Optional. Stop resolving (default: `game_over?`). A good place for end checks and safety caps |
24
+ | `intercept_frame(frame)` | Optional. Return true to stop before a frame and handle it in your own code (e.g. hand maintenance) |
25
+
26
+ You call `advance_frames` once at the end of `setup`, and `resolve_frame(action.values.first)` from `apply_action_effects`. `frame_for(type, player, **data)` builds a frame. The hooks in the table are public; `advance_frames`, `resolve_frame`, `frame` and `frame_for` are private helpers your runner calls.
27
+
28
+ ## Example: attack and block
29
+
30
+ Each turn the attacker plays a card. The defender may block with a card of equal or higher value, or take the damage. Damage is an `auto` frame, so it resolves without a decision; reaching 10 damage loses.
31
+
32
+ ```ruby
33
+ module Strike
34
+ module Session
35
+ class Runner < PlaiTestr::Session::Runner
36
+ include PlaiTestr::Session::FrameQueue
37
+
38
+ HAND_SIZE = 4
39
+ KNOCKOUT = 10
40
+
41
+ def setup
42
+ deck = PlaiTestr::Deck.build(@rules).shuffle(random: @rng)
43
+ players.each { |p| hand(p).concat(deck.shift(HAND_SIZE)) }
44
+ data.merge!('queue' => [], 'deck' => deck, 'damage' => players.to_h { |p| [p, 0] }, 'attacker' => players.first)
45
+ advance_frames
46
+ end
47
+
48
+ def frame_queue = data['queue']
49
+
50
+ def schedule_frames
51
+ attacker = data['attacker']
52
+ hand(attacker).concat(data['deck'].shift(HAND_SIZE - hand(attacker).length))
53
+ frame_queue.push(frame_for('attack', attacker))
54
+ end
55
+
56
+ def halt_frames?
57
+ end_game(nil, 'deck_empty') if !game_over? && data['deck'].empty? && hand(data['attacker']).empty?
58
+ game_over?
59
+ end
60
+
61
+ def next_player = frame['player']
62
+
63
+ def legal_moves(player)
64
+ case frame['type']
65
+ when 'attack'
66
+ hand(player).map { |c| option("Attack with #{c['name']}", attack: { value: c['uid'] }) }
67
+ when 'block'
68
+ blockers = hand(player).select { |c| c['value'] >= frame['value'] }
69
+ blockers.map { |c| option("Block with #{c['name']}", block: { value: c['uid'] }) } + [option("Take #{frame['value']} damage", block: {})]
70
+ end
71
+ end
72
+
73
+ def apply_action_effects(_player, action) = resolve_frame(action.values.first)
74
+
75
+ private
76
+
77
+ def on_attack(frame, params)
78
+ card = take(frame['player'], params[:value])
79
+ defender = opponent(frame['player'])
80
+ frame_queue.push(frame_for('block', defender, value: card['value']))
81
+ end
82
+
83
+ def on_block(frame, params)
84
+ if params[:value]
85
+ take(frame['player'], params[:value])
86
+ else
87
+ frame_queue.push(frame_for('damage', frame['player'], auto: true, amount: frame['value']))
88
+ end
89
+ data['attacker'] = frame['player']
90
+ end
91
+
92
+ def on_damage(frame)
93
+ data['damage'][frame['player']] += frame['amount']
94
+ end_game(opponent(frame['player']), 'knockout') if data['damage'][frame['player']] >= KNOCKOUT
95
+ end
96
+
97
+ def end_game(winner, reason)
98
+ @game_state.apply_changes('status' => { 'game_over' => true, 'winner' => winner, 'end_reason' => reason })
99
+ end
100
+
101
+ def take(player, uid) = hand(player).delete_at(hand(player).index { |c| c['uid'] == uid })
102
+ def opponent(player) = (players - [player]).first
103
+ def players = @game_state.players
104
+ def hand(player) = @game_state.player_state(player).hand
105
+ end
106
+ end
107
+ end
108
+ ```
109
+
110
+ The sequence for one exchange:
111
+
112
+ 1. The stack is empty, so `schedule_frames` pushes `attack` for Player 1.
113
+ 2. Player 1 attacks; `on_attack` pushes `block` for Player 2. It is not `auto`, so play stops for Player 2's decision.
114
+ 3. Player 2 takes the damage; `on_block` pushes an `auto` `damage` frame, which resolves at once.
115
+ 4. The stack is empty again, so `schedule_frames` pushes `attack` for Player 2.
116
+
117
+ ## Tips
118
+
119
+ - **Stack order.** To have A happen before B, push B first, then A.
120
+ - **Pick-N frames.** Put `pick: [min, max]` on the frame and return `frame['pick']` from `pick_range`.
121
+ - **Say what the frame is for.** `turn_phase` can return `frame['type']`, and views should say why the player is deciding ("Player 1 attacks with Blade 5: block or take 5 damage").
122
+ - **Cap runaway auto-frame loops in `halt_frames?`.** The decision limit (`max_decisions`) counts only decisions, so a loop of `auto` frames never reaches it. Count resolved frames and end the game past a limit the players never see.
123
+ - **Bot rules** read `frame['type']` the same way `legal_moves` does.
data/docs/referee.md ADDED
@@ -0,0 +1,59 @@
1
+ # Yes/no rulings with the referee
2
+
3
+ Prefer encoding rules in Ruby: it is fast, deterministic and testable. Some conditions are awkward to encode, though, especially free-text card conditions in a game that is still changing ("If your opponent is ahead…", "If this is your first attack this turn…"). The referee is a second model that answers one yes/no question about such a condition.
4
+
5
+ ## How it works
6
+
7
+ Run with `--referee <model>` (or set `PLAI_TESTR_REFEREE_MODEL`). The runner then has `@referee`, an `Agents::RefereeAgent`; without the option, `@referee` is `nil`.
8
+
9
+ ```ruby
10
+ @referee.ruling(question: 'Is the condition on Ambush met?', facts: "...")
11
+ # => { 'met' => true, 'why' => 'The opponent has 12 points and the player has 8.', 'fallback' => false }
12
+ ```
13
+
14
+ The referee sees only the game's name, the question and the facts you pass, with thinking on and temperature 0.1. If it cannot produce a valid answer after retries, it returns `met: false` with `fallback: true`.
15
+
16
+ Nothing calls the referee for you, and rulings are not added to reports automatically. Your runner asks, records and reports.
17
+
18
+ ## Example: a card with a free-text condition
19
+
20
+ Cards in `rules.json` carry their condition as text:
21
+
22
+ ```json
23
+ { "name": "Ambush", "value": 2, "condition": "your opponent has more points than you", "bonus": 3, "copies": 2 }
24
+ ```
25
+
26
+ When a card with a condition is played, the runner asks for a ruling and records it (`data` and `log` are runner helpers):
27
+
28
+ ```ruby
29
+ def bonus_for(player, card)
30
+ return 0 unless card['condition']
31
+
32
+ ruling = ruling("Is this condition met for #{player}: #{card['condition']}?", view_text_for(player))
33
+ ruling['met'] ? card['bonus'] : 0
34
+ end
35
+
36
+ def ruling(question, facts)
37
+ result = if @referee
38
+ @referee.ruling(question: question, facts: facts)
39
+ else
40
+ { 'met' => false, 'why' => 'no referee: run with --referee to rule on conditions', 'fallback' => true }
41
+ end
42
+ (data['rulings'] ||= []) << result.merge('turn' => @game_state.turn + 1, 'question' => question)
43
+ log("Ruling: #{question} #{result['met'] ? 'Yes' : 'No'} (#{result['why']})")
44
+ result
45
+ end
46
+
47
+ def report_sections
48
+ rows = data.fetch('rulings', []).map { |r| "| #{r['turn']} | #{r['question']} | #{r['met'] ? 'yes' : 'no'} | #{r['why']}#{' (fallback)' if r['fallback']} |" }
49
+ ["## Rulings\n\n| Turn | Question | Met | Why |\n|---|---|---|---|\n#{rows.join("\n")}"]
50
+ end
51
+ ```
52
+
53
+ ## Things to know
54
+
55
+ - **Facts are everything.** The referee knows nothing about the game state except the facts string. Passing the deciding player's view (`view_text_for`) is a good default; add anything the question depends on.
56
+ - **Botruns and `--mode bot` have no referee.** Decide the fallback on purpose (above: the condition is not met) and say so in the log, or give the bot its own Ruby check for the condition.
57
+ - **Rulings break seed reproducibility.** The same seed can play differently when a ruling changes. Record every ruling, as above, so a report shows why.
58
+ - **Rulings are slow.** Each is a model call with thinking on. Ask only when the condition actually matters (a card with a condition was played), never on every turn.
59
+ - **Turn rulings into code.** When a condition comes up often and the designer agrees on what it means, encode it in Ruby and keep the referee for the rest.
@@ -0,0 +1,177 @@
1
+ module PlaiTestr
2
+ module Agents
3
+ class OllamaClient
4
+ READ_TIMEOUT = 300
5
+ MAX_RETRIES = 3
6
+ CONTEXT_WARN_RATIO = 0.9
7
+ SPINNER_FRAMES = %w[⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏].freeze
8
+ UNREACHABLE = [Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::EADDRNOTAVAIL, SocketError, Net::OpenTimeout].freeze
9
+
10
+ def initialize(host: PlaiTestr.config.ollama_host, port: PlaiTestr.config.ollama_port, think_log: nil, out: $stdout, verbose: false)
11
+ @host = host
12
+ @port = port
13
+ @think_log = think_log
14
+ @out = out
15
+ @verbose = verbose
16
+ @frame_index = 0
17
+ @tty = out.tty?
18
+ end
19
+
20
+ # `think`, `format` and `keep_alive` are top-level request fields in Ollama's /api/chat;
21
+ # only sampling/runtime settings belong in `options`. `format` may be "json" or a JSON schema.
22
+ def chat(model:, messages:, temperature: 0.7, think: nil, num_predict: nil, num_ctx: nil, format: nil, keep_alive: nil)
23
+ options = { temperature: temperature }
24
+ options[:num_predict] = num_predict unless num_predict.nil?
25
+ options[:num_ctx] = num_ctx unless num_ctx.nil?
26
+
27
+ payload = { model: model, messages: messages, stream: true, options: options }
28
+ payload[:think] = think unless think.nil?
29
+ payload[:format] = format unless format.nil?
30
+ payload[:keep_alive] = keep_alive unless keep_alive.nil?
31
+
32
+ response = post('/api/chat', payload)
33
+ used = response['prompt_eval_count']
34
+ return response unless used && num_ctx && used >= num_ctx * CONTEXT_WARN_RATIO
35
+
36
+ File.open(@think_log, 'a') { |f| f.puts "[WARNING] context nearly full: #{used}/#{num_ctx} prompt tokens (model=#{model})\n---\n\n" } if @think_log
37
+ response.merge('context_full' => true)
38
+ end
39
+
40
+ def model_names
41
+ request(Net::HTTP::Get.new('/api/tags')) { |response| JSON.parse(response.body) }.fetch('models', []).map { |m| m['name'] }
42
+ end
43
+
44
+ def context_length(model)
45
+ info = request(json_post('/api/show', model: model)) { |response| JSON.parse(response.body) }.fetch('model_info', {})
46
+ info["#{info['general.architecture']}.context_length"]
47
+ end
48
+
49
+ def pull(model)
50
+ request(json_post('/api/pull', model: model, stream: true), read_timeout: nil) do |response|
51
+ each_line(response) do |data|
52
+ raise ModelError, "Pulling #{model} failed: #{data['error']}" if data['error']
53
+
54
+ @out.print "\r Pulling #{model}: #{data['status']}#{" #{(100.0 * data['completed'] / data['total']).round}%" if data['total'].to_i.positive? && data['completed']}\e[K"
55
+ end
56
+ end
57
+ @out.puts
58
+ end
59
+
60
+ private
61
+
62
+ def json_post(path, payload)
63
+ Net::HTTP::Post.new(path, 'Content-Type' => 'application/json').tap { |r| r.body = JSON.generate(payload) }
64
+ end
65
+
66
+ def request(req, read_timeout: 10, &block)
67
+ http = Net::HTTP.new(@host, @port)
68
+ http.open_timeout = 5
69
+ http.read_timeout = read_timeout
70
+ http.request(req) do |response|
71
+ raise ModelError, "Ollama API error #{response.code} for #{req.path}: #{response.read_body}" unless response.is_a?(Net::HTTPSuccess)
72
+
73
+ return block.call(response)
74
+ end
75
+ rescue *UNREACHABLE => e
76
+ raise ModelError, "Cannot reach Ollama at #{@host}:#{@port} (#{e.class}: #{e.message})"
77
+ end
78
+
79
+ def each_line(response)
80
+ buffer = +''
81
+ response.read_body do |chunk|
82
+ buffer << chunk
83
+ while (newline = buffer.index("\n"))
84
+ data = parse_line(buffer.slice!(0, newline + 1))
85
+ yield data if data
86
+ end
87
+ end
88
+ data = parse_line(buffer)
89
+ yield data if data
90
+ end
91
+
92
+ def parse_line(line)
93
+ JSON.parse(line) unless line.strip.empty?
94
+ rescue JSON::ParserError
95
+ nil
96
+ end
97
+
98
+ def post(path, payload)
99
+ attempts = 0
100
+ begin
101
+ attempts += 1
102
+ post_once(path, payload)
103
+ rescue Net::ReadTimeout, Errno::ECONNRESET, EOFError => e
104
+ clear_spinner
105
+ raise ModelError, "Ollama stopped responding (#{e.class}) after #{attempts} tries" if attempts >= MAX_RETRIES
106
+
107
+ wait = 2**attempts
108
+ warn " [connection error: #{e.class} — retrying in #{wait}s]"
109
+ sleep wait
110
+ retry
111
+ end
112
+ end
113
+
114
+ def post_once(path, payload)
115
+ content = +''
116
+ thinking = +''
117
+ final_chunk = nil
118
+
119
+ request(json_post(path, payload), read_timeout: READ_TIMEOUT) do |response|
120
+ each_line(response) do |data|
121
+ piece = data.dig('message', 'content').to_s
122
+ content << piece
123
+ thinking << data.dig('message', 'thinking').to_s
124
+ final_chunk = data if data['done']
125
+ @verbose ? @out.print(piece) : advance_spinner
126
+ end
127
+ end
128
+
129
+ clear_spinner
130
+ raise ModelError, 'Ollama API error: stream ended without a done chunk' unless final_chunk
131
+
132
+ @out.puts "\n [#{format_stats(final_chunk).strip}]" if @verbose
133
+ write_think_log(payload[:model], thinking.empty? ? content : "<think>\n#{thinking.strip}\n</think>\n#{content}", final_chunk)
134
+ final_chunk.merge('message' => { 'role' => 'assistant', 'content' => content })
135
+ end
136
+
137
+ def write_think_log(model, content, stats = {})
138
+ return unless @think_log
139
+
140
+ File.open(@think_log, 'a') do |f|
141
+ f.puts "[#{Time.now.utc.iso8601}] model=#{model}#{format_stats(stats)}"
142
+ f.puts content
143
+ f.puts '---'
144
+ f.puts
145
+ end
146
+ end
147
+
148
+ def format_stats(stats)
149
+ nanos = 1_000_000_000.0
150
+ rate = ->(count, duration) { count && duration.to_i.positive? ? (count / (duration / nanos)).round(1) : nil }
151
+ {
152
+ 'prompt_tokens' => stats['prompt_eval_count'],
153
+ 'prompt_tps' => rate.call(stats['prompt_eval_count'], stats['prompt_eval_duration']),
154
+ 'eval_tokens' => stats['eval_count'],
155
+ 'eval_tps' => rate.call(stats['eval_count'], stats['eval_duration']),
156
+ 'load_s' => stats['load_duration'] && (stats['load_duration'] / nanos).round(2),
157
+ 'total_s' => stats['total_duration'] && (stats['total_duration'] / nanos).round(2)
158
+ }.filter_map { |key, value| " #{key}=#{value}" unless value.nil? }.join
159
+ end
160
+
161
+ def advance_spinner
162
+ return unless @tty
163
+
164
+ @out.print "\r #{SPINNER_FRAMES[@frame_index % SPINNER_FRAMES.length]}"
165
+ @out.flush
166
+ @frame_index += 1
167
+ end
168
+
169
+ def clear_spinner
170
+ return unless @tty
171
+
172
+ @out.print "\r \r"
173
+ @out.flush
174
+ end
175
+ end
176
+ end
177
+ end