plai_testr 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +16 -0
- data/LICENSE.txt +21 -0
- data/README.md +176 -0
- data/bin/plai_testr +5 -0
- data/docs/frame_queue.md +123 -0
- data/docs/referee.md +59 -0
- data/lib/plai_testr/agents/ollama_client.rb +177 -0
- data/lib/plai_testr/agents/player_agent.rb +216 -0
- data/lib/plai_testr/agents/referee_agent.rb +56 -0
- data/lib/plai_testr/botrun.rb +60 -0
- data/lib/plai_testr/cli.rb +190 -0
- data/lib/plai_testr/config.rb +39 -0
- data/lib/plai_testr/deck.rb +11 -0
- data/lib/plai_testr/errors.rb +9 -0
- data/lib/plai_testr/event_log.rb +24 -0
- data/lib/plai_testr/game.rb +50 -0
- data/lib/plai_testr/generator.rb +50 -0
- data/lib/plai_testr/model_settings.rb +25 -0
- data/lib/plai_testr/names.rb +7 -0
- data/lib/plai_testr/personas.rb +97 -0
- data/lib/plai_testr/preflight.rb +70 -0
- data/lib/plai_testr/reporter/botrun_report.rb +141 -0
- data/lib/plai_testr/reporter/model_comparison.rb +104 -0
- data/lib/plai_testr/reporter/report_builder.rb +154 -0
- data/lib/plai_testr/reporter/think_log.rb +32 -0
- data/lib/plai_testr/rules_loader.rb +40 -0
- data/lib/plai_testr/session/frame_queue.rb +47 -0
- data/lib/plai_testr/session/manager.rb +77 -0
- data/lib/plai_testr/session/runner.rb +555 -0
- data/lib/plai_testr/state/base.rb +13 -0
- data/lib/plai_testr/state/game_state.rb +102 -0
- data/lib/plai_testr/state/player_state.rb +61 -0
- data/lib/plai_testr/version.rb +3 -0
- data/lib/plai_testr.rb +61 -0
- data/templates/app/.gitignore +3 -0
- data/templates/app/.rspec +3 -0
- data/templates/app/AGENTS.md +158 -0
- data/templates/app/CLAUDE.md +1 -0
- data/templates/app/Gemfile.erb +7 -0
- data/templates/app/README.md.erb +91 -0
- data/templates/app/config/plai_testr.rb +15 -0
- data/templates/app/spec/spec_helper.rb +25 -0
- data/templates/examples/gin_rummy/NOTES.md +12 -0
- data/templates/examples/gin_rummy/data/bots.yml +18 -0
- data/templates/examples/gin_rummy/data/personas.yml +26 -0
- data/templates/examples/gin_rummy/data/rules.json +76 -0
- data/templates/examples/gin_rummy/data/rules.md +44 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/bot.rb +34 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/game.rb +8 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/melds.rb +51 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/session/runner.rb +197 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/state/game_state.rb +19 -0
- data/templates/examples/gin_rummy/lib/gin_rummy.rb +9 -0
- data/templates/examples/gin_rummy/spec/melds_spec.rb +26 -0
- data/templates/examples/gin_rummy/spec/session/runner_spec.rb +105 -0
- data/templates/examples/hearts/data/bots.yml +22 -0
- data/templates/examples/hearts/data/personas.yml +27 -0
- data/templates/examples/hearts/data/rules.json +73 -0
- data/templates/examples/hearts/data/rules.md +30 -0
- data/templates/examples/hearts/lib/hearts/bot.rb +53 -0
- data/templates/examples/hearts/lib/hearts/game.rb +16 -0
- data/templates/examples/hearts/lib/hearts/reporter/botrun_report.rb +29 -0
- data/templates/examples/hearts/lib/hearts/session/runner.rb +255 -0
- data/templates/examples/hearts/lib/hearts/state/game_state.rb +19 -0
- data/templates/examples/hearts/lib/hearts.rb +11 -0
- data/templates/examples/hearts/spec/session/runner_spec.rb +122 -0
- data/templates/game/data/bots.yml +13 -0
- data/templates/game/data/personas.yml +16 -0
- data/templates/game/data/rules.json.erb +16 -0
- data/templates/game/data/rules.md.erb +11 -0
- data/templates/game/lib/__name__/bot.rb.erb +7 -0
- data/templates/game/lib/__name__/session/runner.rb.erb +36 -0
- data/templates/game/lib/__name__/state/game_state.rb.erb +11 -0
- data/templates/game/lib/__name__.rb.erb +10 -0
- data/templates/game/spec/session/runner_spec.rb.erb +29 -0
- metadata +153 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: c248d4cb1b694d6260effa15b48ef4805638dfc19691f622defc0564c5334328
|
|
4
|
+
data.tar.gz: 32b542793975ea15359ca7b6d6def786656fd362b72d490fcd810a20fac98a26
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 0f21750ca5f57820932d029dcc5cd5461c9633568246dd1e7578155fc2445ac7eb90486c64c02063596b86b69b606a28ee98b26a4924e45e85ab9992f950a85b
|
|
7
|
+
data.tar.gz: c6ef28a4491f87a94ccfcbd8591af5cd2ecac0b5f1722b8a174cbce01e4a91ffce628a45ac8199a8004fbfd58526058e518f1716977add17f0744f26eea2a4e2
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.1 (2026-10-23)
|
|
4
|
+
|
|
5
|
+
- Author name in the gemspec and license.
|
|
6
|
+
|
|
7
|
+
## 0.1.0 (2026-10-23, yanked)
|
|
8
|
+
|
|
9
|
+
First public release.
|
|
10
|
+
|
|
11
|
+
- `plai_testr new` creates a playtesting app with two example games (Hearts, Gin Rummy); `plai_testr generate game` adds a game skeleton.
|
|
12
|
+
- `play` runs one model game with preflight checks; `botrun` plays many bot-only games and reports balance; `resume`, `compare`, `list` and `preflight` round it out.
|
|
13
|
+
- Games list their legal moves as a numbered menu; the model picks one. Bot rules can play obvious moves (hybrid mode) or every move (bot mode).
|
|
14
|
+
- Personas (`personas.yml`) and bot parameters (`bots.yml`) per game.
|
|
15
|
+
- Checkpoints after every decision; resume refuses sessions whose game has changed.
|
|
16
|
+
- Configurable decision limit and model settings in `config/plai_testr.rb`.
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aaron H
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# PlAI Testr
|
|
2
|
+
|
|
3
|
+
Automated tabletop game playtesting with local LLM players via [Ollama](https://ollama.com).
|
|
4
|
+
|
|
5
|
+
Your game lists the legal moves each turn as a numbered menu; a model (or a scripted bot) picks one; PlAI Testr applies it, checkpoints, and writes a markdown report. Bots alone can play thousands of games in minutes for balance numbers; model players add reasoning and end-of-game feedback.
|
|
6
|
+
|
|
7
|
+
## Requirements
|
|
8
|
+
|
|
9
|
+
- Ruby 3.4+
|
|
10
|
+
- [Ollama](https://ollama.com), local or on your network (not needed for bot-only runs)
|
|
11
|
+
|
|
12
|
+
## Quick start
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
gem install plai_testr # or in a Gemfile: gem 'plai_testr', github: 'armahillo/plai_testr'
|
|
16
|
+
plai_testr new my_playtests # an app with two example games: Hearts and Gin Rummy
|
|
17
|
+
cd my_playtests
|
|
18
|
+
bundle install
|
|
19
|
+
|
|
20
|
+
bundle exec plai_testr hearts botrun --games 100 # bots only, no model: a report in seconds
|
|
21
|
+
bundle exec plai_testr gin_rummy play # one model game (pulls qwen3:8b if missing)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Reports land in `games/<game>/reports/`. Add your own game with `bundle exec plai_testr generate game my_game`, then fill in the hooks below. The generated app's `AGENTS.md` walks an AI coding assistant through building a harness, so you can have your own agent do it.
|
|
25
|
+
|
|
26
|
+
## Commands
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
plai_testr <game> play [--seed N] [--model M] [--think] [--mode llm|hybrid|bot] [--players N] [--personas A,B] [--referee M]
|
|
30
|
+
plai_testr <game> botrun [--games N | --seeds 1,2,3] [--players N] [--personas all|A,B]
|
|
31
|
+
plai_testr <game> resume <session_id>
|
|
32
|
+
plai_testr <game> preflight
|
|
33
|
+
plai_testr <game> compare [--since ID]
|
|
34
|
+
plai_testr <game> list
|
|
35
|
+
plai_testr new <dir>
|
|
36
|
+
plai_testr generate game <name>
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
- `play` runs one game. Without `--seed` it picks one from 1–100 and prints it. Before playing, preflight checks Ollama, pulls missing models, warms them up, measures the first prompt against the context size, and plays 5 bot games to catch menu bugs. `--skip-preflight` skips all of that.
|
|
40
|
+
- `--mode hybrid` lets the bot play moves it marks *obvious*; `--mode bot` uses no model.
|
|
41
|
+
- `botrun` plays many bot-only games in memory and writes one report: endings, wins by persona and seat with 95% ranges, game length, bot rules used, warnings and crashes.
|
|
42
|
+
- `--quiet` shows a progress line and the result; `--verbose` adds raw model output (`play`) or every game (`botrun`).
|
|
43
|
+
- Ctrl+C (or SIGTERM) saves and stops after the current move; a botrun finishes its game and reports. A second Ctrl+C quits at once.
|
|
44
|
+
- Errors print one line. `PLAI_TESTR_DEBUG=1` shows the backtrace; mistakes in game code (`MoveError`) always do.
|
|
45
|
+
|
|
46
|
+
## Writing a game
|
|
47
|
+
|
|
48
|
+
`generate game` creates `games/<name>/` with everything below. The examples in [`templates/examples/`](templates/examples) are complete games to copy from.
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
games/<name>/
|
|
52
|
+
lib/<name>.rb Requires the game and calls PlaiTestr.register
|
|
53
|
+
lib/<name>/bot.rb Bot rules (bot_choice), included in the runner
|
|
54
|
+
lib/<name>/session/ Runner (rules engine)
|
|
55
|
+
lib/<name>/state/ GameState / PlayerState subclasses
|
|
56
|
+
data/rules.json Name, version, player count, card deck, numbers the engine uses
|
|
57
|
+
data/rules.md Rules text the model players read
|
|
58
|
+
data/personas.yml Optional: strategy and play styles for model players
|
|
59
|
+
data/bots.yml Optional: parameters the bot rules read
|
|
60
|
+
spec/
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Hooks
|
|
64
|
+
|
|
65
|
+
A game subclasses `PlaiTestr::Session::Runner`. These hooks are its public API ([`runner.rb`](lib/plai_testr/session/runner.rb)):
|
|
66
|
+
|
|
67
|
+
| Hook | Required | Purpose |
|
|
68
|
+
|---|---|---|
|
|
69
|
+
| `setup` | yes | Build the starting state. Not called on resume. |
|
|
70
|
+
| `legal_moves(player)` | yes | The menu: an array of `option(...)` entries |
|
|
71
|
+
| `apply_action_effects(player, action)` | yes | Apply the chosen move's action to `@game_state` |
|
|
72
|
+
| `next_player` | | Who acts next (default: alternate) |
|
|
73
|
+
| `pick_range(player)` | | `[min, max]` to pick several options at once |
|
|
74
|
+
| `game_over?` | | Default: `status['game_over']` |
|
|
75
|
+
| `turn_phase(player)` | | Short step label for the console and report |
|
|
76
|
+
| `rules_text_for`, `view_text_for`, `prompt_note_for` | | Prompt text: rules, the player's view, a one-off note |
|
|
77
|
+
| `recent_events_for(player)` | | Recent events from the event log (default: the last 8 this player may see) |
|
|
78
|
+
| `persona_for(player)` | | Strategy text for the system prompt (default: built from `personas.yml`) |
|
|
79
|
+
| `display_name(player)` | | Name in the console and reports (default: the persona's key, title-cased, else the player id) |
|
|
80
|
+
| `bot_choice(player, moves, pick: nil)` | | Bot rules: `nil` or `{ 'choice' => n, 'rule' => name, 'obvious' => bool }` (`'choices'` on pick turns) |
|
|
81
|
+
| `think_about?(player)` | | With `--think`: whether this turn gets thinking (default: every turn) |
|
|
82
|
+
| `debrief_questions`, `debrief_view_for` | | End-of-game feedback questions |
|
|
83
|
+
| `report_sections`, `report_debrief?` | | Extra report sections; false to render the debrief yourself |
|
|
84
|
+
| `botrun_stats` | | Game-specific numbers for one botrun game |
|
|
85
|
+
|
|
86
|
+
Helpers for hooks: `option`, `menu(player)` (the menu as the runner sees it), `data`, `log(text, to: nil, turn: decision)`, `event_log`, `persona(player)`, `persona_name(player)`, `bot_strategy_for(player)`, `@rng`, `@rules`, `@game_state`, `out`. Specs drive a runner with `setup`, `play_turn`, `play_out` and the hooks above.
|
|
87
|
+
|
|
88
|
+
### Moves
|
|
89
|
+
|
|
90
|
+
```ruby
|
|
91
|
+
def legal_moves(player)
|
|
92
|
+
hand(player).map { |card| option("Play #{card['name']}", public: "played #{card['name']}", play: { value: card['uid'] }) }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def apply_action_effects(player, action)
|
|
96
|
+
case action
|
|
97
|
+
in { play: { value: uid } } then play(player, uid)
|
|
98
|
+
in { pass: { value: uids } } then pass(player, uids)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`label` (the first argument) is what the acting player sees; `public:` is what others see; `log:` is a shorter console line. An action has one name; `value:` is the main parameter and others sit beside it. Moves and actions have symbol keys, so `apply_action_effects` can pattern-match them; a hand-written move with string keys is converted. On a pick turn the chosen values arrive together: `{ pass: { value: [42, 17, 9] } }`. Every menu is checked before it is used: a misspelled key or a malformed action raises `PlaiTestr::MoveError` naming the hook, player, decision and option. A menu with one option is applied without asking anyone.
|
|
104
|
+
|
|
105
|
+
### State
|
|
106
|
+
|
|
107
|
+
`@game_state` ([`game_state.rb`](lib/plai_testr/state/game_state.rb)) holds `turn` (decisions so far), `players`, `status` and `data`; each `player_state(name)` ([`player_state.rb`](lib/plai_testr/state/player_state.rb)) holds `score`, `hand` and `data`.
|
|
108
|
+
|
|
109
|
+
- `status` belongs to the runner: set `game_over`, `winner` and `end_reason` when the game ends, with `@game_state.apply_changes('status' => {...})`.
|
|
110
|
+
- `data` is the game's own state, as plain JSON-safe values so checkpoints work without extra code.
|
|
111
|
+
- Subclass `GameState` to name your keys (`def step = @data['step']`) and to hide information per player with `data_for(observer)`; subclass `PlayerState` and override `serialize_for` / `public_data` to choose what opponents see. The examples show both.
|
|
112
|
+
|
|
113
|
+
### Personas and bots
|
|
114
|
+
|
|
115
|
+
`data/personas.yml` gives model players a shared `basic_strategy` and play styles under `personas`; `data/bots.yml` gives the bot rules parameters, per persona, read with `bot_strategy_for(player)`. Both files are optional and the generated copies explain their format. Each player gets a persona from `--personas a,b` or at random per seed; `botrun --personas all` cycles every pairing.
|
|
116
|
+
|
|
117
|
+
### Randomness and limits
|
|
118
|
+
|
|
119
|
+
- Use `@rng` for all randomness. It is reset for each decision from the seed and the decision number, so the same seed and the same choices always give the same game, interrupted or not.
|
|
120
|
+
- When the model never gives a valid answer, the bot's choice is played instead (or a random legal move from a separate generator), and the turn is flagged.
|
|
121
|
+
- Every game stops at a decision limit (`end_reason: 'decision_limit_reached'`, no winner) so a buggy game cannot run forever. Set `max_decisions:` in `PlaiTestr.register`; otherwise the app backstop applies.
|
|
122
|
+
|
|
123
|
+
### Nested decisions and rulings
|
|
124
|
+
|
|
125
|
+
- [FrameQueue](docs/frame_queue.md): attacks and responses, triggered effects, decisions inside decisions.
|
|
126
|
+
- [Referee](docs/referee.md): yes/no rulings on free-text conditions by a second model.
|
|
127
|
+
|
|
128
|
+
## Configuration
|
|
129
|
+
|
|
130
|
+
`config/plai_testr.rb` in the app (generated, all commented out):
|
|
131
|
+
|
|
132
|
+
```ruby
|
|
133
|
+
PlaiTestr.configure do |config|
|
|
134
|
+
config.max_decisions = 100_000 # backstop for games without their own limit
|
|
135
|
+
config.player_model = 'qwen3:8b'
|
|
136
|
+
config.ollama_host = 'localhost'
|
|
137
|
+
config.ollama_port = 11434
|
|
138
|
+
config.model_settings = { context_size: 8192 }
|
|
139
|
+
end
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
A game can override model settings in `PlaiTestr.register(..., model_settings: { ... })`.
|
|
143
|
+
|
|
144
|
+
| Model setting | Default | Used for |
|
|
145
|
+
|---|---|---|
|
|
146
|
+
| `context_size` | 8192 | Menu turns without `--think`; the referee |
|
|
147
|
+
| `long_context_size` | 16384 | Sessions with `--think`; the debrief |
|
|
148
|
+
| `menu_budget` | 512 | Output tokens for a menu answer |
|
|
149
|
+
| `think_budget` | 8192 | Output tokens for a thinking turn |
|
|
150
|
+
| `debrief_budget` | 1024 | Output tokens for the debrief |
|
|
151
|
+
| `referee_budget` | 2048 | Output tokens for a ruling |
|
|
152
|
+
| `temperature` | 0.6 | Player sampling |
|
|
153
|
+
|
|
154
|
+
Values outside the hard limits are clamped with a warning: context sizes between 2048 and the model's own maximum, temperature 0–2, budgets above 0 and below the context size. The values used are recorded in each report. Environment variables set the defaults: `PLAI_TESTR_PLAYER_MODEL`, `PLAI_TESTR_REFEREE_MODEL`, `PLAI_TESTR_OLLAMA_HOST`, `PLAI_TESTR_OLLAMA_PORT`.
|
|
155
|
+
|
|
156
|
+
## Session output
|
|
157
|
+
|
|
158
|
+
- `sessions/<id>/state.json`: the checkpoint, replaced atomically after every decision
|
|
159
|
+
- `sessions/<id>/transcript.jsonl`: one JSON object per decision, with the model's reasoning. After a crash the last line may be cut off; skip lines that do not parse.
|
|
160
|
+
- `sessions/<id>/think.log`: every model reply with Ollama's timings
|
|
161
|
+
- `reports/<id>.md`: the session report
|
|
162
|
+
|
|
163
|
+
`resume` continues an interrupted session exactly, or refuses: if the game's data, code or the gem version changed since the session started, start a new one with `play`.
|
|
164
|
+
|
|
165
|
+
## Development
|
|
166
|
+
|
|
167
|
+
```sh
|
|
168
|
+
bundle install
|
|
169
|
+
bundle exec rspec
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
`lib/plai_testr/` holds the gem: [`session/runner.rb`](lib/plai_testr/session/runner.rb) (the decision loop and hooks), [`agents/`](lib/plai_testr/agents) (Ollama client, player and referee prompts), [`reporter/`](lib/plai_testr/reporter), [`state/`](lib/plai_testr/state), [`cli.rb`](lib/plai_testr/cli.rb), [`preflight.rb`](lib/plai_testr/preflight.rb), [`botrun.rb`](lib/plai_testr/botrun.rb). `templates/` holds what `new` and `generate game` copy, including the example games and their specs.
|
|
173
|
+
|
|
174
|
+
## License
|
|
175
|
+
|
|
176
|
+
MIT. See [LICENSE.txt](LICENSE.txt).
|
data/bin/plai_testr
ADDED
data/docs/frame_queue.md
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# Nested decisions with FrameQueue
|
|
2
|
+
|
|
3
|
+
Most turns are one decision: the active player picks from a menu. Some games need decisions *inside* other decisions: an attack the defender may block, a card that triggers an opponent's response, an effect that asks someone else to choose. `PlaiTestr::Session::FrameQueue` handles these.
|
|
4
|
+
|
|
5
|
+
## The idea
|
|
6
|
+
|
|
7
|
+
A **frame** is one pending step: a hash with a `type`, the `player` who decides it (if anyone), and any data the step needs. Frames live on a stack in game state, so a checkpoint saved in the middle of an attack resumes in the middle of that attack.
|
|
8
|
+
|
|
9
|
+
- The frame on top of the stack is the current decision. `next_player` and `legal_moves` read it.
|
|
10
|
+
- When the player picks a move, `resolve_frame(params)` pops the frame and calls `on_<type>(frame, params)`.
|
|
11
|
+
- Handlers push new frames for whatever happens next. **The last frame pushed resolves first.**
|
|
12
|
+
- Frames marked `auto: true` need no decision: they are popped and resolved with `on_<type>(frame)` as soon as they reach the top.
|
|
13
|
+
- When the stack is empty, `schedule_frames` pushes the next normal turn.
|
|
14
|
+
|
|
15
|
+
## What you implement
|
|
16
|
+
|
|
17
|
+
| Method | Purpose |
|
|
18
|
+
|---|---|
|
|
19
|
+
| `frame_queue` | The array that holds the frames, kept in `@game_state.data` |
|
|
20
|
+
| `schedule_frames` | Push the next turn's frame(s) when the stack is empty |
|
|
21
|
+
| `on_<type>(frame, params)` | Resolve a decision frame |
|
|
22
|
+
| `on_<type>(frame)` | Resolve an `auto` frame |
|
|
23
|
+
| `halt_frames?` | Optional. Stop resolving (default: `game_over?`). A good place for end checks and safety caps |
|
|
24
|
+
| `intercept_frame(frame)` | Optional. Return true to stop before a frame and handle it in your own code (e.g. hand maintenance) |
|
|
25
|
+
|
|
26
|
+
You call `advance_frames` once at the end of `setup`, and `resolve_frame(action.values.first)` from `apply_action_effects`. `frame_for(type, player, **data)` builds a frame. The hooks in the table are public; `advance_frames`, `resolve_frame`, `frame` and `frame_for` are private helpers your runner calls.
|
|
27
|
+
|
|
28
|
+
## Example: attack and block
|
|
29
|
+
|
|
30
|
+
Each turn the attacker plays a card. The defender may block with a card of equal or higher value, or take the damage. Damage is an `auto` frame, so it resolves without a decision; reaching 10 damage loses.
|
|
31
|
+
|
|
32
|
+
```ruby
|
|
33
|
+
module Strike
|
|
34
|
+
module Session
|
|
35
|
+
class Runner < PlaiTestr::Session::Runner
|
|
36
|
+
include PlaiTestr::Session::FrameQueue
|
|
37
|
+
|
|
38
|
+
HAND_SIZE = 4
|
|
39
|
+
KNOCKOUT = 10
|
|
40
|
+
|
|
41
|
+
def setup
|
|
42
|
+
deck = PlaiTestr::Deck.build(@rules).shuffle(random: @rng)
|
|
43
|
+
players.each { |p| hand(p).concat(deck.shift(HAND_SIZE)) }
|
|
44
|
+
data.merge!('queue' => [], 'deck' => deck, 'damage' => players.to_h { |p| [p, 0] }, 'attacker' => players.first)
|
|
45
|
+
advance_frames
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def frame_queue = data['queue']
|
|
49
|
+
|
|
50
|
+
def schedule_frames
|
|
51
|
+
attacker = data['attacker']
|
|
52
|
+
hand(attacker).concat(data['deck'].shift(HAND_SIZE - hand(attacker).length))
|
|
53
|
+
frame_queue.push(frame_for('attack', attacker))
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def halt_frames?
|
|
57
|
+
end_game(nil, 'deck_empty') if !game_over? && data['deck'].empty? && hand(data['attacker']).empty?
|
|
58
|
+
game_over?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def next_player = frame['player']
|
|
62
|
+
|
|
63
|
+
def legal_moves(player)
|
|
64
|
+
case frame['type']
|
|
65
|
+
when 'attack'
|
|
66
|
+
hand(player).map { |c| option("Attack with #{c['name']}", attack: { value: c['uid'] }) }
|
|
67
|
+
when 'block'
|
|
68
|
+
blockers = hand(player).select { |c| c['value'] >= frame['value'] }
|
|
69
|
+
blockers.map { |c| option("Block with #{c['name']}", block: { value: c['uid'] }) } + [option("Take #{frame['value']} damage", block: {})]
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def apply_action_effects(_player, action) = resolve_frame(action.values.first)
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def on_attack(frame, params)
|
|
78
|
+
card = take(frame['player'], params[:value])
|
|
79
|
+
defender = opponent(frame['player'])
|
|
80
|
+
frame_queue.push(frame_for('block', defender, value: card['value']))
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def on_block(frame, params)
|
|
84
|
+
if params[:value]
|
|
85
|
+
take(frame['player'], params[:value])
|
|
86
|
+
else
|
|
87
|
+
frame_queue.push(frame_for('damage', frame['player'], auto: true, amount: frame['value']))
|
|
88
|
+
end
|
|
89
|
+
data['attacker'] = frame['player']
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def on_damage(frame)
|
|
93
|
+
data['damage'][frame['player']] += frame['amount']
|
|
94
|
+
end_game(opponent(frame['player']), 'knockout') if data['damage'][frame['player']] >= KNOCKOUT
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def end_game(winner, reason)
|
|
98
|
+
@game_state.apply_changes('status' => { 'game_over' => true, 'winner' => winner, 'end_reason' => reason })
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def take(player, uid) = hand(player).delete_at(hand(player).index { |c| c['uid'] == uid })
|
|
102
|
+
def opponent(player) = (players - [player]).first
|
|
103
|
+
def players = @game_state.players
|
|
104
|
+
def hand(player) = @game_state.player_state(player).hand
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The sequence for one exchange:
|
|
111
|
+
|
|
112
|
+
1. The stack is empty, so `schedule_frames` pushes `attack` for Player 1.
|
|
113
|
+
2. Player 1 attacks; `on_attack` pushes `block` for Player 2. It is not `auto`, so play stops for Player 2's decision.
|
|
114
|
+
3. Player 2 takes the damage; `on_block` pushes an `auto` `damage` frame, which resolves at once.
|
|
115
|
+
4. The stack is empty again, so `schedule_frames` pushes `attack` for Player 2.
|
|
116
|
+
|
|
117
|
+
## Tips
|
|
118
|
+
|
|
119
|
+
- **Stack order.** To have A happen before B, push B first, then A.
|
|
120
|
+
- **Pick-N frames.** Put `pick: [min, max]` on the frame and return `frame['pick']` from `pick_range`.
|
|
121
|
+
- **Say what the frame is for.** `turn_phase` can return `frame['type']`, and views should say why the player is deciding ("Player 1 attacks with Blade 5: block or take 5 damage").
|
|
122
|
+
- **Cap runaway auto-frame loops in `halt_frames?`.** The decision limit (`max_decisions`) counts only decisions, so a loop of `auto` frames never reaches it. Count resolved frames and end the game past a limit the players never see.
|
|
123
|
+
- **Bot rules** read `frame['type']` the same way `legal_moves` does.
|
data/docs/referee.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Yes/no rulings with the referee
|
|
2
|
+
|
|
3
|
+
Prefer encoding rules in Ruby: it is fast, deterministic and testable. Some conditions are awkward to encode, though, especially free-text card conditions in a game that is still changing ("If your opponent is ahead…", "If this is your first attack this turn…"). The referee is a second model that answers one yes/no question about such a condition.
|
|
4
|
+
|
|
5
|
+
## How it works
|
|
6
|
+
|
|
7
|
+
Run with `--referee <model>` (or set `PLAI_TESTR_REFEREE_MODEL`). The runner then has `@referee`, an `Agents::RefereeAgent`; without the option, `@referee` is `nil`.
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
@referee.ruling(question: 'Is the condition on Ambush met?', facts: "...")
|
|
11
|
+
# => { 'met' => true, 'why' => 'The opponent has 12 points and the player has 8.', 'fallback' => false }
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
The referee sees only the game's name, the question and the facts you pass, with thinking on and temperature 0.1. If it cannot produce a valid answer after retries, it returns `met: false` with `fallback: true`.
|
|
15
|
+
|
|
16
|
+
Nothing calls the referee for you, and rulings are not added to reports automatically. Your runner asks, records and reports.
|
|
17
|
+
|
|
18
|
+
## Example: a card with a free-text condition
|
|
19
|
+
|
|
20
|
+
Cards in `rules.json` carry their condition as text:
|
|
21
|
+
|
|
22
|
+
```json
|
|
23
|
+
{ "name": "Ambush", "value": 2, "condition": "your opponent has more points than you", "bonus": 3, "copies": 2 }
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
When a card with a condition is played, the runner asks for a ruling and records it (`data` and `log` are runner helpers):
|
|
27
|
+
|
|
28
|
+
```ruby
|
|
29
|
+
def bonus_for(player, card)
|
|
30
|
+
return 0 unless card['condition']
|
|
31
|
+
|
|
32
|
+
ruling = ruling("Is this condition met for #{player}: #{card['condition']}?", view_text_for(player))
|
|
33
|
+
ruling['met'] ? card['bonus'] : 0
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def ruling(question, facts)
|
|
37
|
+
result = if @referee
|
|
38
|
+
@referee.ruling(question: question, facts: facts)
|
|
39
|
+
else
|
|
40
|
+
{ 'met' => false, 'why' => 'no referee: run with --referee to rule on conditions', 'fallback' => true }
|
|
41
|
+
end
|
|
42
|
+
(data['rulings'] ||= []) << result.merge('turn' => @game_state.turn + 1, 'question' => question)
|
|
43
|
+
log("Ruling: #{question} #{result['met'] ? 'Yes' : 'No'} (#{result['why']})")
|
|
44
|
+
result
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def report_sections
|
|
48
|
+
rows = data.fetch('rulings', []).map { |r| "| #{r['turn']} | #{r['question']} | #{r['met'] ? 'yes' : 'no'} | #{r['why']}#{' (fallback)' if r['fallback']} |" }
|
|
49
|
+
["## Rulings\n\n| Turn | Question | Met | Why |\n|---|---|---|---|\n#{rows.join("\n")}"]
|
|
50
|
+
end
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Things to know
|
|
54
|
+
|
|
55
|
+
- **Facts are everything.** The referee knows nothing about the game state except the facts string. Passing the deciding player's view (`view_text_for`) is a good default; add anything the question depends on.
|
|
56
|
+
- **Botruns and `--mode bot` have no referee.** Decide the fallback on purpose (above: the condition is not met) and say so in the log, or give the bot its own Ruby check for the condition.
|
|
57
|
+
- **Rulings break seed reproducibility.** The same seed can play differently when a ruling changes. Record every ruling, as above, so a report shows why.
|
|
58
|
+
- **Rulings are slow.** Each is a model call with thinking on. Ask only when the condition actually matters (a card with a condition was played), never on every turn.
|
|
59
|
+
- **Turn rulings into code.** When a condition comes up often and the designer agrees on what it means, encode it in Ruby and keep the referee for the rest.
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
module PlaiTestr
|
|
2
|
+
module Agents
|
|
3
|
+
class OllamaClient
|
|
4
|
+
READ_TIMEOUT = 300
|
|
5
|
+
MAX_RETRIES = 3
|
|
6
|
+
CONTEXT_WARN_RATIO = 0.9
|
|
7
|
+
SPINNER_FRAMES = %w[⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏].freeze
|
|
8
|
+
UNREACHABLE = [Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::EADDRNOTAVAIL, SocketError, Net::OpenTimeout].freeze
|
|
9
|
+
|
|
10
|
+
def initialize(host: PlaiTestr.config.ollama_host, port: PlaiTestr.config.ollama_port, think_log: nil, out: $stdout, verbose: false)
|
|
11
|
+
@host = host
|
|
12
|
+
@port = port
|
|
13
|
+
@think_log = think_log
|
|
14
|
+
@out = out
|
|
15
|
+
@verbose = verbose
|
|
16
|
+
@frame_index = 0
|
|
17
|
+
@tty = out.tty?
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# `think`, `format` and `keep_alive` are top-level request fields in Ollama's /api/chat;
|
|
21
|
+
# only sampling/runtime settings belong in `options`. `format` may be "json" or a JSON schema.
|
|
22
|
+
def chat(model:, messages:, temperature: 0.7, think: nil, num_predict: nil, num_ctx: nil, format: nil, keep_alive: nil)
|
|
23
|
+
options = { temperature: temperature }
|
|
24
|
+
options[:num_predict] = num_predict unless num_predict.nil?
|
|
25
|
+
options[:num_ctx] = num_ctx unless num_ctx.nil?
|
|
26
|
+
|
|
27
|
+
payload = { model: model, messages: messages, stream: true, options: options }
|
|
28
|
+
payload[:think] = think unless think.nil?
|
|
29
|
+
payload[:format] = format unless format.nil?
|
|
30
|
+
payload[:keep_alive] = keep_alive unless keep_alive.nil?
|
|
31
|
+
|
|
32
|
+
response = post('/api/chat', payload)
|
|
33
|
+
used = response['prompt_eval_count']
|
|
34
|
+
return response unless used && num_ctx && used >= num_ctx * CONTEXT_WARN_RATIO
|
|
35
|
+
|
|
36
|
+
File.open(@think_log, 'a') { |f| f.puts "[WARNING] context nearly full: #{used}/#{num_ctx} prompt tokens (model=#{model})\n---\n\n" } if @think_log
|
|
37
|
+
response.merge('context_full' => true)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def model_names
|
|
41
|
+
request(Net::HTTP::Get.new('/api/tags')) { |response| JSON.parse(response.body) }.fetch('models', []).map { |m| m['name'] }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def context_length(model)
|
|
45
|
+
info = request(json_post('/api/show', model: model)) { |response| JSON.parse(response.body) }.fetch('model_info', {})
|
|
46
|
+
info["#{info['general.architecture']}.context_length"]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def pull(model)
|
|
50
|
+
request(json_post('/api/pull', model: model, stream: true), read_timeout: nil) do |response|
|
|
51
|
+
each_line(response) do |data|
|
|
52
|
+
raise ModelError, "Pulling #{model} failed: #{data['error']}" if data['error']
|
|
53
|
+
|
|
54
|
+
@out.print "\r Pulling #{model}: #{data['status']}#{" #{(100.0 * data['completed'] / data['total']).round}%" if data['total'].to_i.positive? && data['completed']}\e[K"
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
@out.puts
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
def json_post(path, payload)
|
|
63
|
+
Net::HTTP::Post.new(path, 'Content-Type' => 'application/json').tap { |r| r.body = JSON.generate(payload) }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def request(req, read_timeout: 10, &block)
|
|
67
|
+
http = Net::HTTP.new(@host, @port)
|
|
68
|
+
http.open_timeout = 5
|
|
69
|
+
http.read_timeout = read_timeout
|
|
70
|
+
http.request(req) do |response|
|
|
71
|
+
raise ModelError, "Ollama API error #{response.code} for #{req.path}: #{response.read_body}" unless response.is_a?(Net::HTTPSuccess)
|
|
72
|
+
|
|
73
|
+
return block.call(response)
|
|
74
|
+
end
|
|
75
|
+
rescue *UNREACHABLE => e
|
|
76
|
+
raise ModelError, "Cannot reach Ollama at #{@host}:#{@port} (#{e.class}: #{e.message})"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def each_line(response)
|
|
80
|
+
buffer = +''
|
|
81
|
+
response.read_body do |chunk|
|
|
82
|
+
buffer << chunk
|
|
83
|
+
while (newline = buffer.index("\n"))
|
|
84
|
+
data = parse_line(buffer.slice!(0, newline + 1))
|
|
85
|
+
yield data if data
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
data = parse_line(buffer)
|
|
89
|
+
yield data if data
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def parse_line(line)
|
|
93
|
+
JSON.parse(line) unless line.strip.empty?
|
|
94
|
+
rescue JSON::ParserError
|
|
95
|
+
nil
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def post(path, payload)
|
|
99
|
+
attempts = 0
|
|
100
|
+
begin
|
|
101
|
+
attempts += 1
|
|
102
|
+
post_once(path, payload)
|
|
103
|
+
rescue Net::ReadTimeout, Errno::ECONNRESET, EOFError => e
|
|
104
|
+
clear_spinner
|
|
105
|
+
raise ModelError, "Ollama stopped responding (#{e.class}) after #{attempts} tries" if attempts >= MAX_RETRIES
|
|
106
|
+
|
|
107
|
+
wait = 2**attempts
|
|
108
|
+
warn " [connection error: #{e.class} — retrying in #{wait}s]"
|
|
109
|
+
sleep wait
|
|
110
|
+
retry
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def post_once(path, payload)
|
|
115
|
+
content = +''
|
|
116
|
+
thinking = +''
|
|
117
|
+
final_chunk = nil
|
|
118
|
+
|
|
119
|
+
request(json_post(path, payload), read_timeout: READ_TIMEOUT) do |response|
|
|
120
|
+
each_line(response) do |data|
|
|
121
|
+
piece = data.dig('message', 'content').to_s
|
|
122
|
+
content << piece
|
|
123
|
+
thinking << data.dig('message', 'thinking').to_s
|
|
124
|
+
final_chunk = data if data['done']
|
|
125
|
+
@verbose ? @out.print(piece) : advance_spinner
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
clear_spinner
|
|
130
|
+
raise ModelError, 'Ollama API error: stream ended without a done chunk' unless final_chunk
|
|
131
|
+
|
|
132
|
+
@out.puts "\n [#{format_stats(final_chunk).strip}]" if @verbose
|
|
133
|
+
write_think_log(payload[:model], thinking.empty? ? content : "<think>\n#{thinking.strip}\n</think>\n#{content}", final_chunk)
|
|
134
|
+
final_chunk.merge('message' => { 'role' => 'assistant', 'content' => content })
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def write_think_log(model, content, stats = {})
|
|
138
|
+
return unless @think_log
|
|
139
|
+
|
|
140
|
+
File.open(@think_log, 'a') do |f|
|
|
141
|
+
f.puts "[#{Time.now.utc.iso8601}] model=#{model}#{format_stats(stats)}"
|
|
142
|
+
f.puts content
|
|
143
|
+
f.puts '---'
|
|
144
|
+
f.puts
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def format_stats(stats)
|
|
149
|
+
nanos = 1_000_000_000.0
|
|
150
|
+
rate = ->(count, duration) { count && duration.to_i.positive? ? (count / (duration / nanos)).round(1) : nil }
|
|
151
|
+
{
|
|
152
|
+
'prompt_tokens' => stats['prompt_eval_count'],
|
|
153
|
+
'prompt_tps' => rate.call(stats['prompt_eval_count'], stats['prompt_eval_duration']),
|
|
154
|
+
'eval_tokens' => stats['eval_count'],
|
|
155
|
+
'eval_tps' => rate.call(stats['eval_count'], stats['eval_duration']),
|
|
156
|
+
'load_s' => stats['load_duration'] && (stats['load_duration'] / nanos).round(2),
|
|
157
|
+
'total_s' => stats['total_duration'] && (stats['total_duration'] / nanos).round(2)
|
|
158
|
+
}.filter_map { |key, value| " #{key}=#{value}" unless value.nil? }.join
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def advance_spinner
|
|
162
|
+
return unless @tty
|
|
163
|
+
|
|
164
|
+
@out.print "\r #{SPINNER_FRAMES[@frame_index % SPINNER_FRAMES.length]}"
|
|
165
|
+
@out.flush
|
|
166
|
+
@frame_index += 1
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def clear_spinner
|
|
170
|
+
return unless @tty
|
|
171
|
+
|
|
172
|
+
@out.print "\r \r"
|
|
173
|
+
@out.flush
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|