plai_testr 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +16 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +176 -0
  5. data/bin/plai_testr +5 -0
  6. data/docs/frame_queue.md +123 -0
  7. data/docs/referee.md +59 -0
  8. data/lib/plai_testr/agents/ollama_client.rb +177 -0
  9. data/lib/plai_testr/agents/player_agent.rb +216 -0
  10. data/lib/plai_testr/agents/referee_agent.rb +56 -0
  11. data/lib/plai_testr/botrun.rb +60 -0
  12. data/lib/plai_testr/cli.rb +190 -0
  13. data/lib/plai_testr/config.rb +39 -0
  14. data/lib/plai_testr/deck.rb +11 -0
  15. data/lib/plai_testr/errors.rb +9 -0
  16. data/lib/plai_testr/event_log.rb +24 -0
  17. data/lib/plai_testr/game.rb +50 -0
  18. data/lib/plai_testr/generator.rb +50 -0
  19. data/lib/plai_testr/model_settings.rb +25 -0
  20. data/lib/plai_testr/names.rb +7 -0
  21. data/lib/plai_testr/personas.rb +97 -0
  22. data/lib/plai_testr/preflight.rb +70 -0
  23. data/lib/plai_testr/reporter/botrun_report.rb +141 -0
  24. data/lib/plai_testr/reporter/model_comparison.rb +104 -0
  25. data/lib/plai_testr/reporter/report_builder.rb +154 -0
  26. data/lib/plai_testr/reporter/think_log.rb +32 -0
  27. data/lib/plai_testr/rules_loader.rb +40 -0
  28. data/lib/plai_testr/session/frame_queue.rb +47 -0
  29. data/lib/plai_testr/session/manager.rb +77 -0
  30. data/lib/plai_testr/session/runner.rb +555 -0
  31. data/lib/plai_testr/state/base.rb +13 -0
  32. data/lib/plai_testr/state/game_state.rb +102 -0
  33. data/lib/plai_testr/state/player_state.rb +61 -0
  34. data/lib/plai_testr/version.rb +3 -0
  35. data/lib/plai_testr.rb +61 -0
  36. data/templates/app/.gitignore +3 -0
  37. data/templates/app/.rspec +3 -0
  38. data/templates/app/AGENTS.md +158 -0
  39. data/templates/app/CLAUDE.md +1 -0
  40. data/templates/app/Gemfile.erb +7 -0
  41. data/templates/app/README.md.erb +91 -0
  42. data/templates/app/config/plai_testr.rb +15 -0
  43. data/templates/app/spec/spec_helper.rb +25 -0
  44. data/templates/examples/gin_rummy/NOTES.md +12 -0
  45. data/templates/examples/gin_rummy/data/bots.yml +18 -0
  46. data/templates/examples/gin_rummy/data/personas.yml +26 -0
  47. data/templates/examples/gin_rummy/data/rules.json +76 -0
  48. data/templates/examples/gin_rummy/data/rules.md +44 -0
  49. data/templates/examples/gin_rummy/lib/gin_rummy/bot.rb +34 -0
  50. data/templates/examples/gin_rummy/lib/gin_rummy/game.rb +8 -0
  51. data/templates/examples/gin_rummy/lib/gin_rummy/melds.rb +51 -0
  52. data/templates/examples/gin_rummy/lib/gin_rummy/session/runner.rb +197 -0
  53. data/templates/examples/gin_rummy/lib/gin_rummy/state/game_state.rb +19 -0
  54. data/templates/examples/gin_rummy/lib/gin_rummy.rb +9 -0
  55. data/templates/examples/gin_rummy/spec/melds_spec.rb +26 -0
  56. data/templates/examples/gin_rummy/spec/session/runner_spec.rb +105 -0
  57. data/templates/examples/hearts/data/bots.yml +22 -0
  58. data/templates/examples/hearts/data/personas.yml +27 -0
  59. data/templates/examples/hearts/data/rules.json +73 -0
  60. data/templates/examples/hearts/data/rules.md +30 -0
  61. data/templates/examples/hearts/lib/hearts/bot.rb +53 -0
  62. data/templates/examples/hearts/lib/hearts/game.rb +16 -0
  63. data/templates/examples/hearts/lib/hearts/reporter/botrun_report.rb +29 -0
  64. data/templates/examples/hearts/lib/hearts/session/runner.rb +255 -0
  65. data/templates/examples/hearts/lib/hearts/state/game_state.rb +19 -0
  66. data/templates/examples/hearts/lib/hearts.rb +11 -0
  67. data/templates/examples/hearts/spec/session/runner_spec.rb +122 -0
  68. data/templates/game/data/bots.yml +13 -0
  69. data/templates/game/data/personas.yml +16 -0
  70. data/templates/game/data/rules.json.erb +16 -0
  71. data/templates/game/data/rules.md.erb +11 -0
  72. data/templates/game/lib/__name__/bot.rb.erb +7 -0
  73. data/templates/game/lib/__name__/session/runner.rb.erb +36 -0
  74. data/templates/game/lib/__name__/state/game_state.rb.erb +11 -0
  75. data/templates/game/lib/__name__.rb.erb +10 -0
  76. data/templates/game/spec/session/runner_spec.rb.erb +29 -0
  77. metadata +153 -0
@@ -0,0 +1,61 @@
1
+ module PlaiTestr
2
+ module State
3
+ class PlayerState < Base
4
+ attr_reader :name, :score, :hand, :data
5
+
6
+ def initialize(name:, score: 0, hand: [], data: {})
7
+ @name = name
8
+ @score = score.to_i
9
+ @hand = hand.dup
10
+ @data = data.dup
11
+ end
12
+
13
+ def self.from_h(data)
14
+ new(
15
+ name: data['name'],
16
+ score: data.fetch('score', 0),
17
+ hand: data.fetch('hand', []),
18
+ data: data.fetch('data', {})
19
+ )
20
+ end
21
+
22
+ def apply_changes(changes)
23
+ return if changes.nil? || changes.empty?
24
+
25
+ @score = changes['score'] if changes.key?('score')
26
+ @hand = changes['hand'] if changes.key?('hand')
27
+ @data.merge!(changes['data']) if changes['data']
28
+ end
29
+
30
+ # Full state — for checkpoints
31
+ def serialize
32
+ {
33
+ 'name' => @name,
34
+ 'score' => @score,
35
+ 'hand' => @hand,
36
+ 'data' => @data
37
+ }
38
+ end
39
+
40
+ # Opponent-visible state — hand contents are hidden, only size shown
41
+ # Subclasses override to expose game-specific public fields (e.g. top stance card)
42
+ def serialize_for(observer)
43
+ return serialize if observer == @name
44
+
45
+ {
46
+ 'name' => @name,
47
+ 'score' => @score,
48
+ 'hand_size' => @hand.length,
49
+ 'data' => public_data
50
+ }
51
+ end
52
+
53
+ private
54
+
55
+ # Override in subclasses to selectively expose data to opponents
56
+ def public_data
57
+ {}
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,3 @@
1
+ module PlaiTestr
2
+ VERSION = '0.1.1'
3
+ end
data/lib/plai_testr.rb ADDED
@@ -0,0 +1,61 @@
1
+ require_relative 'plai_testr/version'
2
+ require 'digest'
3
+ require 'fileutils'
4
+ require 'json'
5
+ require 'net/http'
6
+ require 'optparse'
7
+ require 'socket'
8
+ require 'stringio'
9
+ require 'time'
10
+ require 'uri'
11
+
12
+ module PlaiTestr
13
+ DECIDER_ICONS = { 'LLM' => '🧠', 'BOT' => '🤖', 'RNG' => '🎲' }.freeze
14
+
15
+ class << self
16
+ attr_writer :sessions_dir, :reports_dir
17
+
18
+ def sessions_dir
19
+ @sessions_dir || File.join(Dir.pwd, 'sessions')
20
+ end
21
+
22
+ def reports_dir
23
+ @reports_dir || File.join(Dir.pwd, 'reports')
24
+ end
25
+
26
+ def games_dir
27
+ @games_dir || File.join(Dir.pwd, 'games')
28
+ end
29
+
30
+ def games_dir=(dir)
31
+ @games_dir = dir && File.expand_path(dir)
32
+ end
33
+ end
34
+ end
35
+
36
+ require_relative 'plai_testr/errors'
37
+ require_relative 'plai_testr/config'
38
+ require_relative 'plai_testr/model_settings'
39
+ require_relative 'plai_testr/agents/ollama_client'
40
+ require_relative 'plai_testr/agents/player_agent'
41
+ require_relative 'plai_testr/agents/referee_agent'
42
+ require_relative 'plai_testr/rules_loader'
43
+ require_relative 'plai_testr/personas'
44
+ require_relative 'plai_testr/reporter/think_log'
45
+ require_relative 'plai_testr/reporter/report_builder'
46
+ require_relative 'plai_testr/reporter/model_comparison'
47
+ require_relative 'plai_testr/reporter/botrun_report'
48
+ require_relative 'plai_testr/session/manager'
49
+ require_relative 'plai_testr/session/frame_queue'
50
+ require_relative 'plai_testr/session/runner'
51
+ require_relative 'plai_testr/state/base'
52
+ require_relative 'plai_testr/state/player_state'
53
+ require_relative 'plai_testr/state/game_state'
54
+ require_relative 'plai_testr/botrun'
55
+ require_relative 'plai_testr/preflight'
56
+ require_relative 'plai_testr/game'
57
+ require_relative 'plai_testr/deck'
58
+ require_relative 'plai_testr/event_log'
59
+ require_relative 'plai_testr/names'
60
+ require_relative 'plai_testr/generator'
61
+ require_relative 'plai_testr/cli'
@@ -0,0 +1,3 @@
1
+ .bundle/
2
+ games/*/sessions/
3
+ games/*/reports/
@@ -0,0 +1,3 @@
1
+ --require spec_helper
2
+ -I spec
3
+ --default-path .
@@ -0,0 +1,158 @@
1
+ # Agent guide
2
+
3
+ This app playtests tabletop games with [PlAI Testr](https://github.com/armahillo/plai_testr). Local LLM players (via Ollama) and scripted bots play a game; every session produces a markdown report for the designer. If you are an AI assistant helping build or change a test harness here, read this first.
4
+
5
+ ## Start here
6
+
7
+ When you open this project:
8
+
9
+ 1. Read this file, then `README.md`.
10
+ 2. Run `bundle exec rspec`. It is fast, needs no model, and shows the state of every game.
11
+ 3. Look at `games/`. Each game's `data/rules.md` is its rulebook, and `NOTES.md` (if present) holds rulings and open questions.
12
+ 4. Read the gem's docs when you need them. They ship with the installed gem, so they match the version in use; `bundle info plai_testr --path` prints its directory:
13
+ - `README.md`: every runner hook, the state classes, configuration
14
+ - `docs/frame_queue.md`: decisions inside decisions (attacks and responses, triggered effects), with a complete example
15
+ - `docs/referee.md`: yes/no rulings on free-text conditions by a second model, with an example
16
+ - `templates/examples/`: pristine copies of the example games, in case they were deleted from `games/`
17
+
18
+ The example games show working patterns to copy:
19
+
20
+ - `games/hearts`: 4 players, hidden hands, a pick-3 pass (`pick_range`), a session report section (`report_sections`), a botrun report subclass (`botrun_stats` + `Reporter::BotrunReport`), and personas whose bots play differently (`lib/hearts/bot.rb`, `data/personas.yml`, `data/bots.yml`)
21
+ - `games/gin_rummy`: 2 players, engine-computed facts on every option, melds computed in plain Ruby (`melds.rb`), a bot (`lib/gin_rummy/bot.rb`) with a per-persona parameter (`knock_at` in `data/bots.yml`), and a **Simplifications** section in `data/rules.md`
22
+
23
+ ## Who you are working for
24
+
25
+ The user is a game designer. The harness exists to tell them how their game plays, so:
26
+
27
+ - Never change the game's rules to make the harness easier. When the rules are unclear or silent, ask the designer and record the answer (see *Rulings*).
28
+ - Ask before running model games (`play`, `resume`, `preflight`) or pulling models: they take minutes to hours and use the machine. Specs and `botrun` are fast and safe to run.
29
+ - Bot results describe how the bot plays, not how people play. Say so when reporting them.
30
+
31
+ ## Layout
32
+
33
+ ```
34
+ games/<name>/
35
+ lib/<name>.rb Requires the game and calls PlaiTestr.register
36
+ lib/<name>/game.rb Constants
37
+ lib/<name>/state/ GameState (and PlayerState) subclasses: what is stored, what each player may see
38
+ lib/<name>/session/runner.rb The rules engine: steps, legal moves, effects, views, prompts
39
+ lib/<name>/bot.rb Bot rules (bot_choice), included in the runner
40
+ lib/<name>/reporter/ Optional report classes
41
+ data/rules.json Machine-readable rules: game name, version, player_count, card_deck, numbers the engine uses
42
+ data/rules.md Rules text the model players read
43
+ data/personas.yml Optional: shared strategy and play styles the model players follow
44
+ data/bots.yml Optional: parameters the bot reads, per persona
45
+ NOTES.md Optional: open questions and unconfirmed engine choices
46
+ spec/ The game's specs
47
+ sessions/ reports/ Output (gitignored)
48
+ ```
49
+
50
+ Every file a game needs lives under `games/<name>/`, so a game is removed by deleting its directory. Keep it that way. Shared code goes in the plai_testr gem, not in another game.
51
+
52
+ App-wide settings (the decision limit backstop, model, Ollama host, model settings) live in `config/plai_testr.rb`. A game can set its own `max_decisions:` and `model_settings:` in `PlaiTestr.register`.
53
+
54
+ `bundle exec plai_testr generate game <name>` creates the skeleton. `games/hearts` (4 players, hidden hands, a pick-3 pass) and `games/gin_rummy` (2 players, draw/discard turns, engine-computed melds) are complete examples.
55
+
56
+ ## How a game runs
57
+
58
+ Each turn the runner asks `next_player` who acts, builds a numbered menu from `legal_moves(player)`, and the player (model or bot) picks a number. `apply_action_effects` applies the chosen move's `action` to `@game_state`. The game ends when `status['game_over']` is true; set `status['winner']` and `status['end_reason']` too (`@game_state.apply_changes('status' => {...})`). A game that never ends is stopped at the decision limit with `end_reason: 'decision_limit_reached'`.
59
+
60
+ Build each menu entry with `option(label, public: ..., log: ..., name => params)`:
61
+
62
+ ```ruby
63
+ option('Play 3♣', public: 'played 3♣', play: { value: card['uid'] })
64
+ # => { label: 'Play 3♣', public: 'played 3♣', action: { play: { value: 42 } } }
65
+
66
+ def apply_action_effects(player, action)
67
+ case action
68
+ in { play: { value: uid } } then play(player, uid)
69
+ end
70
+ end
71
+ ```
72
+
73
+ `label` is what the acting player sees, `public` what other players see, `log` the short console text. The action has exactly one name; `value:` is the main parameter, and others sit beside it. Moves and actions use symbol keys (the runner converts hand-written string keys), so `apply_action_effects` and the bot rules can pattern-match them. The runner checks every menu before using it and raises `PlaiTestr::MoveError` for a misspelled key or a malformed action.
74
+
75
+ A menu with one move is applied without asking anyone. `pick_range(player)` returning `[min, max]` turns the menu into a pick-N choice, applied as one action whose `value` lists the chosen moves' values: `{ pass: { value: [42, 17, 9] } }`.
76
+
77
+ Randomness comes from `@rng`. It is reset for each decision from the seed and the decision number, so the same seed and the same choices always give the same game, interrupted or not.
78
+
79
+ The gem's `README.md` (see *Start here*) lists every hook.
80
+
81
+ ## Building a harness: the order that works
82
+
83
+ 1. **Rules first.** Put the rules text in `data/rules.md` and the numbers in `data/rules.json`. Read them closely; list every gap or ambiguity and ask the designer before coding around it.
84
+ 2. **State.** Decide what lives in `data` (`@game_state.data`: decks, piles, step, active player). `status` belongs to the runner. Use stable ids (card `uid`s, fixed pile indexes); never re-index things players refer to. A GameState subclass can name its keys with readers (`def step = @data['step']`).
85
+ 3. **Steps.** Model a turn as a small state machine in `data['step']`. `legal_moves` and `apply_action_effects` switch on it.
86
+ 4. **Specs for the rules.** Before any model plays, write specs that set up a situation and check the menu (`legal_moves`) and the effects. This is where rules bugs are cheapest.
87
+ 5. **Bot rules.** Implement `bot_choice` so the game can play itself. Then `bundle exec plai_testr <game> botrun --games 1000` must finish with no crashes and sensible endings. Add a botrun spec over a few seeds.
88
+ 6. **Views and prompts.** Write `view_text_for`, `rules_text_for`, `turn_phase`, `debrief_questions`. Read a few prompts yourself (see *Checking a prompt*).
89
+ 7. **Model games.** With the designer's go-ahead: one game with `--mode hybrid` (the bot plays obvious moves), then `--mode llm`.
90
+
91
+ ## Designing what the model sees
92
+
93
+ Small local models (4B–14B) play well only when the engine does the bookkeeping.
94
+
95
+ - **The engine lists the legal moves; the model picks a number.** Never ask the model to name a card, type a move, or check legality.
96
+ - **Put engine-computed facts on each option, never advice.** "Discard 9♥ (deadwood after: 31)" is a fact; "Discard 9♥ (best)" decides for the model and ruins the test.
97
+ - **Generate views from state; never hand-write summaries.** Hand-written summaries leak hidden information or go stale.
98
+ - **Show each player only what they may know.** Override `data_for(observer)` in GameState (and `serialize_for` in PlayerState) to hide decks and other hands. Log events with `log(text)`, and `log(text, to: player)` for private ones ("You drew the 2♠"); `recent_events_for` shows each player what they may see.
99
+ - **Keep menus short.** Merge equivalent options (identical cards, interchangeable targets). Split one huge decision into steps.
100
+ - **Hide safety caps from players.** Turn or hand limits that stop runaway games (`MAX_HANDS`, `max_decisions`) must not appear in views or rules text, or models play for the cap.
101
+ - **Scope long rules.** If `rules.md` is long, `rules_text_for` can send only the sections the current step needs.
102
+ - **Personas** (optional): `data/personas.yml` holds a shared `basic_strategy` and play styles under `personas`; each player gets one (`--personas a,b`, or at random) and its strategy goes into the system prompt. `data/bots.yml` holds parameters the bot policy reads with `bot_strategy_for(player)`, so botruns compare styles (see `games/hearts` and `games/gin_rummy`; the generated files explain the format). `botrun --personas all` cycles every arrangement and needs at least as many personas as seats.
103
+
104
+ ## Reports
105
+
106
+ - `report_sections` adds markdown sections to each session report: game-specific tallies the designer cares about.
107
+ - Debrief answers get their own section in each session report. A game that renders them itself in `report_sections` returns false from `report_debrief?`.
108
+ - `botrun_stats` returns numbers for one bot game; a `PlaiTestr::Reporter::BotrunReport` subclass (registered as `botrun_report_class:`) turns them into sections with `extra_sections`. See `games/hearts`.
109
+ - Keep the counters in `data` so they survive checkpoints and resume.
110
+
111
+ ## Bot rules
112
+
113
+ `bot_choice(player, moves, pick: nil)` returns `nil` or `{ 'choice' => n, 'rule' => 'name', 'obvious' => bool }` (`'choices' => [...]` for pick-N). When the model never gives a valid answer, the runner plays the bot's choice instead (or a random legal move when there is none) and flags the turn.
114
+
115
+ - Mark a rule `obvious` only when the move is never worse than any alternative. `--mode hybrid` plays obvious moves without calling the model, and the report scores the model against them.
116
+ - Name rules after the idea (`duck`, `dump_queen`, `shed_deadwood`); the botrun report counts them.
117
+ - Keep the bot simple and readable. It is a baseline and a crash test, not a strong player.
118
+
119
+ ## Rulings and simplifications
120
+
121
+ - When the designer settles a rules question, add it to a **Rulings** section in `data/rules.md` (dated), so models and future readers see it.
122
+ - When the harness deliberately simplifies a rule, list it in a **Simplifications** section in `data/rules.md` (see `games/gin_rummy`).
123
+ - Keep open questions and engine choices the designer has not confirmed in `games/<name>/NOTES.md`, marked as unconfirmed, and move them into **Rulings** once settled.
124
+
125
+ ## Specs
126
+
127
+ - Unit specs drive the runner directly: build it with `game.runner_class.new(game: game, players: game.players, seed: 1, mode: 'bot', persist: false, out: StringIO.new)`, call `setup`, arrange the state (`runner.game_state`, `runner.data`), and check `legal_moves` or the effects of `apply_action_effects`. Hooks are public, so specs call them directly. The examples' `spec/` folders show the pattern.
128
+ - Every game keeps a botrun spec: a handful of seeds, no crashes, a winner every time.
129
+ - Botruns never build prompts, so a bug in a view only shows up in a model game. Every game also keeps a spec that plays a few bot games and calls the prompt hooks (`view_text_for`, `rules_text_for`, `recent_events_for`, `persona_for`, `turn_phase`) on every turn; see *builds every prompt a model player would see* in the examples.
130
+ - If `data/` is copied from another repository, add a spec that fails when the copy is behind.
131
+ - `bundle exec rspec` runs every game; `bundle exec rspec games/<name>` runs one.
132
+
133
+ ## Checking a prompt
134
+
135
+ ```ruby
136
+ # bundle exec ruby -e '...'
137
+ require 'plai_testr'
138
+ game = PlaiTestr.load('hearts')
139
+ runner = game.runner_class.new(game: game, players: game.players, seed: 1, mode: 'bot', persist: false, out: StringIO.new)
140
+ runner.setup
141
+ 5.times { runner.play_turn }
142
+ player = runner.next_player
143
+ puts runner.persona_for(player), runner.view_text_for(player), runner.menu(player).map { |m| m[:label] }
144
+ ```
145
+
146
+ ## Commands
147
+
148
+ ```
149
+ bundle exec plai_testr <game> botrun --games 1000 [--personas all] Bots only; report in games/<game>/reports/
150
+ bundle exec plai_testr <game> play [--seed 1] [--mode hybrid] One model game, after preflight checks (ask first)
151
+ bundle exec plai_testr <game> resume <session_id> (ask first)
152
+ bundle exec plai_testr <game> compare | list | preflight
153
+ bundle exec plai_testr generate game <name>
154
+ bundle exec rspec
155
+
156
+ --quiet shows only progress and the result; --verbose adds the raw model output (play) or every game (botrun).
157
+ Set PLAI_TESTR_DEBUG=1 to see the full backtrace of an error.
158
+ ```
@@ -0,0 +1 @@
1
+ @AGENTS.md
@@ -0,0 +1,7 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'plai_testr', '~> <%= version %>'
4
+
5
+ group :development, :test do
6
+ gem 'rspec', '~> 3'
7
+ end
@@ -0,0 +1,91 @@
1
+ # <%= app_name %>
2
+
3
+ Automated playtesting for tabletop games with [PlAI Testr](https://github.com/armahillo/plai_testr): local LLM players (via [Ollama](https://ollama.com)) and scripted bots play your game, and every session ends in a markdown report.
4
+
5
+ ## Quick start
6
+
7
+ ```sh
8
+ bundle install
9
+
10
+ # Bots only, no model needed: 100 games in a few seconds, then a report
11
+ bundle exec plai_testr hearts botrun --games 100
12
+
13
+ # One game with a local model (needs Ollama running; missing models are pulled)
14
+ bundle exec plai_testr gin_rummy play
15
+ ```
16
+
17
+ Reports go in `games/<game>/reports/`; resumable sessions in `games/<game>/sessions/`.
18
+
19
+ ## Commands
20
+
21
+ ```
22
+ bundle exec plai_testr <game> play [--seed N] [--model M] [--mode llm|hybrid|bot] One game (random seed 1-100 unless given)
23
+ bundle exec plai_testr <game> botrun [--games N] [--personas all] Bots only, many games
24
+ bundle exec plai_testr <game> resume <session_id> Continue an interrupted session
25
+ bundle exec plai_testr <game> preflight Check Ollama, pull and warm up models, lint
26
+ bundle exec plai_testr <game> compare [--since ID] Compare models across sessions
27
+ bundle exec plai_testr <game> list List sessions
28
+ bundle exec plai_testr generate game <name> Add a new game
29
+ ```
30
+
31
+ `--quiet` shows only a progress line and the result; `--verbose` shows more. Ctrl+C once saves and stops after the current move (a botrun finishes its current game and reports); Ctrl+C twice quits immediately.
32
+
33
+ Settings for the whole app (decision limit, model, Ollama host, context size) are in `config/plai_testr.rb`.
34
+
35
+ ## Layout
36
+
37
+ ```
38
+ Gemfile
39
+ AGENTS.md Guide for AI assistants helping build harnesses (CLAUDE.md points to it)
40
+ config/plai_testr.rb App-wide settings
41
+ spec/spec_helper.rb Loads every game under games/
42
+ games/
43
+ hearts/ Example: 4-player trick-taking
44
+ gin_rummy/ Example: 2-player rummy
45
+ ```
46
+
47
+ Each game lives entirely in its own directory:
48
+
49
+ ```
50
+ games/<name>/
51
+ lib/<name>.rb Requires the game and registers it with PlaiTestr
52
+ lib/<name>/bot.rb Bot rules: how a bot picks moves
53
+ lib/<name>/ Runner (rules engine), state classes
54
+ data/rules.json Machine-readable rules: game name, version, player count, card deck
55
+ data/rules.md Rules text the model players read
56
+ data/personas.yml Optional: strategy and play styles for the model players
57
+ data/bots.yml Optional: parameters for the bot rules
58
+ NOTES.md Optional: open rules questions and unconfirmed engine choices
59
+ spec/ The game's specs
60
+ sessions/ reports/ Output (gitignored)
61
+ ```
62
+
63
+ ## Working with an AI assistant
64
+
65
+ `AGENTS.md` tells an AI coding assistant how this project works: where the docs are, how to build a harness, and what to ask you before doing. Claude Code reads it through `CLAUDE.md`; run `claude` in this directory and start talking.
66
+
67
+ Some ways to start:
68
+
69
+ - *Look around this project and tell me what's here and how to run it.*
70
+ - *I want to playtest my game. The rules are in ~/designs/my_game/rules.md. Read them, list every rules question you have, then set up a harness for it.*
71
+ - *Write bot rules for my_game so it can play itself, run a 1000-game botrun, and summarize the report.*
72
+ - *Read the newest report in games/my_game/reports and tell me what it says about balance. Separate what might be the bot's habits from what might be the game.*
73
+ - *The model players in my_game keep making the same bad choice. Show me the prompt they see and suggest how to make the menu clearer, without telling them what to pick.*
74
+ - *I've ruled that a tie goes to the defender. Update the engine, add a spec, and record the ruling in data/rules.md.*
75
+ - *Add personas to my_game: one player who races, one who blocks.*
76
+
77
+ ## Adding a game
78
+
79
+ ```sh
80
+ bundle exec plai_testr generate game my_game
81
+ ```
82
+
83
+ This creates the skeleton above. Fill in `data/rules.json` and `data/rules.md`, then implement the runner hooks (`setup`, `legal_moves`, `apply_action_effects`) and, optionally, bot rules in `bot_choice`. The examples show complete games; the [PlAI Testr README](https://github.com/armahillo/plai_testr#writing-a-game) lists every hook.
84
+
85
+ `bundle exec rspec games/my_game` runs its specs. The generated botrun spec stays pending until the game can play itself to the end.
86
+
87
+ ## Removing a game
88
+
89
+ Delete its directory: `rm -rf games/hearts`. Nothing else refers to it.
90
+
91
+ Keep all of a game's files under `games/<name>/` so that stays true. If a game ever needs files somewhere else (for example `spec/<name>/`), add a rake task that removes them along with the game directory.
@@ -0,0 +1,15 @@
1
+ PlaiTestr.configure do |config|
2
+ # config.max_decisions = 100_000
3
+ # config.player_model = 'qwen3:8b'
4
+ # config.ollama_host = 'localhost'
5
+ # config.ollama_port = 11434
6
+ # config.model_settings = {
7
+ # context_size: 8192,
8
+ # long_context_size: 16_384,
9
+ # menu_budget: 512,
10
+ # think_budget: 8192,
11
+ # debrief_budget: 1024,
12
+ # referee_budget: 2048,
13
+ # temperature: 0.6
14
+ # }
15
+ end
@@ -0,0 +1,25 @@
1
+ require 'fileutils'
2
+ require 'tmpdir'
3
+
4
+ require 'plai_testr'
5
+
6
+ PlaiTestr.load_config(File.expand_path('../config/plai_testr.rb', __dir__))
7
+ PlaiTestr.games_dir = File.expand_path('../games', __dir__)
8
+ PlaiTestr.names.each { |name| PlaiTestr.load(name) }
9
+
10
+ RSpec.configure do |config|
11
+ config.expect_with :rspec do |expectations|
12
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
13
+ end
14
+
15
+ config.mock_with :rspec do |mocks|
16
+ mocks.verify_partial_doubles = true
17
+ end
18
+
19
+ config.shared_context_metadata_behavior = :apply_to_host_groups
20
+ config.filter_run_when_matching :focus
21
+ config.disable_monkey_patching!
22
+ config.warnings = true
23
+ config.order = :random
24
+ Kernel.srand config.seed
25
+ end
@@ -0,0 +1,12 @@
1
+ # Gin Rummy harness notes
2
+
3
+ For the designer and anyone working on this harness. Model players never see this file.
4
+
5
+ ## Hidden safety caps
6
+
7
+ - `MAX_TURNS` (120 turns in one hand): the hand is a draw, logged as an ordinary drawn hand and counted as `turn_cap` in `data['results']`. Small models can swap the same few cards through the discard pile forever; a 2026-10-17 qwen3:8b test went 155 turns in hand 1 before the draw options showed deadwood.
8
+ - `MAX_HANDS` (50 hands): the game ends and the higher score wins (`hand_limit`).
9
+
10
+ ## Open questions
11
+
12
+ - None yet.
@@ -0,0 +1,18 @@
1
+ # bot_strategy: parameters your bot (lib/<game>/bot.rb) reads (never shown to the model).
2
+ # Read them in bot_choice with bot_strategy_for(player).
3
+ # knock_at: 10
4
+ # lead_low: true
5
+ #
6
+ # bots: per-persona overrides, keyed by the persona's key in personas.yml.
7
+ # A key with no matching persona is an error.
8
+ # gin_hunter:
9
+ # <<: *bot_defaults
10
+ # knock_at: 3
11
+
12
+ bot_strategy: &bot_defaults
13
+ knock_at: 10
14
+
15
+ bots:
16
+ gin_hunter:
17
+ <<: *bot_defaults
18
+ knock_at: 3
@@ -0,0 +1,26 @@
1
+ # basic_strategy: shared strategy, injected into each persona with <<: *basics.
2
+ # complete_tricks: Points come only from tricks you win; count them before you lead.
3
+ # avoid_penalties: Discard high cards early, while they're hard to punish.
4
+ #
5
+ # personas: play styles, picked with --personas a,b (or at random). The key is
6
+ # the persona's name; every field under it is shown to the model. To override
7
+ # a basic_strategy item, reuse its key exactly; a new key adds an item.
8
+ # moon_shooter:
9
+ # <<: *basics
10
+ # avoid_penalties: Keep hearts and the Q♠ if you can take every trick.
11
+ # safe_player:
12
+ # <<: *basics
13
+ # goal: Finish every hand with as few points as you can.
14
+
15
+ basic_strategy: &basics
16
+ near_melds: Keep cards that are close to melds (pairs, and two cards of one suit in a row); discard unmatched high cards (face cards count 10) early.
17
+ watch_discards: Taking the top discard shows your opponent what you are collecting. Watch what they take, and avoid discarding cards that fit it.
18
+ read_deadwood: Every option shows your deadwood after the move. Lower deadwood means a sooner knock and fewer points lost if your opponent knocks first.
19
+
20
+ personas:
21
+ quick_knocker:
22
+ <<: *basics
23
+ goal: End hands early. Knock the first time your deadwood is 10 or less, before your opponent can improve.
24
+ gin_hunter:
25
+ <<: *basics
26
+ goal: Win big. Hold out for gin, or knock only with 3 or less deadwood, to earn the gin bonus and avoid undercuts. Knock sooner once the stock runs low.
@@ -0,0 +1,76 @@
1
+ {
2
+ "game": {
3
+ "name": "Gin Rummy",
4
+ "version": "1.0.0",
5
+ "description": "A 2-player rummy game. Draw and discard to form melds (sets of 3-4 of a rank, runs of 3+ in a suit) and knock when your unmatched cards total 10 or less. First to 100 points wins.",
6
+ "player_count": { "min": 2, "max": 2, "default": 2 }
7
+ },
8
+
9
+ "points_to_win": 100,
10
+ "knock_limit": 10,
11
+ "gin_bonus": 25,
12
+ "undercut_bonus": 25,
13
+
14
+ "components": [
15
+ {
16
+ "name": "Deck",
17
+ "type": "card_deck",
18
+ "count": 52,
19
+ "description": "Standard 52-card deck. Aces are low (rank 1); face cards count 10.",
20
+ "values": [
21
+ { "name": "A♣", "rank": 1, "suit": "clubs" },
22
+ { "name": "2♣", "rank": 2, "suit": "clubs" },
23
+ { "name": "3♣", "rank": 3, "suit": "clubs" },
24
+ { "name": "4♣", "rank": 4, "suit": "clubs" },
25
+ { "name": "5♣", "rank": 5, "suit": "clubs" },
26
+ { "name": "6♣", "rank": 6, "suit": "clubs" },
27
+ { "name": "7♣", "rank": 7, "suit": "clubs" },
28
+ { "name": "8♣", "rank": 8, "suit": "clubs" },
29
+ { "name": "9♣", "rank": 9, "suit": "clubs" },
30
+ { "name": "10♣", "rank": 10, "suit": "clubs" },
31
+ { "name": "J♣", "rank": 11, "suit": "clubs" },
32
+ { "name": "Q♣", "rank": 12, "suit": "clubs" },
33
+ { "name": "K♣", "rank": 13, "suit": "clubs" },
34
+ { "name": "A♦", "rank": 1, "suit": "diamonds" },
35
+ { "name": "2♦", "rank": 2, "suit": "diamonds" },
36
+ { "name": "3♦", "rank": 3, "suit": "diamonds" },
37
+ { "name": "4♦", "rank": 4, "suit": "diamonds" },
38
+ { "name": "5♦", "rank": 5, "suit": "diamonds" },
39
+ { "name": "6♦", "rank": 6, "suit": "diamonds" },
40
+ { "name": "7♦", "rank": 7, "suit": "diamonds" },
41
+ { "name": "8♦", "rank": 8, "suit": "diamonds" },
42
+ { "name": "9♦", "rank": 9, "suit": "diamonds" },
43
+ { "name": "10♦", "rank": 10, "suit": "diamonds" },
44
+ { "name": "J♦", "rank": 11, "suit": "diamonds" },
45
+ { "name": "Q♦", "rank": 12, "suit": "diamonds" },
46
+ { "name": "K♦", "rank": 13, "suit": "diamonds" },
47
+ { "name": "A♠", "rank": 1, "suit": "spades" },
48
+ { "name": "2♠", "rank": 2, "suit": "spades" },
49
+ { "name": "3♠", "rank": 3, "suit": "spades" },
50
+ { "name": "4♠", "rank": 4, "suit": "spades" },
51
+ { "name": "5♠", "rank": 5, "suit": "spades" },
52
+ { "name": "6♠", "rank": 6, "suit": "spades" },
53
+ { "name": "7♠", "rank": 7, "suit": "spades" },
54
+ { "name": "8♠", "rank": 8, "suit": "spades" },
55
+ { "name": "9♠", "rank": 9, "suit": "spades" },
56
+ { "name": "10♠", "rank": 10, "suit": "spades" },
57
+ { "name": "J♠", "rank": 11, "suit": "spades" },
58
+ { "name": "Q♠", "rank": 12, "suit": "spades" },
59
+ { "name": "K♠", "rank": 13, "suit": "spades" },
60
+ { "name": "A♥", "rank": 1, "suit": "hearts" },
61
+ { "name": "2♥", "rank": 2, "suit": "hearts" },
62
+ { "name": "3♥", "rank": 3, "suit": "hearts" },
63
+ { "name": "4♥", "rank": 4, "suit": "hearts" },
64
+ { "name": "5♥", "rank": 5, "suit": "hearts" },
65
+ { "name": "6♥", "rank": 6, "suit": "hearts" },
66
+ { "name": "7♥", "rank": 7, "suit": "hearts" },
67
+ { "name": "8♥", "rank": 8, "suit": "hearts" },
68
+ { "name": "9♥", "rank": 9, "suit": "hearts" },
69
+ { "name": "10♥", "rank": 10, "suit": "hearts" },
70
+ { "name": "J♥", "rank": 11, "suit": "hearts" },
71
+ { "name": "Q♥", "rank": 12, "suit": "hearts" },
72
+ { "name": "K♥", "rank": 13, "suit": "hearts" }
73
+ ]
74
+ }
75
+ ]
76
+ }
@@ -0,0 +1,44 @@
1
+ # Gin Rummy
2
+
3
+ v1.0.0
4
+
5
+ A card game for 2 players. Form melds, keep your unmatched cards (deadwood) low, and knock before your opponent does. The first player to 100 points wins.
6
+
7
+ ## Cards
8
+
9
+ A standard 52-card deck. Aces are low: A-2-3 is a run, Q-K-A is not. Face cards count 10, aces 1, and number cards their number.
10
+
11
+ ## Melds
12
+
13
+ - **Set:** 3 or 4 cards of the same rank (7♣ 7♦ 7♥).
14
+ - **Run:** 3 or more cards in a row of the same suit (4♠ 5♠ 6♠).
15
+
16
+ A card can belong to only one meld. Cards that are in no meld are **deadwood**.
17
+
18
+ ## Each hand
19
+
20
+ 1. **Deal.** Each player gets 10 cards. The next card is turned face up to start the discard pile; the rest is the face-down stock. The player who did not deal goes first.
21
+ 2. **Turn.** Draw one card, either the top card of the stock or the top card of the discard pile. Then discard one card face up. You may not discard the card you just took from the discard pile.
22
+ 3. **Knock.** Instead of a plain discard, you may discard and **knock** if your deadwood after discarding totals 10 or less. Knocking with 0 deadwood is **going gin**.
23
+ 4. **Lay off.** After a knock, the knocker shows their melds. The other player shows theirs and may lay off deadwood cards onto the knocker's melds (adding to a set or extending a run). No laying off against gin.
24
+ 5. **Dead hand.** If the stock is down to 2 cards after a discard and nobody knocked, the hand is a draw and is dealt again.
25
+
26
+ ## Scoring
27
+
28
+ - **Knock:** if the knocker has less deadwood, they score the difference.
29
+ - **Undercut:** if the other player's deadwood (after laying off) is equal or less, they score the difference plus 25.
30
+ - **Gin:** the knocker scores the other player's deadwood plus 25.
31
+
32
+ The winner of a hand deals the next one. The first player to reach 100 points wins the game.
33
+
34
+ ## Simplifications in this version
35
+
36
+ - Melds and lay-offs are arranged automatically to give each player the lowest deadwood.
37
+ - There is no special first-turn offer of the upcard; the first player simply draws.
38
+ - No box, line or game bonuses, and no Big Gin.
39
+
40
+ ## Strategy notes
41
+
42
+ - Early on, keep cards that are close to melds (pairs, two cards in a row of one suit).
43
+ - Taking from the discard pile tells your opponent what you are collecting.
44
+ - Discard high deadwood (face cards) first; knock early when your deadwood is low.