plai_testr 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +16 -0
- data/LICENSE.txt +21 -0
- data/README.md +176 -0
- data/bin/plai_testr +5 -0
- data/docs/frame_queue.md +123 -0
- data/docs/referee.md +59 -0
- data/lib/plai_testr/agents/ollama_client.rb +177 -0
- data/lib/plai_testr/agents/player_agent.rb +216 -0
- data/lib/plai_testr/agents/referee_agent.rb +56 -0
- data/lib/plai_testr/botrun.rb +60 -0
- data/lib/plai_testr/cli.rb +190 -0
- data/lib/plai_testr/config.rb +39 -0
- data/lib/plai_testr/deck.rb +11 -0
- data/lib/plai_testr/errors.rb +9 -0
- data/lib/plai_testr/event_log.rb +24 -0
- data/lib/plai_testr/game.rb +50 -0
- data/lib/plai_testr/generator.rb +50 -0
- data/lib/plai_testr/model_settings.rb +25 -0
- data/lib/plai_testr/names.rb +7 -0
- data/lib/plai_testr/personas.rb +97 -0
- data/lib/plai_testr/preflight.rb +70 -0
- data/lib/plai_testr/reporter/botrun_report.rb +141 -0
- data/lib/plai_testr/reporter/model_comparison.rb +104 -0
- data/lib/plai_testr/reporter/report_builder.rb +154 -0
- data/lib/plai_testr/reporter/think_log.rb +32 -0
- data/lib/plai_testr/rules_loader.rb +40 -0
- data/lib/plai_testr/session/frame_queue.rb +47 -0
- data/lib/plai_testr/session/manager.rb +77 -0
- data/lib/plai_testr/session/runner.rb +555 -0
- data/lib/plai_testr/state/base.rb +13 -0
- data/lib/plai_testr/state/game_state.rb +102 -0
- data/lib/plai_testr/state/player_state.rb +61 -0
- data/lib/plai_testr/version.rb +3 -0
- data/lib/plai_testr.rb +61 -0
- data/templates/app/.gitignore +3 -0
- data/templates/app/.rspec +3 -0
- data/templates/app/AGENTS.md +158 -0
- data/templates/app/CLAUDE.md +1 -0
- data/templates/app/Gemfile.erb +7 -0
- data/templates/app/README.md.erb +91 -0
- data/templates/app/config/plai_testr.rb +15 -0
- data/templates/app/spec/spec_helper.rb +25 -0
- data/templates/examples/gin_rummy/NOTES.md +12 -0
- data/templates/examples/gin_rummy/data/bots.yml +18 -0
- data/templates/examples/gin_rummy/data/personas.yml +26 -0
- data/templates/examples/gin_rummy/data/rules.json +76 -0
- data/templates/examples/gin_rummy/data/rules.md +44 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/bot.rb +34 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/game.rb +8 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/melds.rb +51 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/session/runner.rb +197 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/state/game_state.rb +19 -0
- data/templates/examples/gin_rummy/lib/gin_rummy.rb +9 -0
- data/templates/examples/gin_rummy/spec/melds_spec.rb +26 -0
- data/templates/examples/gin_rummy/spec/session/runner_spec.rb +105 -0
- data/templates/examples/hearts/data/bots.yml +22 -0
- data/templates/examples/hearts/data/personas.yml +27 -0
- data/templates/examples/hearts/data/rules.json +73 -0
- data/templates/examples/hearts/data/rules.md +30 -0
- data/templates/examples/hearts/lib/hearts/bot.rb +53 -0
- data/templates/examples/hearts/lib/hearts/game.rb +16 -0
- data/templates/examples/hearts/lib/hearts/reporter/botrun_report.rb +29 -0
- data/templates/examples/hearts/lib/hearts/session/runner.rb +255 -0
- data/templates/examples/hearts/lib/hearts/state/game_state.rb +19 -0
- data/templates/examples/hearts/lib/hearts.rb +11 -0
- data/templates/examples/hearts/spec/session/runner_spec.rb +122 -0
- data/templates/game/data/bots.yml +13 -0
- data/templates/game/data/personas.yml +16 -0
- data/templates/game/data/rules.json.erb +16 -0
- data/templates/game/data/rules.md.erb +11 -0
- data/templates/game/lib/__name__/bot.rb.erb +7 -0
- data/templates/game/lib/__name__/session/runner.rb.erb +36 -0
- data/templates/game/lib/__name__/state/game_state.rb.erb +11 -0
- data/templates/game/lib/__name__.rb.erb +10 -0
- data/templates/game/spec/session/runner_spec.rb.erb +29 -0
- metadata +153 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
module PlaiTestr
|
|
2
|
+
module State
|
|
3
|
+
class PlayerState < Base
|
|
4
|
+
attr_reader :name, :score, :hand, :data
|
|
5
|
+
|
|
6
|
+
def initialize(name:, score: 0, hand: [], data: {})
|
|
7
|
+
@name = name
|
|
8
|
+
@score = score.to_i
|
|
9
|
+
@hand = hand.dup
|
|
10
|
+
@data = data.dup
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.from_h(data)
|
|
14
|
+
new(
|
|
15
|
+
name: data['name'],
|
|
16
|
+
score: data.fetch('score', 0),
|
|
17
|
+
hand: data.fetch('hand', []),
|
|
18
|
+
data: data.fetch('data', {})
|
|
19
|
+
)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def apply_changes(changes)
|
|
23
|
+
return if changes.nil? || changes.empty?
|
|
24
|
+
|
|
25
|
+
@score = changes['score'] if changes.key?('score')
|
|
26
|
+
@hand = changes['hand'] if changes.key?('hand')
|
|
27
|
+
@data.merge!(changes['data']) if changes['data']
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Full state — for checkpoints
|
|
31
|
+
def serialize
|
|
32
|
+
{
|
|
33
|
+
'name' => @name,
|
|
34
|
+
'score' => @score,
|
|
35
|
+
'hand' => @hand,
|
|
36
|
+
'data' => @data
|
|
37
|
+
}
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Opponent-visible state — hand contents are hidden, only size shown
|
|
41
|
+
# Subclasses override to expose game-specific public fields (e.g. top stance card)
|
|
42
|
+
def serialize_for(observer)
|
|
43
|
+
return serialize if observer == @name
|
|
44
|
+
|
|
45
|
+
{
|
|
46
|
+
'name' => @name,
|
|
47
|
+
'score' => @score,
|
|
48
|
+
'hand_size' => @hand.length,
|
|
49
|
+
'data' => public_data
|
|
50
|
+
}
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
# Override in subclasses to selectively expose data to opponents
|
|
56
|
+
def public_data
|
|
57
|
+
{}
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
data/lib/plai_testr.rb
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
require_relative 'plai_testr/version'
|
|
2
|
+
require 'digest'
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'net/http'
|
|
6
|
+
require 'optparse'
|
|
7
|
+
require 'socket'
|
|
8
|
+
require 'stringio'
|
|
9
|
+
require 'time'
|
|
10
|
+
require 'uri'
|
|
11
|
+
|
|
12
|
+
module PlaiTestr
|
|
13
|
+
DECIDER_ICONS = { 'LLM' => '🧠', 'BOT' => '🤖', 'RNG' => '🎲' }.freeze
|
|
14
|
+
|
|
15
|
+
class << self
|
|
16
|
+
attr_writer :sessions_dir, :reports_dir
|
|
17
|
+
|
|
18
|
+
def sessions_dir
|
|
19
|
+
@sessions_dir || File.join(Dir.pwd, 'sessions')
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def reports_dir
|
|
23
|
+
@reports_dir || File.join(Dir.pwd, 'reports')
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def games_dir
|
|
27
|
+
@games_dir || File.join(Dir.pwd, 'games')
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def games_dir=(dir)
|
|
31
|
+
@games_dir = dir && File.expand_path(dir)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
require_relative 'plai_testr/errors'
|
|
37
|
+
require_relative 'plai_testr/config'
|
|
38
|
+
require_relative 'plai_testr/model_settings'
|
|
39
|
+
require_relative 'plai_testr/agents/ollama_client'
|
|
40
|
+
require_relative 'plai_testr/agents/player_agent'
|
|
41
|
+
require_relative 'plai_testr/agents/referee_agent'
|
|
42
|
+
require_relative 'plai_testr/rules_loader'
|
|
43
|
+
require_relative 'plai_testr/personas'
|
|
44
|
+
require_relative 'plai_testr/reporter/think_log'
|
|
45
|
+
require_relative 'plai_testr/reporter/report_builder'
|
|
46
|
+
require_relative 'plai_testr/reporter/model_comparison'
|
|
47
|
+
require_relative 'plai_testr/reporter/botrun_report'
|
|
48
|
+
require_relative 'plai_testr/session/manager'
|
|
49
|
+
require_relative 'plai_testr/session/frame_queue'
|
|
50
|
+
require_relative 'plai_testr/session/runner'
|
|
51
|
+
require_relative 'plai_testr/state/base'
|
|
52
|
+
require_relative 'plai_testr/state/player_state'
|
|
53
|
+
require_relative 'plai_testr/state/game_state'
|
|
54
|
+
require_relative 'plai_testr/botrun'
|
|
55
|
+
require_relative 'plai_testr/preflight'
|
|
56
|
+
require_relative 'plai_testr/game'
|
|
57
|
+
require_relative 'plai_testr/deck'
|
|
58
|
+
require_relative 'plai_testr/event_log'
|
|
59
|
+
require_relative 'plai_testr/names'
|
|
60
|
+
require_relative 'plai_testr/generator'
|
|
61
|
+
require_relative 'plai_testr/cli'
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# Agent guide
|
|
2
|
+
|
|
3
|
+
This app playtests tabletop games with [PlAI Testr](https://github.com/armahillo/plai_testr). Local LLM players (via Ollama) and scripted bots play a game; every session produces a markdown report for the designer. If you are an AI assistant helping build or change a test harness here, read this first.
|
|
4
|
+
|
|
5
|
+
## Start here
|
|
6
|
+
|
|
7
|
+
When you open this project:
|
|
8
|
+
|
|
9
|
+
1. Read this file, then `README.md`.
|
|
10
|
+
2. Run `bundle exec rspec`. It is fast, needs no model, and shows the state of every game.
|
|
11
|
+
3. Look at `games/`. Each game's `data/rules.md` is its rulebook, and `NOTES.md` (if present) holds rulings and open questions.
|
|
12
|
+
4. Read the gem's docs when you need them. They ship with the installed gem, so they match the version in use; `bundle info plai_testr --path` prints its directory:
|
|
13
|
+
- `README.md`: every runner hook, the state classes, configuration
|
|
14
|
+
- `docs/frame_queue.md`: decisions inside decisions (attacks and responses, triggered effects), with a complete example
|
|
15
|
+
- `docs/referee.md`: yes/no rulings on free-text conditions by a second model, with an example
|
|
16
|
+
- `templates/examples/`: pristine copies of the example games, in case they were deleted from `games/`
|
|
17
|
+
|
|
18
|
+
The example games show working patterns to copy:
|
|
19
|
+
|
|
20
|
+
- `games/hearts`: 4 players, hidden hands, a pick-3 pass (`pick_range`), a session report section (`report_sections`), a botrun report subclass (`botrun_stats` + `Reporter::BotrunReport`), and personas whose bots play differently (`lib/hearts/bot.rb`, `data/personas.yml`, `data/bots.yml`)
|
|
21
|
+
- `games/gin_rummy`: 2 players, engine-computed facts on every option, melds computed in plain Ruby (`melds.rb`), a bot (`lib/gin_rummy/bot.rb`) with a per-persona parameter (`knock_at` in `data/bots.yml`), and a **Simplifications** section in `data/rules.md`
|
|
22
|
+
|
|
23
|
+
## Who you are working for
|
|
24
|
+
|
|
25
|
+
The user is a game designer. The harness exists to tell them how their game plays, so:
|
|
26
|
+
|
|
27
|
+
- Never change the game's rules to make the harness easier. When the rules are unclear or silent, ask the designer and record the answer (see *Rulings*).
|
|
28
|
+
- Ask before running model games (`play`, `resume`, `preflight`) or pulling models: they take minutes to hours and use the machine. Specs and `botrun` are fast and safe to run.
|
|
29
|
+
- Bot results describe how the bot plays, not how people play. Say so when reporting them.
|
|
30
|
+
|
|
31
|
+
## Layout
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
games/<name>/
|
|
35
|
+
lib/<name>.rb Requires the game and calls PlaiTestr.register
|
|
36
|
+
lib/<name>/game.rb Constants
|
|
37
|
+
lib/<name>/state/ GameState (and PlayerState) subclasses: what is stored, what each player may see
|
|
38
|
+
lib/<name>/session/runner.rb The rules engine: steps, legal moves, effects, views, prompts
|
|
39
|
+
lib/<name>/bot.rb Bot rules (bot_choice), included in the runner
|
|
40
|
+
lib/<name>/reporter/ Optional report classes
|
|
41
|
+
data/rules.json Machine-readable rules: game name, version, player_count, card_deck, numbers the engine uses
|
|
42
|
+
data/rules.md Rules text the model players read
|
|
43
|
+
data/personas.yml Optional: shared strategy and play styles the model players follow
|
|
44
|
+
data/bots.yml Optional: parameters the bot reads, per persona
|
|
45
|
+
NOTES.md Optional: open questions and unconfirmed engine choices
|
|
46
|
+
spec/ The game's specs
|
|
47
|
+
sessions/ reports/ Output (gitignored)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Every file a game needs lives under `games/<name>/`, so a game is removed by deleting its directory. Keep it that way. Shared code goes in the plai_testr gem, not in another game.
|
|
51
|
+
|
|
52
|
+
App-wide settings (the decision limit backstop, model, Ollama host, model settings) live in `config/plai_testr.rb`. A game can set its own `max_decisions:` and `model_settings:` in `PlaiTestr.register`.
|
|
53
|
+
|
|
54
|
+
`bundle exec plai_testr generate game <name>` creates the skeleton. `games/hearts` (4 players, hidden hands, a pick-3 pass) and `games/gin_rummy` (2 players, draw/discard turns, engine-computed melds) are complete examples.
|
|
55
|
+
|
|
56
|
+
## How a game runs
|
|
57
|
+
|
|
58
|
+
Each turn the runner asks `next_player` who acts, builds a numbered menu from `legal_moves(player)`, and the player (model or bot) picks a number. `apply_action_effects` applies the chosen move's `action` to `@game_state`. The game ends when `status['game_over']` is true; set `status['winner']` and `status['end_reason']` too (`@game_state.apply_changes('status' => {...})`). A game that never ends is stopped at the decision limit with `end_reason: 'decision_limit_reached'`.
|
|
59
|
+
|
|
60
|
+
Build each menu entry with `option(label, public: ..., log: ..., name => params)`:
|
|
61
|
+
|
|
62
|
+
```ruby
|
|
63
|
+
option('Play 3♣', public: 'played 3♣', play: { value: card['uid'] })
|
|
64
|
+
# => { label: 'Play 3♣', public: 'played 3♣', action: { play: { value: 42 } } }
|
|
65
|
+
|
|
66
|
+
def apply_action_effects(player, action)
|
|
67
|
+
case action
|
|
68
|
+
in { play: { value: uid } } then play(player, uid)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`label` is what the acting player sees, `public` what other players see, `log` the short console text. The action has exactly one name; `value:` is the main parameter, and others sit beside it. Moves and actions use symbol keys (the runner converts hand-written string keys), so `apply_action_effects` and the bot rules can pattern-match them. The runner checks every menu before using it and raises `PlaiTestr::MoveError` for a misspelled key or a malformed action.
|
|
74
|
+
|
|
75
|
+
A menu with one move is applied without asking anyone. `pick_range(player)` returning `[min, max]` turns the menu into a pick-N choice, applied as one action whose `value` lists the chosen moves' values: `{ pass: { value: [42, 17, 9] } }`.
|
|
76
|
+
|
|
77
|
+
Randomness comes from `@rng`. It is reset for each decision from the seed and the decision number, so the same seed and the same choices always give the same game, interrupted or not.
|
|
78
|
+
|
|
79
|
+
The gem's `README.md` (see *Start here*) lists every hook.
|
|
80
|
+
|
|
81
|
+
## Building a harness: the order that works
|
|
82
|
+
|
|
83
|
+
1. **Rules first.** Put the rules text in `data/rules.md` and the numbers in `data/rules.json`. Read them closely; list every gap or ambiguity and ask the designer before coding around it.
|
|
84
|
+
2. **State.** Decide what lives in `data` (`@game_state.data`: decks, piles, step, active player). `status` belongs to the runner. Use stable ids (card `uid`s, fixed pile indexes); never re-index things players refer to. A GameState subclass can name its keys with readers (`def step = @data['step']`).
|
|
85
|
+
3. **Steps.** Model a turn as a small state machine in `data['step']`. `legal_moves` and `apply_action_effects` switch on it.
|
|
86
|
+
4. **Specs for the rules.** Before any model plays, write specs that set up a situation and check the menu (`legal_moves`) and the effects. This is where rules bugs are cheapest.
|
|
87
|
+
5. **Bot rules.** Implement `bot_choice` so the game can play itself. Then `bundle exec plai_testr <game> botrun --games 1000` must finish with no crashes and sensible endings. Add a botrun spec over a few seeds.
|
|
88
|
+
6. **Views and prompts.** Write `view_text_for`, `rules_text_for`, `turn_phase`, `debrief_questions`. Read a few prompts yourself (see *Checking a prompt*).
|
|
89
|
+
7. **Model games.** With the designer's go-ahead: one game with `--mode hybrid` (the bot plays obvious moves), then `--mode llm`.
|
|
90
|
+
|
|
91
|
+
## Designing what the model sees
|
|
92
|
+
|
|
93
|
+
Small local models (4B–14B) play well only when the engine does the bookkeeping.
|
|
94
|
+
|
|
95
|
+
- **The engine lists the legal moves; the model picks a number.** Never ask the model to name a card, type a move, or check legality.
|
|
96
|
+
- **Put engine-computed facts on each option, never advice.** "Discard 9♥ (deadwood after: 31)" is a fact; "Discard 9♥ (best)" decides for the model and ruins the test.
|
|
97
|
+
- **Generate views from state; never hand-write summaries.** Hand-written summaries leak hidden information or go stale.
|
|
98
|
+
- **Show each player only what they may know.** Override `data_for(observer)` in GameState (and `serialize_for` in PlayerState) to hide decks and other hands. Log events with `log(text)`, and `log(text, to: player)` for private ones ("You drew the 2♠"); `recent_events_for` shows each player what they may see.
|
|
99
|
+
- **Keep menus short.** Merge equivalent options (identical cards, interchangeable targets). Split one huge decision into steps.
|
|
100
|
+
- **Hide safety caps from players.** Turn or hand limits that stop runaway games (`MAX_HANDS`, `max_decisions`) must not appear in views or rules text, or models play for the cap.
|
|
101
|
+
- **Scope long rules.** If `rules.md` is long, `rules_text_for` can send only the sections the current step needs.
|
|
102
|
+
- **Personas** (optional): `data/personas.yml` holds a shared `basic_strategy` and play styles under `personas`; each player gets one (`--personas a,b`, or at random) and its strategy goes into the system prompt. `data/bots.yml` holds parameters the bot policy reads with `bot_strategy_for(player)`, so botruns compare styles (see `games/hearts` and `games/gin_rummy`; the generated files explain the format). `botrun --personas all` cycles every arrangement and needs at least as many personas as seats.
|
|
103
|
+
|
|
104
|
+
## Reports
|
|
105
|
+
|
|
106
|
+
- `report_sections` adds markdown sections to each session report: game-specific tallies the designer cares about.
|
|
107
|
+
- Debrief answers get their own section in each session report. A game that renders them itself in `report_sections` returns false from `report_debrief?`.
|
|
108
|
+
- `botrun_stats` returns numbers for one bot game; a `PlaiTestr::Reporter::BotrunReport` subclass (registered as `botrun_report_class:`) turns them into sections with `extra_sections`. See `games/hearts`.
|
|
109
|
+
- Keep the counters in `data` so they survive checkpoints and resume.
|
|
110
|
+
|
|
111
|
+
## Bot rules
|
|
112
|
+
|
|
113
|
+
`bot_choice(player, moves, pick: nil)` returns `nil` or `{ 'choice' => n, 'rule' => 'name', 'obvious' => bool }` (`'choices' => [...]` for pick-N). When the model never gives a valid answer, the runner plays the bot's choice instead (or a random legal move when there is none) and flags the turn.
|
|
114
|
+
|
|
115
|
+
- Mark a rule `obvious` only when the move is never worse than any alternative. `--mode hybrid` plays obvious moves without calling the model, and the report scores the model against them.
|
|
116
|
+
- Name rules after the idea (`duck`, `dump_queen`, `shed_deadwood`); the botrun report counts them.
|
|
117
|
+
- Keep the bot simple and readable. It is a baseline and a crash test, not a strong player.
|
|
118
|
+
|
|
119
|
+
## Rulings and simplifications
|
|
120
|
+
|
|
121
|
+
- When the designer settles a rules question, add it to a **Rulings** section in `data/rules.md` (dated), so models and future readers see it.
|
|
122
|
+
- When the harness deliberately simplifies a rule, list it in a **Simplifications** section in `data/rules.md` (see `games/gin_rummy`).
|
|
123
|
+
- Keep open questions and engine choices the designer has not confirmed in `games/<name>/NOTES.md`, marked as unconfirmed, and move them into **Rulings** once settled.
|
|
124
|
+
|
|
125
|
+
## Specs
|
|
126
|
+
|
|
127
|
+
- Unit specs drive the runner directly: build it with `game.runner_class.new(game: game, players: game.players, seed: 1, mode: 'bot', persist: false, out: StringIO.new)`, call `setup`, arrange the state (`runner.game_state`, `runner.data`), and check `legal_moves` or the effects of `apply_action_effects`. Hooks are public, so specs call them directly. The examples' `spec/` folders show the pattern.
|
|
128
|
+
- Every game keeps a botrun spec: a handful of seeds, no crashes, a winner every time.
|
|
129
|
+
- Botruns never build prompts, so a bug in a view only shows up in a model game. Every game also keeps a spec that plays a few bot games and calls the prompt hooks (`view_text_for`, `rules_text_for`, `recent_events_for`, `persona_for`, `turn_phase`) on every turn; see *builds every prompt a model player would see* in the examples.
|
|
130
|
+
- If `data/` is copied from another repository, add a spec that fails when the copy is behind.
|
|
131
|
+
- `bundle exec rspec` runs every game; `bundle exec rspec games/<name>` runs one.
|
|
132
|
+
|
|
133
|
+
## Checking a prompt
|
|
134
|
+
|
|
135
|
+
```ruby
|
|
136
|
+
# bundle exec ruby -e '...'
|
|
137
|
+
require 'plai_testr'
|
|
138
|
+
game = PlaiTestr.load('hearts')
|
|
139
|
+
runner = game.runner_class.new(game: game, players: game.players, seed: 1, mode: 'bot', persist: false, out: StringIO.new)
|
|
140
|
+
runner.setup
|
|
141
|
+
5.times { runner.play_turn }
|
|
142
|
+
player = runner.next_player
|
|
143
|
+
puts runner.persona_for(player), runner.view_text_for(player), runner.menu(player).map { |m| m[:label] }
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## Commands
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
bundle exec plai_testr <game> botrun --games 1000 [--personas all] Bots only; report in games/<game>/reports/
|
|
150
|
+
bundle exec plai_testr <game> play [--seed 1] [--mode hybrid] One model game, after preflight checks (ask first)
|
|
151
|
+
bundle exec plai_testr <game> resume <session_id> (ask first)
|
|
152
|
+
bundle exec plai_testr <game> compare | list | preflight
|
|
153
|
+
bundle exec plai_testr generate game <name>
|
|
154
|
+
bundle exec rspec
|
|
155
|
+
|
|
156
|
+
--quiet shows only progress and the result; --verbose adds the raw model output (play) or every game (botrun).
|
|
157
|
+
Set PLAI_TESTR_DEBUG=1 to see the full backtrace of an error.
|
|
158
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
@AGENTS.md
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# <%= app_name %>
|
|
2
|
+
|
|
3
|
+
Automated playtesting for tabletop games with [PlAI Testr](https://github.com/armahillo/plai_testr): local LLM players (via [Ollama](https://ollama.com)) and scripted bots play your game, and every session ends in a markdown report.
|
|
4
|
+
|
|
5
|
+
## Quick start
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
bundle install
|
|
9
|
+
|
|
10
|
+
# Bots only, no model needed: 100 games in a few seconds, then a report
|
|
11
|
+
bundle exec plai_testr hearts botrun --games 100
|
|
12
|
+
|
|
13
|
+
# One game with a local model (needs Ollama running; missing models are pulled)
|
|
14
|
+
bundle exec plai_testr gin_rummy play
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Reports go in `games/<game>/reports/`; resumable sessions in `games/<game>/sessions/`.
|
|
18
|
+
|
|
19
|
+
## Commands
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
bundle exec plai_testr <game> play [--seed N] [--model M] [--mode llm|hybrid|bot] One game (random seed 1-100 unless given)
|
|
23
|
+
bundle exec plai_testr <game> botrun [--games N] [--personas all] Bots only, many games
|
|
24
|
+
bundle exec plai_testr <game> resume <session_id> Continue an interrupted session
|
|
25
|
+
bundle exec plai_testr <game> preflight Check Ollama, pull and warm up models, lint
|
|
26
|
+
bundle exec plai_testr <game> compare [--since ID] Compare models across sessions
|
|
27
|
+
bundle exec plai_testr <game> list List sessions
|
|
28
|
+
bundle exec plai_testr generate game <name> Add a new game
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
`--quiet` shows only a progress line and the result; `--verbose` shows more. Ctrl+C once saves and stops after the current move (a botrun finishes its current game and reports); Ctrl+C twice quits immediately.
|
|
32
|
+
|
|
33
|
+
Settings for the whole app (decision limit, model, Ollama host, context size) are in `config/plai_testr.rb`.
|
|
34
|
+
|
|
35
|
+
## Layout
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
Gemfile
|
|
39
|
+
AGENTS.md Guide for AI assistants helping build harnesses (CLAUDE.md points to it)
|
|
40
|
+
config/plai_testr.rb App-wide settings
|
|
41
|
+
spec/spec_helper.rb Loads every game under games/
|
|
42
|
+
games/
|
|
43
|
+
hearts/ Example: 4-player trick-taking
|
|
44
|
+
gin_rummy/ Example: 2-player rummy
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Each game lives entirely in its own directory:
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
games/<name>/
|
|
51
|
+
lib/<name>.rb Requires the game and registers it with PlaiTestr
|
|
52
|
+
lib/<name>/bot.rb Bot rules: how a bot picks moves
|
|
53
|
+
lib/<name>/ Runner (rules engine), state classes
|
|
54
|
+
data/rules.json Machine-readable rules: game name, version, player count, card deck
|
|
55
|
+
data/rules.md Rules text the model players read
|
|
56
|
+
data/personas.yml Optional: strategy and play styles for the model players
|
|
57
|
+
data/bots.yml Optional: parameters for the bot rules
|
|
58
|
+
NOTES.md Optional: open rules questions and unconfirmed engine choices
|
|
59
|
+
spec/ The game's specs
|
|
60
|
+
sessions/ reports/ Output (gitignored)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Working with an AI assistant
|
|
64
|
+
|
|
65
|
+
`AGENTS.md` tells an AI coding assistant how this project works: where the docs are, how to build a harness, and what to ask you before doing. Claude Code reads it through `CLAUDE.md`; run `claude` in this directory and start talking.
|
|
66
|
+
|
|
67
|
+
Some ways to start:
|
|
68
|
+
|
|
69
|
+
- *Look around this project and tell me what's here and how to run it.*
|
|
70
|
+
- *I want to playtest my game. The rules are in ~/designs/my_game/rules.md. Read them, list every rules question you have, then set up a harness for it.*
|
|
71
|
+
- *Write bot rules for my_game so it can play itself, run a 1000-game botrun, and summarize the report.*
|
|
72
|
+
- *Read the newest report in games/my_game/reports and tell me what it says about balance. Separate what might be the bot's habits from what might be the game.*
|
|
73
|
+
- *The model players in my_game keep making the same bad choice. Show me the prompt they see and suggest how to make the menu clearer, without telling them what to pick.*
|
|
74
|
+
- *I've ruled that a tie goes to the defender. Update the engine, add a spec, and record the ruling in data/rules.md.*
|
|
75
|
+
- *Add personas to my_game: one player who races, one who blocks.*
|
|
76
|
+
|
|
77
|
+
## Adding a game
|
|
78
|
+
|
|
79
|
+
```sh
|
|
80
|
+
bundle exec plai_testr generate game my_game
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
This creates the skeleton above. Fill in `data/rules.json` and `data/rules.md`, then implement the runner hooks (`setup`, `legal_moves`, `apply_action_effects`) and, optionally, bot rules in `bot_choice`. The examples show complete games; the [PlAI Testr README](https://github.com/armahillo/plai_testr#writing-a-game) lists every hook.
|
|
84
|
+
|
|
85
|
+
`bundle exec rspec games/my_game` runs its specs. The generated botrun spec stays pending until the game can play itself to the end.
|
|
86
|
+
|
|
87
|
+
## Removing a game
|
|
88
|
+
|
|
89
|
+
Delete its directory: `rm -rf games/hearts`. Nothing else refers to it.
|
|
90
|
+
|
|
91
|
+
Keep all of a game's files under `games/<name>/` so that stays true. If a game ever needs files somewhere else (for example `spec/<name>/`), add a rake task that removes them along with the game directory.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
PlaiTestr.configure do |config|
|
|
2
|
+
# config.max_decisions = 100_000
|
|
3
|
+
# config.player_model = 'qwen3:8b'
|
|
4
|
+
# config.ollama_host = 'localhost'
|
|
5
|
+
# config.ollama_port = 11434
|
|
6
|
+
# config.model_settings = {
|
|
7
|
+
# context_size: 8192,
|
|
8
|
+
# long_context_size: 16_384,
|
|
9
|
+
# menu_budget: 512,
|
|
10
|
+
# think_budget: 8192,
|
|
11
|
+
# debrief_budget: 1024,
|
|
12
|
+
# referee_budget: 2048,
|
|
13
|
+
# temperature: 0.6
|
|
14
|
+
# }
|
|
15
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
require 'fileutils'
|
|
2
|
+
require 'tmpdir'
|
|
3
|
+
|
|
4
|
+
require 'plai_testr'
|
|
5
|
+
|
|
6
|
+
PlaiTestr.load_config(File.expand_path('../config/plai_testr.rb', __dir__))
|
|
7
|
+
PlaiTestr.games_dir = File.expand_path('../games', __dir__)
|
|
8
|
+
PlaiTestr.names.each { |name| PlaiTestr.load(name) }
|
|
9
|
+
|
|
10
|
+
RSpec.configure do |config|
|
|
11
|
+
config.expect_with :rspec do |expectations|
|
|
12
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
config.mock_with :rspec do |mocks|
|
|
16
|
+
mocks.verify_partial_doubles = true
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
config.shared_context_metadata_behavior = :apply_to_host_groups
|
|
20
|
+
config.filter_run_when_matching :focus
|
|
21
|
+
config.disable_monkey_patching!
|
|
22
|
+
config.warnings = true
|
|
23
|
+
config.order = :random
|
|
24
|
+
Kernel.srand config.seed
|
|
25
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Gin Rummy harness notes
|
|
2
|
+
|
|
3
|
+
For the designer and anyone working on this harness. Model players never see this file.
|
|
4
|
+
|
|
5
|
+
## Hidden safety caps
|
|
6
|
+
|
|
7
|
+
- `MAX_TURNS` (120 turns in one hand): the hand is a draw, logged as an ordinary drawn hand and counted as `turn_cap` in `data['results']`. Small models can swap the same few cards through the discard pile forever; a 2026-10-17 qwen3:8b test went 155 turns in hand 1 before the draw options showed deadwood.
|
|
8
|
+
- `MAX_HANDS` (50 hands): the game ends and the higher score wins (`hand_limit`).
|
|
9
|
+
|
|
10
|
+
## Open questions
|
|
11
|
+
|
|
12
|
+
- None yet.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# bot_strategy: parameters your bot (lib/<game>/bot.rb) reads (never shown to the model).
|
|
2
|
+
# Read them in bot_choice with bot_strategy_for(player).
|
|
3
|
+
# knock_at: 10
|
|
4
|
+
# lead_low: true
|
|
5
|
+
#
|
|
6
|
+
# bots: per-persona overrides, keyed by the persona's key in personas.yml.
|
|
7
|
+
# A key with no matching persona is an error.
|
|
8
|
+
# gin_hunter:
|
|
9
|
+
# <<: *bot_defaults
|
|
10
|
+
# knock_at: 3
|
|
11
|
+
|
|
12
|
+
bot_strategy: &bot_defaults
|
|
13
|
+
knock_at: 10
|
|
14
|
+
|
|
15
|
+
bots:
|
|
16
|
+
gin_hunter:
|
|
17
|
+
<<: *bot_defaults
|
|
18
|
+
knock_at: 3
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# basic_strategy: shared strategy, injected into each persona with <<: *basics.
|
|
2
|
+
# complete_tricks: Points come only from tricks you win; count them before you lead.
|
|
3
|
+
# avoid_penalties: Discard high cards early, while they're hard to punish.
|
|
4
|
+
#
|
|
5
|
+
# personas: play styles, picked with --personas a,b (or at random). The key is
|
|
6
|
+
# the persona's name; every field under it is shown to the model. To override
|
|
7
|
+
# a basic_strategy item, reuse its key exactly; a new key adds an item.
|
|
8
|
+
# moon_shooter:
|
|
9
|
+
# <<: *basics
|
|
10
|
+
# avoid_penalties: Keep hearts and the Q♠ if you can take every trick.
|
|
11
|
+
# safe_player:
|
|
12
|
+
# <<: *basics
|
|
13
|
+
# goal: Finish every hand with as few points as you can.
|
|
14
|
+
|
|
15
|
+
basic_strategy: &basics
|
|
16
|
+
near_melds: Keep cards that are close to melds (pairs, and two cards of one suit in a row); discard unmatched high cards (face cards count 10) early.
|
|
17
|
+
watch_discards: Taking the top discard shows your opponent what you are collecting. Watch what they take, and avoid discarding cards that fit it.
|
|
18
|
+
read_deadwood: Every option shows your deadwood after the move. Lower deadwood means a sooner knock and fewer points lost if your opponent knocks first.
|
|
19
|
+
|
|
20
|
+
personas:
|
|
21
|
+
quick_knocker:
|
|
22
|
+
<<: *basics
|
|
23
|
+
goal: End hands early. Knock the first time your deadwood is 10 or less, before your opponent can improve.
|
|
24
|
+
gin_hunter:
|
|
25
|
+
<<: *basics
|
|
26
|
+
goal: Win big. Hold out for gin, or knock only with 3 or less deadwood, to earn the gin bonus and avoid undercuts. Knock sooner once the stock runs low.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
{
|
|
2
|
+
"game": {
|
|
3
|
+
"name": "Gin Rummy",
|
|
4
|
+
"version": "1.0.0",
|
|
5
|
+
"description": "A 2-player rummy game. Draw and discard to form melds (sets of 3-4 of a rank, runs of 3+ in a suit) and knock when your unmatched cards total 10 or less. First to 100 points wins.",
|
|
6
|
+
"player_count": { "min": 2, "max": 2, "default": 2 }
|
|
7
|
+
},
|
|
8
|
+
|
|
9
|
+
"points_to_win": 100,
|
|
10
|
+
"knock_limit": 10,
|
|
11
|
+
"gin_bonus": 25,
|
|
12
|
+
"undercut_bonus": 25,
|
|
13
|
+
|
|
14
|
+
"components": [
|
|
15
|
+
{
|
|
16
|
+
"name": "Deck",
|
|
17
|
+
"type": "card_deck",
|
|
18
|
+
"count": 52,
|
|
19
|
+
"description": "Standard 52-card deck. Aces are low (rank 1); face cards count 10.",
|
|
20
|
+
"values": [
|
|
21
|
+
{ "name": "A♣", "rank": 1, "suit": "clubs" },
|
|
22
|
+
{ "name": "2♣", "rank": 2, "suit": "clubs" },
|
|
23
|
+
{ "name": "3♣", "rank": 3, "suit": "clubs" },
|
|
24
|
+
{ "name": "4♣", "rank": 4, "suit": "clubs" },
|
|
25
|
+
{ "name": "5♣", "rank": 5, "suit": "clubs" },
|
|
26
|
+
{ "name": "6♣", "rank": 6, "suit": "clubs" },
|
|
27
|
+
{ "name": "7♣", "rank": 7, "suit": "clubs" },
|
|
28
|
+
{ "name": "8♣", "rank": 8, "suit": "clubs" },
|
|
29
|
+
{ "name": "9♣", "rank": 9, "suit": "clubs" },
|
|
30
|
+
{ "name": "10♣", "rank": 10, "suit": "clubs" },
|
|
31
|
+
{ "name": "J♣", "rank": 11, "suit": "clubs" },
|
|
32
|
+
{ "name": "Q♣", "rank": 12, "suit": "clubs" },
|
|
33
|
+
{ "name": "K♣", "rank": 13, "suit": "clubs" },
|
|
34
|
+
{ "name": "A♦", "rank": 1, "suit": "diamonds" },
|
|
35
|
+
{ "name": "2♦", "rank": 2, "suit": "diamonds" },
|
|
36
|
+
{ "name": "3♦", "rank": 3, "suit": "diamonds" },
|
|
37
|
+
{ "name": "4♦", "rank": 4, "suit": "diamonds" },
|
|
38
|
+
{ "name": "5♦", "rank": 5, "suit": "diamonds" },
|
|
39
|
+
{ "name": "6♦", "rank": 6, "suit": "diamonds" },
|
|
40
|
+
{ "name": "7♦", "rank": 7, "suit": "diamonds" },
|
|
41
|
+
{ "name": "8♦", "rank": 8, "suit": "diamonds" },
|
|
42
|
+
{ "name": "9♦", "rank": 9, "suit": "diamonds" },
|
|
43
|
+
{ "name": "10♦", "rank": 10, "suit": "diamonds" },
|
|
44
|
+
{ "name": "J♦", "rank": 11, "suit": "diamonds" },
|
|
45
|
+
{ "name": "Q♦", "rank": 12, "suit": "diamonds" },
|
|
46
|
+
{ "name": "K♦", "rank": 13, "suit": "diamonds" },
|
|
47
|
+
{ "name": "A♠", "rank": 1, "suit": "spades" },
|
|
48
|
+
{ "name": "2♠", "rank": 2, "suit": "spades" },
|
|
49
|
+
{ "name": "3♠", "rank": 3, "suit": "spades" },
|
|
50
|
+
{ "name": "4♠", "rank": 4, "suit": "spades" },
|
|
51
|
+
{ "name": "5♠", "rank": 5, "suit": "spades" },
|
|
52
|
+
{ "name": "6♠", "rank": 6, "suit": "spades" },
|
|
53
|
+
{ "name": "7♠", "rank": 7, "suit": "spades" },
|
|
54
|
+
{ "name": "8♠", "rank": 8, "suit": "spades" },
|
|
55
|
+
{ "name": "9♠", "rank": 9, "suit": "spades" },
|
|
56
|
+
{ "name": "10♠", "rank": 10, "suit": "spades" },
|
|
57
|
+
{ "name": "J♠", "rank": 11, "suit": "spades" },
|
|
58
|
+
{ "name": "Q♠", "rank": 12, "suit": "spades" },
|
|
59
|
+
{ "name": "K♠", "rank": 13, "suit": "spades" },
|
|
60
|
+
{ "name": "A♥", "rank": 1, "suit": "hearts" },
|
|
61
|
+
{ "name": "2♥", "rank": 2, "suit": "hearts" },
|
|
62
|
+
{ "name": "3♥", "rank": 3, "suit": "hearts" },
|
|
63
|
+
{ "name": "4♥", "rank": 4, "suit": "hearts" },
|
|
64
|
+
{ "name": "5♥", "rank": 5, "suit": "hearts" },
|
|
65
|
+
{ "name": "6♥", "rank": 6, "suit": "hearts" },
|
|
66
|
+
{ "name": "7♥", "rank": 7, "suit": "hearts" },
|
|
67
|
+
{ "name": "8♥", "rank": 8, "suit": "hearts" },
|
|
68
|
+
{ "name": "9♥", "rank": 9, "suit": "hearts" },
|
|
69
|
+
{ "name": "10♥", "rank": 10, "suit": "hearts" },
|
|
70
|
+
{ "name": "J♥", "rank": 11, "suit": "hearts" },
|
|
71
|
+
{ "name": "Q♥", "rank": 12, "suit": "hearts" },
|
|
72
|
+
{ "name": "K♥", "rank": 13, "suit": "hearts" }
|
|
73
|
+
]
|
|
74
|
+
}
|
|
75
|
+
]
|
|
76
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Gin Rummy
|
|
2
|
+
|
|
3
|
+
v1.0.0
|
|
4
|
+
|
|
5
|
+
A card game for 2 players. Form melds, keep your unmatched cards (deadwood) low, and knock before your opponent does. The first player to 100 points wins.
|
|
6
|
+
|
|
7
|
+
## Cards
|
|
8
|
+
|
|
9
|
+
A standard 52-card deck. Aces are low: A-2-3 is a run, Q-K-A is not. Face cards count 10, aces 1, and number cards their number.
|
|
10
|
+
|
|
11
|
+
## Melds
|
|
12
|
+
|
|
13
|
+
- **Set:** 3 or 4 cards of the same rank (7♣ 7♦ 7♥).
|
|
14
|
+
- **Run:** 3 or more cards in a row of the same suit (4♠ 5♠ 6♠).
|
|
15
|
+
|
|
16
|
+
A card can belong to only one meld. Cards that are in no meld are **deadwood**.
|
|
17
|
+
|
|
18
|
+
## Each hand
|
|
19
|
+
|
|
20
|
+
1. **Deal.** Each player gets 10 cards. The next card is turned face up to start the discard pile; the rest is the face-down stock. The player who did not deal goes first.
|
|
21
|
+
2. **Turn.** Draw one card, either the top card of the stock or the top card of the discard pile. Then discard one card face up. You may not discard the card you just took from the discard pile.
|
|
22
|
+
3. **Knock.** Instead of a plain discard, you may discard and **knock** if your deadwood after discarding totals 10 or less. Knocking with 0 deadwood is **going gin**.
|
|
23
|
+
4. **Lay off.** After a knock, the knocker shows their melds. The other player shows theirs and may lay off deadwood cards onto the knocker's melds (adding to a set or extending a run). No laying off against gin.
|
|
24
|
+
5. **Dead hand.** If the stock is down to 2 cards after a discard and nobody knocked, the hand is a draw and is dealt again.
|
|
25
|
+
|
|
26
|
+
## Scoring
|
|
27
|
+
|
|
28
|
+
- **Knock:** if the knocker has less deadwood, they score the difference.
|
|
29
|
+
- **Undercut:** if the other player's deadwood (after laying off) is equal or less, they score the difference plus 25.
|
|
30
|
+
- **Gin:** the knocker scores the other player's deadwood plus 25.
|
|
31
|
+
|
|
32
|
+
The winner of a hand deals the next one. The first player to reach 100 points wins the game.
|
|
33
|
+
|
|
34
|
+
## Simplifications in this version
|
|
35
|
+
|
|
36
|
+
- Melds and lay-offs are arranged automatically to give each player the lowest deadwood.
|
|
37
|
+
- There is no special first-turn offer of the upcard; the first player simply draws.
|
|
38
|
+
- No box, line or game bonuses, and no Big Gin.
|
|
39
|
+
|
|
40
|
+
## Strategy notes
|
|
41
|
+
|
|
42
|
+
- Early on, keep cards that are close to melds (pairs, two cards in a row of one suit).
|
|
43
|
+
- Taking from the discard pile tells your opponent what you are collecting.
|
|
44
|
+
- Discard high deadwood (face cards) first; knock early when your deadwood is low.
|