plai_testr 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +16 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +176 -0
  5. data/bin/plai_testr +5 -0
  6. data/docs/frame_queue.md +123 -0
  7. data/docs/referee.md +59 -0
  8. data/lib/plai_testr/agents/ollama_client.rb +177 -0
  9. data/lib/plai_testr/agents/player_agent.rb +216 -0
  10. data/lib/plai_testr/agents/referee_agent.rb +56 -0
  11. data/lib/plai_testr/botrun.rb +60 -0
  12. data/lib/plai_testr/cli.rb +190 -0
  13. data/lib/plai_testr/config.rb +39 -0
  14. data/lib/plai_testr/deck.rb +11 -0
  15. data/lib/plai_testr/errors.rb +9 -0
  16. data/lib/plai_testr/event_log.rb +24 -0
  17. data/lib/plai_testr/game.rb +50 -0
  18. data/lib/plai_testr/generator.rb +50 -0
  19. data/lib/plai_testr/model_settings.rb +25 -0
  20. data/lib/plai_testr/names.rb +7 -0
  21. data/lib/plai_testr/personas.rb +97 -0
  22. data/lib/plai_testr/preflight.rb +70 -0
  23. data/lib/plai_testr/reporter/botrun_report.rb +141 -0
  24. data/lib/plai_testr/reporter/model_comparison.rb +104 -0
  25. data/lib/plai_testr/reporter/report_builder.rb +154 -0
  26. data/lib/plai_testr/reporter/think_log.rb +32 -0
  27. data/lib/plai_testr/rules_loader.rb +40 -0
  28. data/lib/plai_testr/session/frame_queue.rb +47 -0
  29. data/lib/plai_testr/session/manager.rb +77 -0
  30. data/lib/plai_testr/session/runner.rb +555 -0
  31. data/lib/plai_testr/state/base.rb +13 -0
  32. data/lib/plai_testr/state/game_state.rb +102 -0
  33. data/lib/plai_testr/state/player_state.rb +61 -0
  34. data/lib/plai_testr/version.rb +3 -0
  35. data/lib/plai_testr.rb +61 -0
  36. data/templates/app/.gitignore +3 -0
  37. data/templates/app/.rspec +3 -0
  38. data/templates/app/AGENTS.md +158 -0
  39. data/templates/app/CLAUDE.md +1 -0
  40. data/templates/app/Gemfile.erb +7 -0
  41. data/templates/app/README.md.erb +91 -0
  42. data/templates/app/config/plai_testr.rb +15 -0
  43. data/templates/app/spec/spec_helper.rb +25 -0
  44. data/templates/examples/gin_rummy/NOTES.md +12 -0
  45. data/templates/examples/gin_rummy/data/bots.yml +18 -0
  46. data/templates/examples/gin_rummy/data/personas.yml +26 -0
  47. data/templates/examples/gin_rummy/data/rules.json +76 -0
  48. data/templates/examples/gin_rummy/data/rules.md +44 -0
  49. data/templates/examples/gin_rummy/lib/gin_rummy/bot.rb +34 -0
  50. data/templates/examples/gin_rummy/lib/gin_rummy/game.rb +8 -0
  51. data/templates/examples/gin_rummy/lib/gin_rummy/melds.rb +51 -0
  52. data/templates/examples/gin_rummy/lib/gin_rummy/session/runner.rb +197 -0
  53. data/templates/examples/gin_rummy/lib/gin_rummy/state/game_state.rb +19 -0
  54. data/templates/examples/gin_rummy/lib/gin_rummy.rb +9 -0
  55. data/templates/examples/gin_rummy/spec/melds_spec.rb +26 -0
  56. data/templates/examples/gin_rummy/spec/session/runner_spec.rb +105 -0
  57. data/templates/examples/hearts/data/bots.yml +22 -0
  58. data/templates/examples/hearts/data/personas.yml +27 -0
  59. data/templates/examples/hearts/data/rules.json +73 -0
  60. data/templates/examples/hearts/data/rules.md +30 -0
  61. data/templates/examples/hearts/lib/hearts/bot.rb +53 -0
  62. data/templates/examples/hearts/lib/hearts/game.rb +16 -0
  63. data/templates/examples/hearts/lib/hearts/reporter/botrun_report.rb +29 -0
  64. data/templates/examples/hearts/lib/hearts/session/runner.rb +255 -0
  65. data/templates/examples/hearts/lib/hearts/state/game_state.rb +19 -0
  66. data/templates/examples/hearts/lib/hearts.rb +11 -0
  67. data/templates/examples/hearts/spec/session/runner_spec.rb +122 -0
  68. data/templates/game/data/bots.yml +13 -0
  69. data/templates/game/data/personas.yml +16 -0
  70. data/templates/game/data/rules.json.erb +16 -0
  71. data/templates/game/data/rules.md.erb +11 -0
  72. data/templates/game/lib/__name__/bot.rb.erb +7 -0
  73. data/templates/game/lib/__name__/session/runner.rb.erb +36 -0
  74. data/templates/game/lib/__name__/state/game_state.rb.erb +11 -0
  75. data/templates/game/lib/__name__.rb.erb +10 -0
  76. data/templates/game/spec/session/runner_spec.rb.erb +29 -0
  77. metadata +153 -0
@@ -0,0 +1,32 @@
1
+ module PlaiTestr
2
+ module Reporter
3
+ class ThinkLog
4
+ HEADER = /^\[(\S+)\] model=(\S+)(.*)$/
5
+
6
+ def self.load(path)
7
+ new(File.exist?(path) ? File.read(path) : '')
8
+ end
9
+
10
+ attr_reader :calls
11
+
12
+ def initialize(text)
13
+ @calls = text.scan(HEADER).map do |time, model, stats|
14
+ stats.scan(/(\w+)=([\d.]+)/).to_h { |key, value| [key, value.to_f] }.merge('time' => Time.parse(time), 'model' => model)
15
+ end
16
+ end
17
+
18
+ def values(key) = @calls.filter_map { |call| call[key] }
19
+
20
+ def average(key)
21
+ list = values(key)
22
+ list.empty? ? nil : list.sum / list.length
23
+ end
24
+
25
+ def minutes
26
+ return nil if @calls.length < 2
27
+
28
+ (@calls.last['time'] - @calls.first['time']) / 60.0
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,40 @@
1
+ module PlaiTestr
2
+ class RulesLoader
3
+ def self.load(path)
4
+ (@cache ||= {})[[File.expand_path(path), File.mtime(path)]] ||= parse(path)
5
+ rescue Errno::ENOENT
6
+ raise RulesError, "Rules file not found: #{path}"
7
+ end
8
+
9
+ def self.parse(path)
10
+ rules = JSON.parse(File.read(path))
11
+ errors = validate(rules)
12
+ raise RulesError, "Invalid rules file #{path}:\n#{errors.map { |e| " - #{e}" }.join("\n")}" unless errors.empty?
13
+
14
+ rules
15
+ rescue Errno::ENOENT
16
+ raise RulesError, "Rules file not found: #{path}"
17
+ rescue JSON::ParserError => e
18
+ raise RulesError, "#{path} is not valid JSON: #{e.message}"
19
+ end
20
+
21
+ def self.validate(rules)
22
+ game = rules['game']
23
+ return ["'game' must be an object with a name and version"] unless game.is_a?(Hash)
24
+
25
+ errors = %w[name version].filter_map { |key| "game.#{key} is required" if game[key].to_s.empty? }
26
+ errors << player_count_error(game['player_count']) if game.key?('player_count')
27
+ rules.fetch('components', []).each do |component|
28
+ errors << "components: #{component['name']} 'values' must be an array" if component['type'] == 'card_deck' && !component.fetch('values', []).is_a?(Array)
29
+ end
30
+ errors.compact
31
+ end
32
+
33
+ def self.player_count_error(range)
34
+ min, default, max = range.is_a?(Hash) ? range.values_at('min', 'default', 'max') : []
35
+ return nil if [min, default, max].all?(Integer) && min <= default && default <= max
36
+
37
+ 'game.player_count needs whole numbers with min <= default <= max'
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,47 @@
1
+ module PlaiTestr
2
+ module Session
3
+ module FrameQueue
4
+ def frame_queue
5
+ raise NotImplementedError, "#{self.class} must implement #frame_queue"
6
+ end
7
+
8
+ def schedule_frames
9
+ raise NotImplementedError, "#{self.class} must implement #schedule_frames"
10
+ end
11
+
12
+ def halt_frames? = game_over?
13
+
14
+ def intercept_frame(_frame) = false
15
+
16
+ private
17
+
18
+ def frame = frame_queue.last
19
+
20
+ def frame_for(type, player = nil, **data)
21
+ { 'type' => type, 'player' => player }.merge(data.transform_keys(&:to_s))
22
+ end
23
+
24
+ def resolve_frame(params)
25
+ current = frame_queue.pop
26
+ send("on_#{current['type']}", current, params)
27
+ advance_frames
28
+ end
29
+
30
+ def advance_frames
31
+ loop do
32
+ return if halt_frames?
33
+
34
+ schedule_frames if frame_queue.empty?
35
+ current = frame
36
+
37
+ if current['auto']
38
+ frame_queue.pop
39
+ send("on_#{current['type']}", current)
40
+ elsif !intercept_frame(current)
41
+ return
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,77 @@
1
+ module PlaiTestr
2
+ module Session
3
+ class Manager
4
+ attr_reader :session_id
5
+
6
+ def initialize(session_id: nil, base_dir: PlaiTestr.sessions_dir, persist: true)
7
+ # A timestamp: readable and sortable. A second session started within the same second
8
+ # gets a counter suffix (-2, -3, …) so ids stay unique.
9
+ @base_dir = base_dir
10
+ @session_id = (session_id || timestamp_id).to_s
11
+ FileUtils.mkdir_p(session_dir) if persist
12
+ end
13
+
14
+ def save_checkpoint(game_state, turn_log, metadata = {})
15
+ data = {
16
+ 'session_id' => @session_id,
17
+ 'saved_at' => Time.now.iso8601,
18
+ 'status' => metadata['status'] || 'in_progress',
19
+ 'game' => metadata['game'],
20
+ 'rules_version' => metadata['rules_version'],
21
+ 'fingerprint' => metadata['fingerprint'],
22
+ 'game_state' => game_state.serialize,
23
+ 'turn_log' => turn_log
24
+ }
25
+
26
+ temp = "#{checkpoint_path}.tmp"
27
+ File.open(temp, 'w') do |file|
28
+ file.write(JSON.pretty_generate(data))
29
+ file.fsync
30
+ end
31
+ File.rename(temp, checkpoint_path)
32
+ end
33
+
34
+ def load_checkpoint
35
+ JSON.parse(File.read(checkpoint_path))
36
+ rescue Errno::ENOENT
37
+ raise SessionError, "No saved session #{@session_id} in #{@base_dir}"
38
+ rescue JSON::ParserError => e
39
+ raise SessionError, "Session #{@session_id} cannot be resumed: #{checkpoint_path} is unreadable (#{e.message.lines.first.strip})"
40
+ end
41
+
42
+ def append_transcript_entry(entry)
43
+ File.open(transcript_path, 'a') { |f| f.puts(JSON.generate(entry)) }
44
+ end
45
+
46
+ def timestamp_id
47
+ stamp = Time.now.strftime('%Y%m%d-%H%M%S')
48
+ id = stamp
49
+ count = 1
50
+ id = "#{stamp}-#{count += 1}" while Dir.exist?(File.join(@base_dir, id))
51
+ id
52
+ end
53
+
54
+ def session_dir
55
+ File.join(@base_dir, @session_id)
56
+ end
57
+
58
+ def checkpoint_path
59
+ File.join(session_dir, 'state.json')
60
+ end
61
+
62
+ def transcript_path
63
+ File.join(session_dir, 'transcript.jsonl')
64
+ end
65
+
66
+ def self.list(base_dir: PlaiTestr.sessions_dir)
67
+ return [] unless Dir.exist?(base_dir)
68
+
69
+ Dir.glob(File.join(base_dir, '*', 'state.json')).filter_map do |path|
70
+ JSON.parse(File.read(path)).slice('session_id', 'game', 'saved_at', 'status')
71
+ rescue JSON::ParserError
72
+ nil
73
+ end.sort_by { |s| s['saved_at'] }.reverse
74
+ end
75
+ end
76
+ end
77
+ end