plai_testr 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +16 -0
- data/LICENSE.txt +21 -0
- data/README.md +176 -0
- data/bin/plai_testr +5 -0
- data/docs/frame_queue.md +123 -0
- data/docs/referee.md +59 -0
- data/lib/plai_testr/agents/ollama_client.rb +177 -0
- data/lib/plai_testr/agents/player_agent.rb +216 -0
- data/lib/plai_testr/agents/referee_agent.rb +56 -0
- data/lib/plai_testr/botrun.rb +60 -0
- data/lib/plai_testr/cli.rb +190 -0
- data/lib/plai_testr/config.rb +39 -0
- data/lib/plai_testr/deck.rb +11 -0
- data/lib/plai_testr/errors.rb +9 -0
- data/lib/plai_testr/event_log.rb +24 -0
- data/lib/plai_testr/game.rb +50 -0
- data/lib/plai_testr/generator.rb +50 -0
- data/lib/plai_testr/model_settings.rb +25 -0
- data/lib/plai_testr/names.rb +7 -0
- data/lib/plai_testr/personas.rb +97 -0
- data/lib/plai_testr/preflight.rb +70 -0
- data/lib/plai_testr/reporter/botrun_report.rb +141 -0
- data/lib/plai_testr/reporter/model_comparison.rb +104 -0
- data/lib/plai_testr/reporter/report_builder.rb +154 -0
- data/lib/plai_testr/reporter/think_log.rb +32 -0
- data/lib/plai_testr/rules_loader.rb +40 -0
- data/lib/plai_testr/session/frame_queue.rb +47 -0
- data/lib/plai_testr/session/manager.rb +77 -0
- data/lib/plai_testr/session/runner.rb +555 -0
- data/lib/plai_testr/state/base.rb +13 -0
- data/lib/plai_testr/state/game_state.rb +102 -0
- data/lib/plai_testr/state/player_state.rb +61 -0
- data/lib/plai_testr/version.rb +3 -0
- data/lib/plai_testr.rb +61 -0
- data/templates/app/.gitignore +3 -0
- data/templates/app/.rspec +3 -0
- data/templates/app/AGENTS.md +158 -0
- data/templates/app/CLAUDE.md +1 -0
- data/templates/app/Gemfile.erb +7 -0
- data/templates/app/README.md.erb +91 -0
- data/templates/app/config/plai_testr.rb +15 -0
- data/templates/app/spec/spec_helper.rb +25 -0
- data/templates/examples/gin_rummy/NOTES.md +12 -0
- data/templates/examples/gin_rummy/data/bots.yml +18 -0
- data/templates/examples/gin_rummy/data/personas.yml +26 -0
- data/templates/examples/gin_rummy/data/rules.json +76 -0
- data/templates/examples/gin_rummy/data/rules.md +44 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/bot.rb +34 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/game.rb +8 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/melds.rb +51 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/session/runner.rb +197 -0
- data/templates/examples/gin_rummy/lib/gin_rummy/state/game_state.rb +19 -0
- data/templates/examples/gin_rummy/lib/gin_rummy.rb +9 -0
- data/templates/examples/gin_rummy/spec/melds_spec.rb +26 -0
- data/templates/examples/gin_rummy/spec/session/runner_spec.rb +105 -0
- data/templates/examples/hearts/data/bots.yml +22 -0
- data/templates/examples/hearts/data/personas.yml +27 -0
- data/templates/examples/hearts/data/rules.json +73 -0
- data/templates/examples/hearts/data/rules.md +30 -0
- data/templates/examples/hearts/lib/hearts/bot.rb +53 -0
- data/templates/examples/hearts/lib/hearts/game.rb +16 -0
- data/templates/examples/hearts/lib/hearts/reporter/botrun_report.rb +29 -0
- data/templates/examples/hearts/lib/hearts/session/runner.rb +255 -0
- data/templates/examples/hearts/lib/hearts/state/game_state.rb +19 -0
- data/templates/examples/hearts/lib/hearts.rb +11 -0
- data/templates/examples/hearts/spec/session/runner_spec.rb +122 -0
- data/templates/game/data/bots.yml +13 -0
- data/templates/game/data/personas.yml +16 -0
- data/templates/game/data/rules.json.erb +16 -0
- data/templates/game/data/rules.md.erb +11 -0
- data/templates/game/lib/__name__/bot.rb.erb +7 -0
- data/templates/game/lib/__name__/session/runner.rb.erb +36 -0
- data/templates/game/lib/__name__/state/game_state.rb.erb +11 -0
- data/templates/game/lib/__name__.rb.erb +10 -0
- data/templates/game/spec/session/runner_spec.rb.erb +29 -0
- metadata +153 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
module PlaiTestr
|
|
2
|
+
module Reporter
|
|
3
|
+
class ThinkLog
|
|
4
|
+
HEADER = /^\[(\S+)\] model=(\S+)(.*)$/
|
|
5
|
+
|
|
6
|
+
def self.load(path)
|
|
7
|
+
new(File.exist?(path) ? File.read(path) : '')
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
attr_reader :calls
|
|
11
|
+
|
|
12
|
+
def initialize(text)
|
|
13
|
+
@calls = text.scan(HEADER).map do |time, model, stats|
|
|
14
|
+
stats.scan(/(\w+)=([\d.]+)/).to_h { |key, value| [key, value.to_f] }.merge('time' => Time.parse(time), 'model' => model)
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def values(key) = @calls.filter_map { |call| call[key] }
|
|
19
|
+
|
|
20
|
+
def average(key)
|
|
21
|
+
list = values(key)
|
|
22
|
+
list.empty? ? nil : list.sum / list.length
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def minutes
|
|
26
|
+
return nil if @calls.length < 2
|
|
27
|
+
|
|
28
|
+
(@calls.last['time'] - @calls.first['time']) / 60.0
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
module PlaiTestr
|
|
2
|
+
class RulesLoader
|
|
3
|
+
def self.load(path)
|
|
4
|
+
(@cache ||= {})[[File.expand_path(path), File.mtime(path)]] ||= parse(path)
|
|
5
|
+
rescue Errno::ENOENT
|
|
6
|
+
raise RulesError, "Rules file not found: #{path}"
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def self.parse(path)
|
|
10
|
+
rules = JSON.parse(File.read(path))
|
|
11
|
+
errors = validate(rules)
|
|
12
|
+
raise RulesError, "Invalid rules file #{path}:\n#{errors.map { |e| " - #{e}" }.join("\n")}" unless errors.empty?
|
|
13
|
+
|
|
14
|
+
rules
|
|
15
|
+
rescue Errno::ENOENT
|
|
16
|
+
raise RulesError, "Rules file not found: #{path}"
|
|
17
|
+
rescue JSON::ParserError => e
|
|
18
|
+
raise RulesError, "#{path} is not valid JSON: #{e.message}"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.validate(rules)
|
|
22
|
+
game = rules['game']
|
|
23
|
+
return ["'game' must be an object with a name and version"] unless game.is_a?(Hash)
|
|
24
|
+
|
|
25
|
+
errors = %w[name version].filter_map { |key| "game.#{key} is required" if game[key].to_s.empty? }
|
|
26
|
+
errors << player_count_error(game['player_count']) if game.key?('player_count')
|
|
27
|
+
rules.fetch('components', []).each do |component|
|
|
28
|
+
errors << "components: #{component['name']} 'values' must be an array" if component['type'] == 'card_deck' && !component.fetch('values', []).is_a?(Array)
|
|
29
|
+
end
|
|
30
|
+
errors.compact
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.player_count_error(range)
|
|
34
|
+
min, default, max = range.is_a?(Hash) ? range.values_at('min', 'default', 'max') : []
|
|
35
|
+
return nil if [min, default, max].all?(Integer) && min <= default && default <= max
|
|
36
|
+
|
|
37
|
+
'game.player_count needs whole numbers with min <= default <= max'
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
module PlaiTestr
|
|
2
|
+
module Session
|
|
3
|
+
module FrameQueue
|
|
4
|
+
def frame_queue
|
|
5
|
+
raise NotImplementedError, "#{self.class} must implement #frame_queue"
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
def schedule_frames
|
|
9
|
+
raise NotImplementedError, "#{self.class} must implement #schedule_frames"
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def halt_frames? = game_over?
|
|
13
|
+
|
|
14
|
+
def intercept_frame(_frame) = false
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def frame = frame_queue.last
|
|
19
|
+
|
|
20
|
+
def frame_for(type, player = nil, **data)
|
|
21
|
+
{ 'type' => type, 'player' => player }.merge(data.transform_keys(&:to_s))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def resolve_frame(params)
|
|
25
|
+
current = frame_queue.pop
|
|
26
|
+
send("on_#{current['type']}", current, params)
|
|
27
|
+
advance_frames
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def advance_frames
|
|
31
|
+
loop do
|
|
32
|
+
return if halt_frames?
|
|
33
|
+
|
|
34
|
+
schedule_frames if frame_queue.empty?
|
|
35
|
+
current = frame
|
|
36
|
+
|
|
37
|
+
if current['auto']
|
|
38
|
+
frame_queue.pop
|
|
39
|
+
send("on_#{current['type']}", current)
|
|
40
|
+
elsif !intercept_frame(current)
|
|
41
|
+
return
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
module PlaiTestr
|
|
2
|
+
module Session
|
|
3
|
+
class Manager
|
|
4
|
+
attr_reader :session_id
|
|
5
|
+
|
|
6
|
+
def initialize(session_id: nil, base_dir: PlaiTestr.sessions_dir, persist: true)
|
|
7
|
+
# A timestamp: readable and sortable. A second session started within the same second
|
|
8
|
+
# gets a counter suffix (-2, -3, …) so ids stay unique.
|
|
9
|
+
@base_dir = base_dir
|
|
10
|
+
@session_id = (session_id || timestamp_id).to_s
|
|
11
|
+
FileUtils.mkdir_p(session_dir) if persist
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def save_checkpoint(game_state, turn_log, metadata = {})
|
|
15
|
+
data = {
|
|
16
|
+
'session_id' => @session_id,
|
|
17
|
+
'saved_at' => Time.now.iso8601,
|
|
18
|
+
'status' => metadata['status'] || 'in_progress',
|
|
19
|
+
'game' => metadata['game'],
|
|
20
|
+
'rules_version' => metadata['rules_version'],
|
|
21
|
+
'fingerprint' => metadata['fingerprint'],
|
|
22
|
+
'game_state' => game_state.serialize,
|
|
23
|
+
'turn_log' => turn_log
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
temp = "#{checkpoint_path}.tmp"
|
|
27
|
+
File.open(temp, 'w') do |file|
|
|
28
|
+
file.write(JSON.pretty_generate(data))
|
|
29
|
+
file.fsync
|
|
30
|
+
end
|
|
31
|
+
File.rename(temp, checkpoint_path)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def load_checkpoint
|
|
35
|
+
JSON.parse(File.read(checkpoint_path))
|
|
36
|
+
rescue Errno::ENOENT
|
|
37
|
+
raise SessionError, "No saved session #{@session_id} in #{@base_dir}"
|
|
38
|
+
rescue JSON::ParserError => e
|
|
39
|
+
raise SessionError, "Session #{@session_id} cannot be resumed: #{checkpoint_path} is unreadable (#{e.message.lines.first.strip})"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def append_transcript_entry(entry)
|
|
43
|
+
File.open(transcript_path, 'a') { |f| f.puts(JSON.generate(entry)) }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def timestamp_id
|
|
47
|
+
stamp = Time.now.strftime('%Y%m%d-%H%M%S')
|
|
48
|
+
id = stamp
|
|
49
|
+
count = 1
|
|
50
|
+
id = "#{stamp}-#{count += 1}" while Dir.exist?(File.join(@base_dir, id))
|
|
51
|
+
id
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def session_dir
|
|
55
|
+
File.join(@base_dir, @session_id)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def checkpoint_path
|
|
59
|
+
File.join(session_dir, 'state.json')
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def transcript_path
|
|
63
|
+
File.join(session_dir, 'transcript.jsonl')
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.list(base_dir: PlaiTestr.sessions_dir)
|
|
67
|
+
return [] unless Dir.exist?(base_dir)
|
|
68
|
+
|
|
69
|
+
Dir.glob(File.join(base_dir, '*', 'state.json')).filter_map do |path|
|
|
70
|
+
JSON.parse(File.read(path)).slice('session_id', 'game', 'saved_at', 'status')
|
|
71
|
+
rescue JSON::ParserError
|
|
72
|
+
nil
|
|
73
|
+
end.sort_by { |s| s['saved_at'] }.reverse
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|