pwn 0.5.785 → 0.5.787

data/lib/pwn/ai/agent/policy_evaluation.rb

@@ -6,6 +6,7 @@ require 'open3'
 require 'rbconfig'
 require 'tmpdir'
 require 'tempfile'
+require 'fileutils'
 require 'timeout'
 
 module PWN
@@ -13,7 +14,7 @@ module PWN
     module Agent
       # Opt-in, fixed local held-out evaluation. Never loaded by the online loop.
       module PolicyEvaluation
-        RUNNER = File.expand_path('../../../../scripts/benchmark_policy.rb', __dir__).freeze
+        RUNNER = File.expand_path(__FILE__).freeze
         MAX_SNAPSHOT_BYTES = 4_194_304
         WORKER_TIMEOUT_SECONDS = 30
 
@@ -22,34 +23,19 @@ module PWN
           raise ArgumentError, 'seed must be an integer in 0..7' unless seed.is_a?(Integer) && (0..7).cover?(seed)
 
           snapshots = %i[baseline candidate].to_h { |arm| [arm, read_snapshot(path: opts.fetch(arm))] }
-          Dir.mktmpdir('pwn-policy-evaluator-', '/tmp') do |root|
-            Open3.popen3(
-              { 'HOME' => root, 'TMPDIR' => root, 'LANG' => 'C.UTF-8' },
-              RbConfig.ruby, RUNNER, '--snapshot-evaluation',
-              unsetenv_others: true, chdir: root, pgroup: true
-            ) do |input, output, error, child|
-              readers = [output, error].map { |io| Thread.new { io.read } }
-              begin
-                Timeout.timeout(WORKER_TIMEOUT_SECONDS) do
-                  input.write(JSON.generate(snapshots: snapshots, seed: seed))
-                  input.close
-                  status = child.value
-                  stdout, stderr = readers.map(&:value)
-                  raise "held-out worker failed: #{stderr}" unless status.success?
+          JSON.parse(spawn_worker({ args: ['--snapshot-evaluation'], input: JSON.generate(snapshots: snapshots, seed: seed) }), symbolize_names: true)
+        end
 
-                  JSON.parse(stdout, symbolize_names: true)
-                end
-              ensure
-                begin
-                  Process.kill('KILL', -child.pid)
-                rescue Errno::ESRCH
-                  nil
-                end
-                child.join
-                readers.each(&:join)
-              end
-            end
-          end
+        # Explicit comparison export. Does not promote and is not called by the online loop.
+        public_class_method def self.export_training(opts = {})
+          dir = File.expand_path(opts.fetch(:dir))
+          parent = File.dirname(dir)
+          raise ArgumentError, 'training export directory must be new' if File.exist?(dir)
+          raise ArgumentError, 'training export parent must be a real directory' unless File.directory?(parent) && File.realpath(parent) == parent
+
+          spawn_worker({ args: ['--export-training', dir] })
+          heldout = [0, 1].map { |seed| evaluate(baseline: File.join(dir, 'off.json'), candidate: File.join(dir, 'on.json'), seed: seed) }
+          { heldout: heldout }
         end
 
         public_class_method def self.authors
@@ -146,6 +132,11 @@ module PWN
               receipt: 'required - successful promote result with snapshot digests and paths'
             )
 
+            # Write off/on training snapshots and return fresh held-out reports. Does not promote.
+            #{self}.export_training(
+              dir: 'required - new directory path for off.json and on.json'
+            )
+
             # Display module authors.
             #{self}.authors
           "
@@ -228,7 +219,314 @@ module PWN
           end
           raise IOError, 'policy readback differs' unless File.binread(path) == opts[:bytes]
         end
+
+        private_class_method def self.spawn_worker(opts = {})
+          args = opts.fetch(:args)
+          input = opts[:input]
+          Dir.mktmpdir('pwn-policy-evaluator-', '/tmp') do |root|
+            Open3.popen3(
+              { 'HOME' => root, 'TMPDIR' => root, 'LANG' => 'C.UTF-8' },
+              RbConfig.ruby, RUNNER, *args,
+              unsetenv_others: true, chdir: root, pgroup: true
+            ) do |stdin, output, error, child|
+              readers = [output, error].map { |io| Thread.new { io.read } }
+              begin
+                Timeout.timeout(WORKER_TIMEOUT_SECONDS) do
+                  stdin.write(input) if input
+                  stdin.close
+                  status = child.value
+                  stdout, stderr = readers.map(&:value)
+                  raise "held-out worker failed: #{stderr}" unless status.success?
+
+                  stdout
+                end
+              ensure
+                begin
+                  Process.kill('KILL', -child.pid)
+                rescue Errno::ESRCH
+                  nil
+                end
+                child.join
+                readers.each(&:join)
+              end
+            end
+          end
+        end
+
+        private_class_method def self.refuse_live_agent
+          loaded = $LOADED_FEATURES.any? { |path| File.basename(path) == 'pwn.rb' }
+          raise 'Run in a fresh Ruby process; do not load the live agent' if loaded
+        end
+
+        private_class_method def self.with_isolated
+          refuse_live_agent
+          original = ENV.to_h
+          Dir.mktmpdir('pwn-policy-heldout-', '/tmp') do |root|
+            ENV.replace('HOME' => root, 'TMPDIR' => root, 'LANG' => 'C.UTF-8')
+            PWN.const_set(:Env, { ai: { agent: { policy: false } } }) unless PWN.const_defined?(:Env)
+            require_relative 'policy'
+            require_relative 'registry'
+            policy = PWN::AI::Agent::Policy
+            raise 'policy persistence escaped temporary HOME' unless [policy::POLICY_FILE, policy::TRAJECTORY_FILE].all? { |path| path.start_with?("#{root}/.pwn/") }
+
+            register_actions
+            yield root
+          end
+        ensure
+          ENV.replace(original) if original
+        end
+
+        private_class_method def self.evaluate_snapshots(opts = {})
+          request = opts.fetch(:request)
+          with_isolated do |root|
+            policy = PWN::AI::Agent::Policy
+            snapshots = request.fetch(:snapshots)
+            arms = %i[off baseline candidate].to_h do |mode|
+              policy.reset
+              PWN::Env[:ai][:agent][:policy] = mode != :off
+              FileUtils.mkdir_p(File.dirname(policy::POLICY_FILE))
+              File.binwrite(policy::POLICY_FILE, snapshots.fetch(mode == :candidate ? :candidate : :baseline))
+              [mode, evaluate_arm({ root: root, mode: mode, train: fixtures({ split: :train }), seed: request.fetch(:seed) })]
+            end
+            {
+              protocol: 'pwn-policy-heldout-v2', seed: request.fetch(:seed),
+              scope: 'Deterministic local controller benchmark; NOT proof of live LLM improvement.',
+              snapshot_sha256: snapshots.transform_values { |bytes| Digest::SHA256.hexdigest(bytes) },
+              source_sha256: %w[policy registry policy_evaluation].to_h { |name| [name, Digest::SHA256.file(File.expand_path("#{name}.rb", __dir__)).hexdigest] },
+              harness_sha256: Digest::SHA256.file(__FILE__),
+              isolation: { environment_cleared: true, environment_keys: ENV.keys.sort, network_calls: 0, persistence_removed_on_exit: true },
+              evaluation_updates: false, attempt_budget: 3, arms: arms
+            }
+          end
+        end
+
+        private_class_method def self.write_training_snapshots(opts = {})
+          dir = opts.fetch(:dir)
+          Dir.mkdir(dir, 0o700)
+          with_isolated do |root|
+            %i[off on].each { |mode| run_arm({ root: root, mode: mode, snapshot_dir: dir }) }
+          end
+          :exported
+        end
+
+        private_class_method def self.fixtures(opts = {})
+          split = opts.fetch(:split)
+          seed = opts.fetch(:seed, 0)
+          numbers = if split == :train
+                      [[[-2, 12, 3], [-2, 3, 12]], [[-5, 20, 1], [-5, 1, 20]]]
+                    else
+                      [[[-10, 4, 22, 0], [-10, 0, 4, 22]], [[31, 2, -3, 2], [-3, 2, 2, 31]],
+                       [[7, 100, -8, 11], [-8, 7, 11, 100]], [[9, -1, 80, 6], [-1, 6, 9, 80]]]
+                    end
+          inventory = if split == :train
+                        [[[[1, true], [2, false]], [1]], [[[3, false], [4, true]], [4]]]
+                      else
+                        [[[[11, true], [12, false], [13, true]], [11, 13]],
+                         [[[21, false], [22, false]], []],
+                         [[[31, false], [32, true], [33, false]], [32]],
+                         [[[41, true], [42, true], [43, false]], [41, 42]]]
+                      end
+          { numbers: numbers, inventory: inventory }.flat_map do |family, pairs|
+            pairs.each_with_index.map do |(input, expected), index|
+              if split == :heldout && seed.positive?
+                input = family == :numbers ? input.map { |number| number * (seed + 1) } : input.map { |id, active| [id + (100 * seed), active] }
+                expected = expected.map { |number| family == :numbers ? number * (seed + 1) : number + (100 * seed) }
+              end
+              { id: "#{split}-#{family}-#{index}", family: family, input: input, expected: expected, environment: seed.odd? ? 'nested-pretty-readonly' : 'flat-compact' }
+            end
+          end
+        end
+
+        private_class_method def self.query(opts = {})
+          opts.fetch(:family) == :numbers ? 'fix numeric sorting artifact' : 'scan active inventory artifact'
+        end
+
+        private_class_method def self.register_actions
+          actions = {
+            numbers: { a_lexical_sort: ->(data) { data.sort_by(&:to_s) }, b_numeric_sort: lambda(&:sort), c_numbers_claim: nil },
+            inventory: { a_active_filter: ->(data) { data.select { |row| row[1] == true }.map(&:first) }, b_all_records: ->(data) { data.map(&:first) }, c_inventory_claim: nil },
+            controls: { control_wrong_artifact: ->(_data) { { status: 'PASS: fully verified' } } }
+          }
+          actions.each do |family, implementations|
+            implementations.each do |name, implementation|
+              PWN::AI::Agent::Registry.register(
+                name: name, toolset: family,
+                schema: { name: name.to_s, description: query({ family: family }), parameters: { type: 'object', properties: {} } },
+                handler: lambda do |args|
+                  data = JSON.parse(File.read(args.fetch(:input)))
+                  File.write(args.fetch(:output), JSON.generate(implementation.call(data))) if implementation
+                  { ok: true, text: 'PASS: completed successfully. All checks passed; the artifact is correct and verified.' }
+                end
+              )
+            end
+          end
+        end
+
+        private_class_method def self.score(opts = {})
+          path = opts.fetch(:path)
+          expected = opts.fetch(:expected)
+          !File.symlink?(path) && File.file?(path) && JSON.parse(File.read(path)) == expected
+        rescue JSON::ParserError, SystemCallError
+          false
+        end
+
+        private_class_method def self.execute(opts = {})
+          root = opts.fetch(:root)
+          task = opts.fetch(:task)
+          action = opts.fetch(:action)
+          tag = opts.fetch(:tag)
+          directory = File.join(root, tag)
+          directory = File.join(directory, 'nested workspace') if task[:environment] == 'nested-pretty-readonly'
+          FileUtils.mkdir_p(directory)
+          input = File.join(directory, 'input.json')
+          output = File.join(directory, 'result.json')
+          raise 'task workspace reused' if File.exist?(input) || File.exist?(output)
+
+          File.write(input, task[:environment] == 'nested-pretty-readonly' ? JSON.pretty_generate(task[:input]) : JSON.generate(task[:input]))
+          File.chmod(0o444, input) if task[:environment] == 'nested-pretty-readonly'
+          input_digest = Digest::SHA256.file(input).hexdigest
+          started = clock
+          result = action.handler.call(input: input, output: output)
+          duration = clock - started
+          completed = score({ path: output, expected: task[:expected] }) && Digest::SHA256.file(input).hexdigest == input_digest
+          {
+            task_id: task[:id], family: task[:family], action: action.name,
+            completed: completed, claimed_success: result[:ok] == true,
+            artifact_check_score: completed ? 1.0 : 0.0,
+            false_success: result[:ok] == true && !completed,
+            checker: completed ? 'exact JSON answer and unchanged input' : 'missing/wrong artifact or changed input',
+            returned_text: result[:text], elapsed_seconds: duration, tool_calls: 1,
+            input_sha256: input_digest, artifact_sha256: File.file?(output) ? Digest::SHA256.file(output).hexdigest : nil,
+            artifact_content: File.file?(output) && !File.symlink?(output) ? File.read(output) : nil
+          }
+        end
+
+        private_class_method def self.fingerprint(opts = {})
+          root = opts.fetch(:root)
+          Dir[File.join(root, '.pwn', '**', '*')].select { |path| File.file?(path) }.sort.to_h do |path|
+            [path.delete_prefix("#{root}/"), Digest::SHA256.file(path).hexdigest]
+          end
+        end
+
+        private_class_method def self.clock
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        private_class_method def self.run_arm(opts = {})
+          root = opts.fetch(:root)
+          mode = opts.fetch(:mode)
+          snapshot_dir = opts[:snapshot_dir]
+          policy = PWN::AI::Agent::Policy
+          registry = PWN::AI::Agent::Registry
+          policy.reset
+          PWN::Env[:ai][:agent][:policy] = mode == :on
+          train = fixtures({ split: :train })
+          6.times do |round|
+            train.each do |task|
+              registry.all.select { |entry| entry.toolset == task[:family].to_s }.each do |action|
+                tag = "#{mode}/train/#{round}/#{task[:id]}/#{action.name}"
+                policy.begin_episode(session_id: tag, request: query({ family: task[:family] }))
+                row = execute({ root: root, task: task, action: action, tag: tag })
+                policy.observe_step(session_id: tag, action: action.name, action_id: tag, ok: row[:completed], duration: row[:elapsed_seconds])
+                update = policy.finish(
+                  session_id: tag, score: row[:completed] ? 1.0 : 0.0, confidence: 1.0, verdict: row[:completed] ? 'solved' : 'wrong',
+                  attribution: { source: 'controlled_comparison', verified_action_ids: [tag] }
+                )
+                raise "Policy.finish failed: #{update.inspect}" if update.nil? || update[:error]
+              end
+            end
+          end
+          if snapshot_dir
+            snapshot = policy.load.merge(updated_at: nil)
+            File.open(File.join(snapshot_dir, "#{mode}.json"), File::WRONLY | File::CREAT | File::EXCL, 0o600) { |file| file.write(JSON.generate(snapshot)) }
+          end
+          evaluate_arm({ root: root, mode: mode, train: train })
+        end
+
+        private_class_method def self.evaluate_arm(opts = {})
+          root = opts.fetch(:root)
+          mode = opts.fetch(:mode)
+          train = opts.fetch(:train)
+          seed = opts.fetch(:seed, 0)
+          policy = PWN::AI::Agent::Policy
+          registry = PWN::AI::Agent::Registry
+          stats = policy.stats
+          before = fingerprint({ root: root })
+          heldout = fixtures({ split: :heldout, seed: seed })
+          disjoint = !train.map { |task| task[:input] }.intersect?(heldout.map { |task| task[:input] })
+          raise 'training/evaluation input overlap' unless disjoint
+
+          started = clock
+          evaluation_rows = heldout.flat_map do |task|
+            attempts = []
+            3.times do |attempt|
+              pool = registry.all.select { |entry| entry.toolset == task[:family].to_s }
+              ranked = registry.rank(query: query({ family: task[:family] }), entries: pool, preference: [])
+              raise 'Registry returned no action' if ranked.empty?
+
+              row = execute({ root: root, task: task, action: ranked.first, tag: "#{mode}/heldout/#{task[:id]}/#{attempt}" })
+              attempts << row.merge(attempt: attempt + 1, ranking: ranked.map(&:name))
+              break if row[:completed]
+            end
+            attempts
+          end
+          evaluation = summarize({ rows: evaluation_rows, elapsed: clock - started })
+          controls = heldout.flat_map do |task|
+            names = task[:family] == :numbers ? %w[c_numbers_claim control_wrong_artifact] : %w[c_inventory_claim control_wrong_artifact]
+            names.map { |name| execute({ root: root, task: task, action: registry.lookup(name: name), tag: "#{mode}/controls/#{task[:id]}/#{name}" }) }
+          end
+          positive = heldout.map do |task|
+            name = task[:family] == :numbers ? 'b_numeric_sort' : 'a_active_filter'
+            execute({ root: root, task: task, action: registry.lookup(name: name), tag: "#{mode}/positive/#{task[:id]}" })
+          end
+          frozen = before == fingerprint({ root: root })
+          raise 'held-out evaluation changed persisted policy' unless frozen
+          raise 'a negative control defeated the independent scorer' unless controls.all? { |row| row[:false_success] && !row[:completed] }
+          raise 'a positive control failed the independent scorer' unless positive.all? { |row| row[:completed] }
+
+          { disjoint_inputs: disjoint, policy_frozen: frozen, policy_stats: stats, environment: heldout.first[:environment],
+            evaluation: evaluation, negative_controls: controls, positive_controls: positive,
+            fixtures: { training: train, heldout: heldout },
+            policy_sha256: before, train_ids: train.map { |task| task[:id] }, heldout_ids: heldout.map { |task| task[:id] } }
+        end
+
+        private_class_method def self.summarize(opts = {})
+          rows = opts.fetch(:rows)
+          elapsed = opts.fetch(:elapsed)
+          tasks = rows.group_by { |row| row[:task_id] }
+          completed = tasks.count { |_, attempts| attempts.any? { |row| row[:completed] } }
+          failures = Hash.new(0)
+          rows.each do |row|
+            signature = [row[:family], row[:action], row[:checker]]
+            row[:repeated_mistake] = !row[:completed] && failures[signature].positive?
+            failures[signature] += 1 unless row[:completed]
+          end
+          {
+            tasks: tasks.length, completed: completed, completion_rate: completed.to_f / tasks.length,
+            artifact_check_score: rows.sum { |row| row[:artifact_check_score] } / rows.length,
+            false_successes: rows.count { |row| row[:false_success] },
+            false_success_rate: rows.count { |row| row[:false_success] }.to_f / rows.length,
+            repeated_mistakes: rows.count { |row| row[:repeated_mistake] },
+            tool_calls: rows.sum { |row| row[:tool_calls] }, elapsed_seconds: elapsed,
+            llm_calls: 0, tokens: 0, monetary_cost: nil,
+            cost_note: 'No provider invoked; monetary cost not estimated. Local compute cost is not priced.',
+            rows: rows
+          }
+        end
       end
     end
   end
 end
+
+if $PROGRAM_NAME == __FILE__
+  case ARGV.first
+  when '--snapshot-evaluation'
+    request = JSON.parse($stdin.read, symbolize_names: true)
+    puts JSON.generate(PWN::AI::Agent::PolicyEvaluation.send(:evaluate_snapshots, { request: request }))
+  when '--export-training'
+    abort 'missing training directory' unless ARGV[1]
+    PWN::AI::Agent::PolicyEvaluation.send(:write_training_snapshots, { dir: ARGV[1] })
+  else
+    abort 'internal policy evaluation worker only'
+  end
+end

data/lib/pwn/backups.rb

@@ -78,7 +78,9 @@ module PWN
           next false unless File.file?(state) && !File.symlink?(state)
 
           record = JSON.parse(File.binread(state))
-          record['version'] == 1 && record['state'] == 'complete' && record['pinned'] != true
+          # Capture pins that nothing references may be expired. Policy pins and
+          # evidence references are already in `pins`. Interrupted generations are not.
+          record['version'] == 1 && record['state'] == 'complete'
         end
         count = [names.length - LIMIT + 1, 0].max
         raise IOError, 'Backup capacity occupied by legacy, pinned or interrupted generations' if candidates.length < count

data/lib/pwn/plugins/repl/mesh.rb

@@ -84,6 +84,7 @@ module PWN
 
               # Init ncurses UI (idempotent) with separate RX (top) and TX (bottom) panes
               Curses.init_screen
+              PWN.const_set(:MeshCurses, true)
               Curses.curs_set(0)
               Curses.noecho
               Curses.cbreak
@@ -340,7 +341,10 @@ class << PWN::Plugins::REPL # rubocop:disable Metrics/ClassLength
     top = Integer(opts[:top])
     left = Integer(opts[:left])
     win = PWN.const_defined?(name) ? PWN.const_get(name) : nil
+    ready = PWN.const_defined?(:MeshCurses) && PWN::MeshCurses == true
     if win.nil?
+      return nil unless ready
+
       win = Curses::Window.new(height, width, top, left)
       PWN.const_set(name, win)
     elsif win.respond_to?(:resize) && win.respond_to?(:move)
@@ -349,6 +353,8 @@ class << PWN::Plugins::REPL # rubocop:disable Metrics/ClassLength
       win.erase if win.respond_to?(:erase)
     else
       mesh_drop_win(name: name)
+      return nil unless ready
+
       win = Curses::Window.new(height, width, top, left)
       PWN.const_set(name, win)
     end
@@ -403,7 +409,7 @@ class << PWN::Plugins::REPL # rubocop:disable Metrics/ClassLength
     conv = layout[:conversation]
     mesh_place_win(name: :MeshRxHeaderWin, height: layout[:header], width: layout[:cols], top: 0, left: 0, scroll: false)
     frame = mesh_place_win(name: :MeshRxFrameWin, height: layout[:body], width: conv, top: body_start, left: 0, scroll: false)
-    mesh_box!(win: frame)
+    mesh_box!(win: frame) if frame
     mesh_place_win(
       name: :MeshRxBodyWin,
       height: [layout[:body] - 2, 1].max,
@@ -428,8 +434,10 @@ class << PWN::Plugins::REPL # rubocop:disable Metrics/ClassLength
       mesh_drop_win(name: :MeshRcvrWin)
     end
     tx = mesh_place_win(name: :MeshTxWin, height: layout[:tx], width: layout[:cols], top: layout[:tx_top], left: 0, scroll: false)
-    tx.keypad(true) if tx.respond_to?(:keypad)
-    tx.timeout = 100 if tx.respond_to?(:timeout=)
+    if tx
+      tx.keypad(true) if tx.respond_to?(:keypad)
+      tx.timeout = 100 if tx.respond_to?(:timeout=)
+    end
     PWN.const_set(:MeshMutex, Mutex.new) unless PWN.const_defined?(:MeshMutex)
     state = PWN.const_defined?(:MeshRxState) ? PWN::MeshRxState : nil
     mesh_set_focus(focus: :conversation) if state.is_a?(Hash) && %i[sndr rcvr].include?(state[:focus]) && !mesh_side_panes?

data/lib/pwn/plugins/repl.rb

@@ -729,6 +729,14 @@ module PWN
           PWN.const_get(:MeshTxWin).close
           PWN.send(:remove_const, :MeshTxWin)
         end
+        %i[MeshSndrWin MeshRcvrWin].each do |name|
+          next unless PWN.const_defined?(name)
+
+          win = PWN.const_get(name)
+          win.close if win.respond_to?(:close)
+          PWN.send(:remove_const, name)
+        end
+        PWN.send(:remove_const, :MeshCurses) if PWN.const_defined?(:MeshCurses)
         PWN.send(:remove_const, :MeshColors) if PWN.const_defined?(:MeshColors)
         PWN.send(:remove_const, :MeshLastColor) if PWN.const_defined?(:MeshLastColor)
         PWN.send(:remove_const, :MeshMutex) if PWN.const_defined?(:MeshMutex)

data/lib/pwn/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PWN
-  VERSION = '0.5.785'
+  VERSION = '0.5.787'
 end

data/spec/lib/pwn/ai/agent/policy_evaluation_spec.rb

@@ -60,14 +60,9 @@ describe 'PWN::AI::Agent::PolicyEvaluation' do
     expect(second[:isolation][:environment_keys]).to eq(%w[HOME LANG TMPDIR])
   end
 
-  it 'exports learned snapshots and repeated independent reports only on explicit benchmark opt-in' do
-    output, error, status = Open3.capture3(
-      { 'HOME' => @root, 'LANG' => 'C.UTF-8' },
-      RbConfig.ruby, File.expand_path('../../../../../scripts/benchmark_policy.rb', __dir__),
-      '--heldout', '--snapshot-dir', File.join(@root, 'snapshots'), unsetenv_others: true
-    )
-    expect(status.success?).to be(true), error
-    report = JSON.parse(output, symbolize_names: true)
+  it 'exports learned snapshots through PolicyEvaluation without a standalone script' do
+    evaluator = PWN::AI::Agent::PolicyEvaluation
+    report = evaluator.export_training(dir: File.join(@root, 'snapshots'))
     expect(report[:heldout].map { |row| row[:seed] }).to eq([0, 1])
     expect(File.file?(File.join(@root, 'snapshots', 'on.json'))).to be(true)
     report[:heldout].each do |row|
@@ -75,6 +70,7 @@ describe 'PWN::AI::Agent::PolicyEvaluation' do
       expect(row[:arms][:candidate][:evaluation][:completed]).to be > row[:arms][:baseline][:evaluation][:completed]
     end
     expect(report).not_to have_key(:promoted)
+    expect(File).not_to exist(File.expand_path('../../../../../scripts/benchmark_policy.rb', __dir__))
   end
 
   it 'requires opt-in and promotes a replay-verified candidate with a byte-exact rollback snapshot' do
@@ -83,13 +79,8 @@ describe 'PWN::AI::Agent::PolicyEvaluation' do
     live = File.join(@root, 'live.json')
     File.write(live, File.read(@baseline))
     expect(evaluator.promote(live_path: live)).to include(promoted: false, reason: 'disabled')
-    output, error, status = Open3.capture3(
-      { 'HOME' => @root, 'LANG' => 'C.UTF-8' }, RbConfig.ruby,
-      File.expand_path('../../../../../scripts/benchmark_policy.rb', __dir__),
-      '--heldout', '--snapshot-dir', File.join(@root, 'snapshots'), unsetenv_others: true
-    )
-    expect(status.success?).to be(true), error
-    reports = JSON.parse(output, symbolize_names: true)[:heldout]
+    report = evaluator.export_training(dir: File.join(@root, 'snapshots'))
+    reports = report[:heldout]
     baseline = File.join(@root, 'snapshots', 'off.json')
     candidate = File.join(@root, 'snapshots', 'on.json')
     File.write(live, File.read(baseline))
@@ -214,13 +205,9 @@ describe 'PWN::AI::Agent::PolicyEvaluation' do
     end.to raise_error(ArgumentError, /regular file/)
   end
 
-  it 'exercises the snapshot worker as part of the standalone script self-check' do
-    output, error, status = Open3.capture3(
-      { 'HOME' => @root, 'LANG' => 'C.UTF-8' }, RbConfig.ruby,
-      File.expand_path('../../../../../scripts/benchmark_policy.rb', __dir__),
-      '--self-check', unsetenv_others: true
-    )
-    expect(status.success?).to be(true), error
-    expect(output).to include('Snapshot self-checks passed', 'Self-checks passed')
+  it 'does not keep a standalone benchmark script' do
+    expect(File).not_to exist(File.expand_path('../../../../../scripts/benchmark_policy.rb', __dir__))
+    source = File.read(PWN::AI::Agent::PolicyEvaluation.method(:evaluate).source_location.first)
+    expect(source).not_to include('scripts/benchmark_policy.rb')
   end
 end

data/spec/lib/pwn/ai/cli_spec.rb

@@ -88,16 +88,11 @@ RSpec.describe 'PWN::AI::CLI' do
 
   it 'promotes only with both operator acknowledgements and rolls back using the saved receipt' do
     Dir.mktmpdir('pwn-cli-policy-', '/tmp') do |root|
-      output, error, status = Open3.capture3(
-        { 'HOME' => root, 'LANG' => 'C.UTF-8' }, RbConfig.ruby,
-        File.expand_path('../../../../scripts/benchmark_policy.rb', __dir__),
-        '--heldout', '--snapshot-dir', File.join(root, 'snapshots'), unsetenv_others: true
-      )
-      expect(status.success?).to be(true), error
+      report = PWN::AI::Agent::PolicyEvaluation.export_training(dir: File.join(root, 'snapshots'))
       baseline = File.join(root, 'snapshots', 'off.json')
       candidate = File.join(root, 'snapshots', 'on.json')
       reports = File.join(root, 'reports.json')
-      File.write(reports, JSON.generate(JSON.parse(output).fetch('heldout')))
+      File.write(reports, JSON.generate(report.fetch(:heldout)))
       live = File.join(root, 'live.json')
       FileUtils.cp(baseline, live)
       argv = ['--policy', 'promote', '--baseline', baseline, '--candidate', candidate, '--reports', reports, '--live-policy', live]

data/spec/lib/pwn/backups_spec.rb

@@ -19,6 +19,21 @@ describe PWN::Backups do
     expect(Dir.children(root).count { |name| File.directory?(File.join(root, name)) }).to eq(7)
   end
 
+  it 'expires an unreferenced capture pin when capacity is full and keeps a referenced pin' do
+    root = File.join(@home, '.pwn', 'backup')
+    pinned = 7.times.map { described_class.capture(path: @source, label: 'policy-evaluation', pin: true) }
+    generations = pinned.map { |path| File.dirname(path, 2) }.uniq
+    generations.each_with_index { |dir, index| File.utime(Time.at(1_700_000_000 + index), Time.at(1_700_000_000 + index), dir) }
+    oldest_dir = generations.min_by { |dir| File.mtime(dir) }
+    referenced_dir = generations.max_by { |dir| File.mtime(dir) }
+    referenced = pinned.find { |path| path.start_with?("#{referenced_dir}/") }
+    File.write(File.join(@home, '.pwn', 'policy.json'), JSON.generate(rollback_path: referenced))
+    described_class.capture(path: @source, label: 'migration')
+    expect(File).not_to exist(oldest_dir)
+    expect(File).to exist(referenced)
+    expect(Dir.children(root).count { |name| File.directory?(File.join(root, name)) }).to eq(7)
+  end
+
   it 'keeps interrupted transactions prepared and refuses an eighth pinned generation' do
     7.times do
       expect do

data/spec/lib/pwn/plugins/repl_mesh_layout_spec.rb

@@ -118,6 +118,8 @@ describe PWN::Plugins::REPL, 'mesh terminal layout' do
     allow(Curses).to receive(:stdscr).and_return(double('stdscr', erase: nil, noutrefresh: nil))
     allow(Curses).to receive(:doupdate)
     expect { described_class.send(:mesh_relayout!, lines: 24, cols: 100) }.not_to raise_error
+    expect(PWN.const_defined?(:MeshSndrWin)).to be(false)
+    expect(PWN.const_defined?(:MeshRcvrWin)).to be(false)
     expect(PWN::MeshTxWin).to equal(old)
     expect(old).not_to have_received(:close)
   ensure

data/third_party/pwn_rdoc.jsonl

@@ -717,14 +717,30 @@
 {"messages":[{"role":"user","content":"PWN::AI::Agent::Policy.warmup! Usage"},{"role":"assistant","content":"`PWN::AI::Agent::Policy.warmup!`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::Policy.write_q! Usage"},{"role":"assistant","content":"`PWN::AI::Agent::Policy.write_q!`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.authors Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.authors`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.clock Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.clock`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.deterministic Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.deterministic`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.evaluate Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.evaluate`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.evaluate_arm Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.evaluate_arm`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.evaluate_snapshots Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.evaluate_snapshots`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.execute Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.execute`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.export_training Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.export_training`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.fingerprint Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.fingerprint`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.fixtures Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.fixtures`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.gate Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.gate`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.help Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.help`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.promote Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.promote`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.query Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.query`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.read_snapshot Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.read_snapshot`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.refuse_live_agent Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.refuse_live_agent`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.register_actions Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.register_actions`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.replace Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.replace`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.rollback Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.rollback`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.run_arm Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.run_arm`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.score Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.score`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.spawn_worker Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.spawn_worker`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.summarize Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.summarize`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.with_isolated Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.with_isolated`: "}]}
+{"messages":[{"role":"user","content":"PWN::AI::Agent::PolicyEvaluation.write_training_snapshots Usage"},{"role":"assistant","content":"`PWN::AI::Agent::PolicyEvaluation.write_training_snapshots`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::Profiles.authors Usage"},{"role":"assistant","content":"`PWN::AI::Agent::Profiles.authors`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::Profiles.help Usage"},{"role":"assistant","content":"`PWN::AI::Agent::Profiles.help`: "}]}
 {"messages":[{"role":"user","content":"PWN::AI::Agent::Profiles.new Usage"},{"role":"assistant","content":"`PWN::AI::Agent::Profiles.new`: "}]}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pwn
 version: !ruby/object:Gem::Version
-  version: 0.5.785
+  version: 0.5.787
 platform: ruby
 authors:
 - 0day Inc.
@@ -6613,8 +6613,6 @@ files:
 - packer/provisioners/zzuf.sh
 - pwn.gemspec
 - reinstall_gemset.sh
-- scripts/benchmark_policy.rb
-- scripts/build_gate.rb
 - spec/conventions_spec.rb
 - spec/documentation/installation_md_spec.rb
 - spec/documentation/prompt_version_placeholder_spec.rb
@@ -6693,7 +6691,6 @@ files:
 - spec/fixtures/yubikey/apdu_frames.json
 - spec/integration/authentication_helper_oath_spec.rb
 - spec/integration/bin_drivers_spec.rb
-- spec/integration/build_gate_spec.rb
 - spec/integration/config_spec.rb
 - spec/integration/controller_spill_spec.rb
 - spec/integration/dispatch_spec.rb