pwn 0.5.686 → 0.5.688

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.ruby-version +1 -1
  4. data/Gemfile +3 -3
  5. data/README.md +3 -4
  6. data/documentation/Agent-Tool-Registry.md +2 -2
  7. data/documentation/Configuration.md +5 -4
  8. data/documentation/General-PWN-Usage.md +1 -1
  9. data/documentation/Home.md +1 -1
  10. data/documentation/How-PWN-Works.md +1 -1
  11. data/documentation/Mistakes.md +1 -1
  12. data/documentation/Reinforcement-Learning.md +4 -6
  13. data/documentation/Troubleshooting.md +1 -1
  14. data/documentation/pwn-REPL.md +1 -1
  15. data/documentation/pwn-ai-Agent.md +2 -3
  16. data/lib/pwn/ai/agent/curriculum.rb +2 -2
  17. data/lib/pwn/ai/agent/dispatch.rb +1 -1
  18. data/lib/pwn/ai/agent/learning.rb +150 -32
  19. data/lib/pwn/ai/agent/loop.rb +291 -129
  20. data/lib/pwn/ai/agent/prompt_builder.rb +47 -29
  21. data/lib/pwn/ai/agent/registry.rb +2 -1
  22. data/lib/pwn/ai/agent/reward.rb +9 -0
  23. data/lib/pwn/ai/agent/swarm.rb +3 -1
  24. data/lib/pwn/ai/agent/task_summarizer.rb +23 -6
  25. data/lib/pwn/ai/agent/tool_guard.rb +109 -0
  26. data/lib/pwn/ai/agent/tools/ruby_eval.rb +44 -20
  27. data/lib/pwn/ai/agent/tools/shell.rb +17 -4
  28. data/lib/pwn/ai/agent/tools/skills.rb +63 -2
  29. data/lib/pwn/ai/anthropic.rb +25 -5
  30. data/lib/pwn/ai/gemini.rb +25 -5
  31. data/lib/pwn/ai/grok.rb +30 -13
  32. data/lib/pwn/ai/http_retry.rb +78 -0
  33. data/lib/pwn/ai/ollama.rb +25 -5
  34. data/lib/pwn/ai/open_ai.rb +26 -13
  35. data/lib/pwn/ai/open_web_ui.rb +25 -5
  36. data/lib/pwn/ai.rb +1 -0
  37. data/lib/pwn/config.rb +1 -1
  38. data/lib/pwn/plugins/log.rb +183 -41
  39. data/lib/pwn/plugins/repl.rb +50 -15
  40. data/lib/pwn/plugins/transparent_browser.rb +16 -2
  41. data/lib/pwn/sessions.rb +50 -1
  42. data/lib/pwn/version.rb +1 -1
  43. data/spec/integration/reinforced_feedback_loop_spec.rb +9 -11
  44. data/spec/lib/pwn/ai/agent/loop_spec.rb +180 -20
  45. data/spec/lib/pwn/ai/agent/prompt_builder_spec.rb +3 -0
  46. data/spec/lib/pwn/ai/agent/registry_spec.rb +1 -1
  47. data/spec/lib/pwn/ai/agent/reward_spec.rb +13 -0
  48. data/spec/lib/pwn/ai/agent/swarm_spec.rb +6 -0
  49. data/spec/lib/pwn/ai/agent/task_summarizer_spec.rb +36 -0
  50. data/spec/lib/pwn/ai/agent/tool_guard_spec.rb +46 -0
  51. data/spec/lib/pwn/ai/agent/tools/ruby_eval_spec.rb +24 -0
  52. data/spec/lib/pwn/ai/agent/tools/shell_spec.rb +10 -0
  53. data/spec/lib/pwn/ai/agent/tools/skills_spec.rb +29 -0
  54. data/spec/lib/pwn/ai/grok_spec.rb +22 -0
  55. data/spec/lib/pwn/ai/http_retry_spec.rb +55 -0
  56. data/spec/lib/pwn/plugins/log_spec.rb +170 -19
  57. data/spec/lib/pwn/plugins/repl_pwn_vault_spec.rb +14 -4
  58. data/spec/lib/pwn/plugins/transparent_browser_spec.rb +8 -0
  59. data/spec/lib/pwn/sessions_spec.rb +24 -0
  60. data/third_party/pwn_rdoc.jsonl +41 -5
  61. data/tmp_pwn_critic_skills_flaw.txt +9 -0
  62. metadata +11 -8
@@ -16,22 +16,21 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
16
16
  end
17
17
 
18
18
  describe 'RL-adjacent loop contracts' do # rubocop:disable Metrics/BlockLength
19
- it 'does not start a TTY spinner on engine calls (leftover spin overwrites PS1)' do
19
+ it 'spins on engine HTTP wait and still dispatches on_tool (debug must not hide the TUI)' do
20
20
  src = File.read(described_class.method(:run).source_location.first)
21
- expect(src).not_to match(/spinner:\s*true/)
22
- expect(src).to match(/spinner:\s*false/)
21
+ expect(src).to match(/spinner:\s*true/)
22
+ expect(src).to match(/on_tool&?\.call/)
23
+ expect(src).not_to match(/on_tool\s*=\s*nil/)
23
24
  end
24
25
 
25
- it 'exhaust path still calls task_summary_flush! and auto_introspect' do
26
+ it 'does not treat a round cap as a finished answer' do
26
27
  src = File.read(described_class.method(:run).source_location.first)
27
- # budget exhausted terminal path
28
- expect(src).to match(/iteration budget exhausted/)
29
- # after the exhausted final_msg, both flush and auto_introspect must appear
30
- exhaust = src[/final_msg = '\[pwn-ai\] iteration budget exhausted'.*/m]
31
- expect(exhaust).to match(/Learning\.auto_introspect/)
32
- expect(exhaust).to match(/task_summary_flush!/)
33
- # flush before return of final_msg
34
- expect(exhaust).to match(/task_summary_flush!.*final_msg/m)
28
+ expect(src).not_to match(/iteration budget exhausted/)
29
+ expect(src).not_to match(/final budget_exhausted/)
30
+ expect(src).not_to match(/shape: :budget_exhausted/)
31
+ expect(src).not_to match(/max_iters\.times/)
32
+ expect(src).to match(/may_finalize\?/)
33
+ expect(src).to match(/loop do/)
35
34
  end
36
35
 
37
36
  it 'plan_first unifies ts_state plan after red_team (P2)' do
@@ -359,6 +358,128 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
359
358
  expect(described_class.needs_host_work?(request: 'Write hello into /tmp/x.txt')).to eq true
360
359
  end
361
360
 
361
+ it 'skips the implement compass on a browse/navigate request' do
362
+ req = 'Navigate to https://0dayinc.com using TransparentBrowser.open and list blog URLs'
363
+ expect(described_class.send(:request_need, request: req)).to eq :browse
364
+ src = File.read(described_class.method(:run).source_location.first)
365
+ expect(src).to match(/skip_compass = .*browse/)
366
+ end
367
+
368
+ it 'does not bounce a text-only turn for missing tool evidence' do
369
+ req = 'Navigate to https://0dayinc.com using TransparentBrowser.open and list blog URLs'
370
+ expect(described_class.needs_host_work?(request: req)).to eq true
371
+ expect(
372
+ described_class.send(
373
+ :request_unsatisfied?,
374
+ request: req,
375
+ messages: [{ role: 'assistant', content: 'SOUND — step 1 is the likely fail.', tool_calls: [] }]
376
+ )
377
+ ).to eq true
378
+ Thread.current[:pwn_loop_no_tools] = true
379
+ expect(
380
+ described_class.send(
381
+ :request_unsatisfied?,
382
+ request: req,
383
+ messages: [{ role: 'assistant', content: 'SOUND — step 1 is the likely fail.', tool_calls: [] }]
384
+ )
385
+ ).to eq false
386
+ src = File.read(described_class.method(:run).source_location.first)
387
+ expect(src).to match(/pwn_loop_no_tools/)
388
+ expect(src).to match(/skip_compass = .*no_tools/)
389
+ ensure
390
+ Thread.current[:pwn_loop_no_tools] = nil
391
+ end
392
+
393
+ it 'does not start post-answer review from a nested or tool-less Loop.run' do
394
+ src = File.read(described_class.method(:run).source_location.first)
395
+ expect(src).to match(/!nested && !no_tools && should_auto_introspect\?/)
396
+ expect(
397
+ described_class.send(
398
+ :should_auto_introspect?,
399
+ local: false,
400
+ turn_fails: {},
401
+ iter: 1
402
+ )
403
+ ).to eq true
404
+ Thread.current[:pwn_loop_no_tools] = true
405
+ expect(
406
+ described_class.send(
407
+ :should_auto_introspect?,
408
+ local: false,
409
+ turn_fails: {},
410
+ iter: 1
411
+ )
412
+ ).to eq false
413
+ ensure
414
+ Thread.current[:pwn_loop_no_tools] = nil
415
+ end
416
+
417
+ it 'writes full tool request/result to the debug log and timestamps Interrupt' do
418
+ src = File.read(described_class.method(:run).source_location.first)
419
+ expect(src).to match(/debug_tool_io!/)
420
+ expect(src).to match(/tool #\{name\} start/)
421
+ expect(src).to match(/rescue Interrupt/)
422
+ expect(src).to match(/note_interrupt!/)
423
+ expect(src).to match(/pwn_log_progress/)
424
+ expect(src).to match(/note_exception!/)
425
+ expect(src).to match(/rescue StandardError/)
426
+ expect(src).to include("debug_progress(msg: 'engine returned no message')")
427
+ expect(src).not_to match(/dispatch #\{name\} ok_len=/)
428
+ end
429
+
430
+ it 'records a reconstruct-first timeout mistake instead of treating success:true as ok' do
431
+ tmp = Dir.mktmpdir
432
+ stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
433
+ raw = JSON.generate(
434
+ success: true,
435
+ result: { stdout: '', error: 'timeout after 20s', hint: 'x', scenario: 'construction' }
436
+ )
437
+ tele = described_class.send(
438
+ :record_metrics,
439
+ name: 'pwn_eval',
440
+ started: Time.now,
441
+ raw: raw,
442
+ args: '{"code":"sleep 3"}'
443
+ )
444
+ expect(tele[:ok]).to eq false
445
+ expect(tele[:mistake]).to be_a(Hash)
446
+ expect(tele[:mistake][:error].to_s).to match(/reconstruct/)
447
+ expect(tele[:mistake][:shape].to_s).to eq('timeout')
448
+ end
449
+
450
+ it 'treats a skills-catalog ask as recall, not trivia or host-work' do
451
+ req = 'what skills are available?'
452
+ expect(described_class.catalog_lookup?(request: req)).to eq true
453
+ expect(described_class.world_knowledge?(request: req)).to eq false
454
+ expect(described_class.needs_host_work?(request: req)).to eq false
455
+ expect(described_class.send(:request_need, request: req)).to eq :read
456
+ expect(
457
+ described_class.send(
458
+ :request_unsatisfied?,
459
+ request: req,
460
+ messages: [{ role: 'assistant', content: '103 skills…', tool_calls: [] }]
461
+ )
462
+ ).to eq true
463
+ expect(
464
+ described_class.send(
465
+ :request_unsatisfied?,
466
+ request: req,
467
+ messages: [
468
+ {
469
+ role: 'assistant',
470
+ content: '',
471
+ tool_calls: [
472
+ { function: { name: 'skills_recall', arguments: '{}' } }
473
+ ]
474
+ }
475
+ ]
476
+ )
477
+ ).to eq false
478
+ src = File.read(described_class.method(:run).source_location.first)
479
+ expect(src).to match(/skip_plan.*catalog/)
480
+ expect(src).to match(/force_plan && cal_state\[:cal\] && !skip_plan/)
481
+ end
482
+
362
483
  it 'does not treat rspec/rubocop green as a finished write/docs ask' do
363
484
  req = 'Update all markdown files to reflect operational changes and rebuild diagrams'
364
485
  verified = [
@@ -415,7 +536,7 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
415
536
  expect(src).not_to match(/blob\.match\?\(MUTATION_EVIDENCE_RX\)/)
416
537
  end
417
538
 
418
- it 'keeps tools on last_iter for any unfinished host-work request' do
539
+ it 'keeps tools for any unfinished host-work request — no last-iter strip' do
419
540
  req = 'Update all markdown files and rebuild diagrams'
420
541
  verified = [
421
542
  { role: 'tool', name: 'shell', content: "12 examples, 0 failures\n" }
@@ -429,8 +550,8 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
429
550
  )
430
551
  ).to eq true
431
552
  src = File.read(described_class.method(:run).source_location.first)
432
- expect(src).to match(/still_open = request_unsatisfied\?/)
433
- expect(src).to match(/last_iter = want_last && !still_open/)
553
+ expect(src).not_to match(/last_iter = want_last && !still_open/)
554
+ expect(src).to match(/call_engine\(messages: messages, tools: tools/)
434
555
  end
435
556
 
436
557
  it 'never accepts a text final while the original host-work request is unsatisfied' do
@@ -546,9 +667,9 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
546
667
  ).to eq false
547
668
  end
548
669
 
549
- it 'last-iter strips tools only on the true last slot; leftover English tasks do not force required' do
670
+ it 'does not strip tools on a last slot; leftover English tasks do not force required' do
550
671
  src = File.read(described_class.method(:run).source_location.first)
551
- expect(src).to match(/text_only_iters = 1/)
672
+ expect(src).not_to match(/text_only_iters = 1/)
552
673
  expect(src).not_to match(/english_open && hot/)
553
674
  expect(src).to match(/has_tool_result \|\| !need_tools \? 'auto' : 'required'/)
554
675
  expect(src).not_to match(/still_acting/)
@@ -671,6 +792,37 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
671
792
  FileUtils.rm_rf(sess_tmp) if sess_tmp
672
793
  end
673
794
 
795
+ it 'sends this session conversation as LLM messages without a recall-cue regex' do
796
+ src = File.read(described_class.method(:run).source_location.first)
797
+ expect(src).to match(/to_llm_messages|session_chat_history/)
798
+ expect(src).not_to match(/VAGUE_MEMORY_RX.*?to_llm_messages/m)
799
+ sess_tmp = Dir.mktmpdir('pwn-sess-hist')
800
+ stub_const('PWN::Sessions::SESSIONS_DIR', sess_tmp)
801
+ sid = 'hist_run_spec'
802
+ PWN::Sessions.create(id: sid, title: 'hist-run')
803
+ PWN::Sessions.append(session_id: sid, role: 'user', content: 'where is OpenGoal implemented?')
804
+ PWN::Sessions.append(session_id: sid, role: 'assistant', content: 'lib/pwn/ai/agent/open_goal.rb')
805
+ seen = []
806
+ allow(described_class).to receive(:should_auto_introspect?).and_return(false)
807
+ allow(described_class).to receive(:plan_first).and_return(nil)
808
+ allow(PWN::AI::Agent::TaskSummarizer).to receive(:enabled?).and_return(false) if defined?(PWN::AI::Agent::TaskSummarizer)
809
+ allow(described_class).to receive(:call_engine) do |opts|
810
+ seen = Array(opts[:messages])
811
+ { role: 'assistant', content: 'in loop.rb Loop.run', tool_calls: [] }
812
+ end
813
+ out = described_class.run(
814
+ request: 'where is this logic implemented in /opt/pwn when using pwn-ai',
815
+ session_id: sid,
816
+ system_role_content: 'test system'
817
+ )
818
+ expect(out).to include('in loop.rb Loop.run')
819
+ expect(seen.map { |m| m[:content].to_s }).to include('where is OpenGoal implemented?')
820
+ expect(seen.map { |m| m[:content].to_s }).to include('lib/pwn/ai/agent/open_goal.rb')
821
+ expect(seen.map { |m| m[:content].to_s }).to include('where is this logic implemented in /opt/pwn when using pwn-ai')
822
+ ensure
823
+ FileUtils.rm_rf(sess_tmp) if sess_tmp
824
+ end
825
+
674
826
  it 'classifies pure prior-turn recall and vague memory cues as :recall' do
675
827
  expect(described_class.request_intent(request: 'what did I just say?')).to eq(:recall)
676
828
  expect(described_class.request_intent(request: 'What did I just say')).to eq(:recall)
@@ -720,14 +872,14 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
720
872
  it 'answer_howto does not register shell tool use in source path' do
721
873
  src = File.read(described_class.method(:run).source_location.first)
722
874
  # how-to return happens before task_summary_plan!
723
- expect(src).to match(/if intent == :howto.*?return answer_howto/m)
875
+ expect(src).to match(/if intent == :howto.*?txt = answer_howto/m)
724
876
  end
725
877
 
726
878
  it 'run short-circuits pure recall via answer_recall before plan_first' do
727
879
  src = File.read(described_class.method(:run).source_location.first)
728
880
  expect(src).to match(/RECALL_RX/)
729
881
  expect(src).to match(/answer_recall/)
730
- expect(src).to match(/intent == :recall.*?return answer_recall/m)
882
+ expect(src).to match(/intent == :recall.*?txt = answer_recall/m)
731
883
  expect(src).to match(/skip_plan/)
732
884
  expect(src).not_to match(/request_kind/)
733
885
  end
@@ -736,7 +888,7 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
736
888
  src = File.read(described_class.method(:run).source_location.first)
737
889
  expect(src).to match(/GREETING_RX/)
738
890
  expect(src).to match(/answer_greeting/)
739
- expect(src).to match(/intent == :greeting.*?return answer_greeting/m)
891
+ expect(src).to match(/intent == :greeting.*?txt = answer_greeting/m)
740
892
  expect(src).to match(/skip_plan/)
741
893
  expect(src).not_to match(/request_kind/)
742
894
  end
@@ -1013,6 +1165,14 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
1013
1165
  it 'streams request-processing progress through PWN::Plugins::Log when debug is on' do
1014
1166
  src = File.read(described_class.method(:run).source_location.first)
1015
1167
  expect(src).to match(/PWN::Plugins::Log\.start_debug/)
1168
+ expect(src).to match(/next_request_log!/)
1169
+ expect(src).to match(/finish_request_log!/)
1170
+ expect(src).to match(/skip_roll|user_path\?/)
1171
+ expect(src).to match(/bounce /)
1172
+ expect(src).to match(/plan_first=/)
1173
+ expect(src).to match(/final text/)
1174
+ expect(src).to match(/debug_tools_line|tools=.*\[/)
1175
+ expect(src).to match(/debug_msgs_line|msgs=.*\[/)
1016
1176
  expect(src).to match(/PWN::Plugins::Log\.progress/)
1017
1177
  expect(src).to match(/quiet_debug_tui!/)
1018
1178
  expect(described_class).to respond_to(:debug_on?)
@@ -73,6 +73,9 @@ describe PWN::AI::Agent::PromptBuilder do
73
73
  expect(src).to match(/learning_block/)
74
74
  expect(src).to match(/KNOWN MISTAKES|mistakes_block/)
75
75
  expect(src).to match(/skills_recall/)
76
+ expect(src).to match(/default_skill_names|SKILLS CATALOG/)
77
+ expect(src).to match(/HOST LOAD/)
78
+ expect(src).to match(/host_load|deadline_s/)
76
79
  end
77
80
  end
78
81
  end
@@ -16,7 +16,7 @@ describe PWN::AI::Agent::Registry do
16
16
  it 'CORE_TOOLS is recall-then-act including skills_recall before pwn_eval' do
17
17
  expect(described_class.const_defined?(:ACT_PREFERENCE)).to eq false
18
18
  expect(described_class::CORE_TOOLS).to eq(
19
- %w[memory_recall session_recall skills_recall pwn_eval shell mistakes_record mistakes_resolve learning_note_outcome memory_remember]
19
+ %w[memory_recall session_recall skills_recall pwn_eval shell mistakes_record mistakes_resolve learning_note_outcome memory_remember skills_update]
20
20
  )
21
21
  expect(described_class::DEFAULT_PREFERENCE).to eq(described_class::CORE_TOOLS)
22
22
  expect(described_class::DEFAULT_PREFERENCE).not_to include('sessions_view')
@@ -68,6 +68,19 @@ describe PWN::AI::Agent::Reward do
68
68
  expect(r[:benign]).to be false
69
69
  end
70
70
 
71
+ it 'treats handler timeout after Ns as not semantic_ok even when success is true' do
72
+ raw = '{"success":true,"result":{"stdout":"","error":"timeout after 20s","exit":null}}'
73
+ r = described_class.semantic_ok(name: 'pwn_eval', raw: raw)
74
+ expect(r[:semantic_ok]).to be false
75
+ expect(r[:shape]).to eq(:timeout)
76
+ r2 = described_class.semantic_ok(
77
+ name: 'shell',
78
+ raw: '{"success":true,"result":{"stdout":"","stderr":"","exit":null,"error":"timeout after 30s"}}'
79
+ )
80
+ expect(r2[:semantic_ok]).to be false
81
+ expect(r2[:shape]).to eq(:timeout)
82
+ end
83
+
71
84
  it 'recoverable_shape classifies common failure classes (2.2)' do
72
85
  expect(described_class.recoverable_shape(exit_code: 127, err: 'nmap: command not found')).to eq :exit127
73
86
  expect(described_class.recoverable_shape(err: 'No such file or directory')).to eq :enoent
@@ -31,4 +31,10 @@ describe PWN::AI::Agent::Swarm do
31
31
  ensure
32
32
  Thread.current[:pwn_swarm_depth] = nil
33
33
  end
34
+
35
+ it 'text_only / empty toolsets strip tools so a reviewer cannot open browsers' do
36
+ src = File.read(described_class.method(:ask).source_location.first)
37
+ expect(src).to match(/text_only/)
38
+ expect(src).to match(/core_only:\s*empty_tools/)
39
+ end
34
40
  end
@@ -104,6 +104,42 @@ describe PWN::AI::Agent::TaskSummarizer do
104
104
  expect(line).not_to match(/about to run shell:/i)
105
105
  end
106
106
 
107
+ it 'does not label a TransparentBrowser navigate as apply code/host changes' do
108
+ line = described_class.about_to(
109
+ tools: [
110
+ {
111
+ name: 'pwn_eval',
112
+ args: {
113
+ 'code' => "$browser_obj = PWN::Plugins::TransparentBrowser.open(browser_type: :chrome); $browser.goto('https://example.com')"
114
+ }
115
+ }
116
+ ],
117
+ state: state,
118
+ request: 'Navigate to https://example.com and list blog URLs'
119
+ )
120
+ expect(line).to be_a(String)
121
+ expect(line).not_to match(%r{apply code/host changes}i)
122
+ expect(line).to match(/navigate|collect|browse|page/i)
123
+ end
124
+
125
+ it 'advances off a Navigate task after one successful browse eval' do
126
+ st = described_class.fresh(request: 'Navigate to https://example.com and list posts')
127
+ st[:plan] = [
128
+ 'Understand the request',
129
+ 'Navigate to the example.com website',
130
+ 'Collect every blog post URL',
131
+ 'Close the browser'
132
+ ]
133
+ st[:plan_idx] = 1
134
+ described_class.record!(
135
+ state: st,
136
+ name: 'pwn_eval',
137
+ args: { 'code' => "browser.goto('https://example.com')" },
138
+ result: '{"success":true,"result":{"value":"{url: \\"https://example.com/\\"}"},"effect":"browse"}'
139
+ )
140
+ expect(st[:plan_idx]).to be > 1
141
+ end
142
+
107
143
  it 'keeps single-tool legacy about_to(name:, args:) as a high-level brief' do
108
144
  line = described_class.about_to(
109
145
  name: 'shell',
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'spec_helper'
4
+ require 'tmpdir'
4
5
 
5
6
  describe PWN::AI::Agent::ToolGuard do
6
7
  it 'should display information for authors' do
@@ -58,4 +59,49 @@ describe PWN::AI::Agent::ToolGuard do
58
59
  expect(out[:stderr]).to include('missing required command')
59
60
  end
60
61
  end
62
+
63
+ describe '.host_load / .deadline_s' do
64
+ it 'reports load1, ncpu, and mem_avail_mb' do
65
+ snap = described_class.host_load
66
+ expect(snap[:ncpu].to_i).to be >= 1
67
+ expect(snap[:load1]).to be_a(Numeric)
68
+ expect(snap[:mem_avail_mb].to_i).to be >= 0
69
+ end
70
+
71
+ it 'clamps an explicit timeout and derives a default when omitted' do
72
+ expect(described_class.deadline_s(timeout: 1, kind: :eval)).to eq(1)
73
+ expect(described_class.deadline_s(timeout: 9_999, kind: :eval)).to eq(90)
74
+ expect(described_class.deadline_s(timeout: 9_999, kind: :shell)).to eq(180)
75
+ omitted = described_class.deadline_s(kind: :eval)
76
+ expect(omitted).to be_between(8, 90)
77
+ end
78
+ end
79
+
80
+ describe '.timeout_lesson' do
81
+ it 'prefers reconstruct-the-payload before raising timeout' do
82
+ tmp = Dir.mktmpdir
83
+ stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
84
+ first = described_class.timeout_lesson(tool: 'pwn_eval', payload: 'sleep 3', timeout: 1)
85
+ expect(first[:scenario]).to eq(:construction)
86
+ expect(first[:hint]).to match(/reconstruct|generated differently|improperly/i)
87
+ expect(first[:hint]).to match(/do not first raise timeout/i)
88
+ expect(first[:error]).to match(/reconstruct/)
89
+ end
90
+
91
+ it 'switches to deadline-too-short only after a prior timeout mistake' do
92
+ tmp = Dir.mktmpdir
93
+ stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
94
+ PWN::AI::Agent::Mistakes.record(
95
+ tool: 'shell',
96
+ error: 'shell timeout: reconstruct payload to same goal before raising timeout',
97
+ args: 'sleep 3',
98
+ source: :tool,
99
+ shape: :timeout
100
+ )
101
+ second = described_class.timeout_lesson(tool: 'shell', payload: 'sleep 3', timeout: 1)
102
+ expect(second[:scenario]).to eq(:deadline)
103
+ expect(second[:hint]).to match(/timeout was too short|deadline too short|raise a conservative/i)
104
+ expect(second[:error]).to match(/too short/)
105
+ end
106
+ end
61
107
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'spec_helper'
4
4
  require 'json'
5
+ require 'tmpdir'
5
6
 
6
7
  describe 'PWN::AI::Agent::Tools ruby_eval' do
7
8
  before(:all) do
@@ -29,6 +30,29 @@ describe 'PWN::AI::Agent::Tools ruby_eval' do
29
30
  expect(result[:value]).to eq('2')
30
31
  end
31
32
 
33
+ it 'keeps locals across pwn_eval calls so a browser_obj can be reused' do
34
+ entry = PWN::AI::Agent::Registry.lookup(name: 'pwn_eval')
35
+ first = entry.handler.call(code: 'pwn_eval_persist_probe = 41')
36
+ expect(first[:error]).to be_nil
37
+ second = entry.handler.call(code: 'pwn_eval_persist_probe + 1')
38
+ expect(second[:error]).to be_nil
39
+ expect(second[:value]).to eq('42')
40
+ end
41
+
42
+ it 'enforces a timeout on pwn_eval and reports timeout after Ns' do
43
+ tmp = Dir.mktmpdir
44
+ stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
45
+ entry = PWN::AI::Agent::Registry.lookup(name: 'pwn_eval')
46
+ schema = entry.schema
47
+ expect(schema.dig(:parameters, :properties, :timeout)).not_to be_nil
48
+ t0 = Time.now
49
+ result = entry.handler.call(code: 'sleep 3', timeout: 1)
50
+ expect(Time.now - t0).to be < 2.5
51
+ expect(result[:error].to_s).to match(/timeout after 1s/)
52
+ expect(result[:hint].to_s).to match(/reconstruct|generated differently|improperly/i)
53
+ expect(result[:scenario].to_s).to eq('construction')
54
+ end
55
+
32
56
  it 'Dispatch.call JSON-wraps syntax errors instead of raising' do
33
57
  out = PWN::AI::Agent::Dispatch.call(
34
58
  tool_call: {
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'spec_helper'
4
+ require 'tmpdir'
4
5
 
5
6
  describe 'PWN::AI::Agent::Tools shell' do
6
7
  before(:all) do
@@ -58,4 +59,13 @@ describe 'PWN::AI::Agent::Tools shell' do
58
59
  expect(r[:error]).not_to eq('unauthorized_recon_blocked')
59
60
  expect(r[:exit]).to eq(0)
60
61
  end
62
+
63
+ it 'on timeout prefers reconstruct-the-command before raising timeout' do
64
+ tmp = Dir.mktmpdir
65
+ stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
66
+ r = handler.call(command: 'sleep 3', timeout: 1)
67
+ expect(r[:error].to_s).to match(/timeout after 1s/)
68
+ expect(r[:hint].to_s).to match(/reconstruct|generated differently|improperly/i)
69
+ expect(r[:scenario].to_s).to eq('construction')
70
+ end
61
71
  end
@@ -14,6 +14,10 @@ describe 'PWN::AI::Agent::Tools skills' do
14
14
  expect(PWN::AI::Agent::Registry.lookup(name: 'skills_recall')).not_to be_nil
15
15
  end
16
16
 
17
+ it 'registers the skills_update tool' do
18
+ expect(PWN::AI::Agent::Registry.lookup(name: 'skills_update')).not_to be_nil
19
+ end
20
+
17
21
  it 'registers the skill_migrate_legacy tool' do
18
22
  expect(PWN::AI::Agent::Registry.lookup(name: 'skill_migrate_legacy')).not_to be_nil
19
23
  end
@@ -37,6 +41,7 @@ describe 'PWN::AI::Agent::Tools skills' do
37
41
  let(:list) { PWN::AI::Agent::Registry.lookup(name: 'skill_list')[:handler] }
38
42
  let(:view) { PWN::AI::Agent::Registry.lookup(name: 'skill_view')[:handler] }
39
43
  let(:recall) { PWN::AI::Agent::Registry.lookup(name: 'skills_recall')[:handler] }
44
+ let(:update) { PWN::AI::Agent::Registry.lookup(name: 'skills_update')[:handler] }
40
45
 
41
46
  it 'skills_recall ranks matching skills by query and skips non-matches' do
42
47
  create.call(name: 'browser-dump', description: 'Dump links via TransparentBrowser.', content: "open once, dump_links, close once\n")
@@ -49,6 +54,30 @@ describe 'PWN::AI::Agent::Tools skills' do
49
54
  expect(names).not_to include('unrelated')
50
55
  end
51
56
 
57
+ it 'skills_recall with no query returns the bundled catalog, not a mixed dump' do
58
+ create.call(name: 'osint', description: 'Drive extro_osint.', content: "osint body\n")
59
+ create.call(name: 'third-party-dump', description: 'Not the pwn-ai catalog.', content: "dump\n")
60
+ hits = recall.call(query: '', limit: 50)
61
+ expect(hits).to be_a(Hash)
62
+ names = Array(hits[:catalog]).map { |h| h[:name].to_s }
63
+ expect(names).to include('osint')
64
+ expect(names).not_to include('third-party-dump')
65
+ expect(hits[:other_installed].to_i).to be >= 1
66
+ end
67
+
68
+ it 'skills_update folds an RL lesson into an existing skill without creating one' do
69
+ create.call(name: 'osint', description: 'Drive extro_osint.', content: "# OSINT\nCollect from public sources.\n")
70
+ out = update.call(name: 'osint', lesson: 'Prefer sitemap.xml over infinite scroll for Wix blogs.')
71
+ expect(out[:updated]).to eq true
72
+ body = File.read(out[:path])
73
+ expect(body).to include('## RL feedback')
74
+ expect(body).to include('Prefer sitemap.xml over infinite scroll')
75
+ expect(body).to include('Collect from public sources')
76
+ again = update.call(name: 'osint', lesson: 'Prefer sitemap.xml over infinite scroll for Wix blogs.')
77
+ expect(again[:updated]).to eq false
78
+ expect(again[:reason]).to eq('already folded')
79
+ end
80
+
52
81
  it 'skill_create writes <name>/SKILL.md with required name+description frontmatter' do
53
82
  out = create.call(name: 'Recon Quick_Scan', description: 'Fast host triage.', content: "# Recon\nnmap -T4 -F {t}\n", references: ['T1046'])
54
83
  expect(out[:saved]).to be true
@@ -45,4 +45,26 @@ describe PWN::AI::Grok do
45
45
  expect(src).to include('PWN::Plugins::TTYSpinner.stop(spin: spin)')
46
46
  expect(src).not_to match(/spin\.stop if spinner/)
47
47
  end
48
+
49
+ it 'does not nil+ on 429 without Retry-After and does not retry 429 forever' do
50
+ src = File.read(described_class.method(:chat_with_tools).source_location.first)
51
+ expect(src).not_to match(/headers\[:retry_after\]&.to_i\s*\|\|=/)
52
+ expect(src).to match(/HttpRetry\.retry_after_s|headers\[:retry_after\]\.to_i/)
53
+ expect(src).to match(/HttpRetry\.max_attempts|max_attempts/)
54
+ end
55
+
56
+ it 'defaults chat REST timeout to 180s and retries ReadTimeout up to 5 times' do
57
+ src = File.read(described_class.method(:chat_with_tools).source_location.first)
58
+ expect(src).to match(/HttpRetry|timeout \|\|= 180/)
59
+ expect(src).not_to match(/timeout \|\|= 900/)
60
+ expect(src).to match(/HttpRetry\.max_attempts|max_attempts/)
61
+ inner = src[/retry_count = 0.*?rescue RestClient::Exceptions::Timeout/m]
62
+ expect(inner).to include('TooManyRequests')
63
+ expect(inner).to include('Exceptions::Timeout')
64
+ end
65
+
66
+ it 'tees grok timeout/429 warnings into the debug request log' do
67
+ src = File.read(described_class.method(:chat_with_tools).source_location.first)
68
+ expect(src).to match(/HttpRetry\.report_event|Log\.progress/)
69
+ end
48
70
  end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+ require 'stringio'
5
+
6
+ describe PWN::AI::HttpRetry do
7
+ it 'defaults timeout to 180 and max_attempts to 5' do
8
+ expect(described_class.timeout_s).to eq(180)
9
+ expect(described_class.timeout_s(timeout: nil)).to eq(180)
10
+ expect(described_class.timeout_s(timeout: 60)).to eq(60)
11
+ expect(described_class.max_attempts).to eq(5)
12
+ expect(described_class.max_attempts(quiet: true)).to eq(1)
13
+ expect(described_class.max_attempts(timeout: 12)).to eq(1)
14
+ expect(described_class.max_attempts(max_attempts: 3)).to eq(3)
15
+ end
16
+
17
+ it 'tees emit lines into the open debug request log' do
18
+ path = PWN::Plugins::Log.start_debug(
19
+ tee: StringIO.new,
20
+ path: "/tmp/pwn-ai-DEBUG-#{Process.pid}-httpretry.log"
21
+ )
22
+ err = RestClient::Exceptions::ReadTimeout.new('Timed out reading data from server')
23
+ described_class.report_event(
24
+ label: 'openai',
25
+ error: err,
26
+ http_method: :post,
27
+ rest_call: 'chat/completions',
28
+ extra: 'timeout=180s attempt=1/5',
29
+ quiet: true
30
+ )
31
+ body = File.read(path)
32
+ expect(body).to include('[pwn-ai/openai]')
33
+ expect(body).to include('Timed out reading data from server')
34
+ expect(body).to include('timeout=180s')
35
+ ensure
36
+ PWN::Plugins::Log.stop_debug if defined?(PWN::Plugins::Log)
37
+ end
38
+
39
+ {
40
+ 'PWN::AI::Grok' => '/opt/pwn/lib/pwn/ai/grok.rb',
41
+ 'PWN::AI::OpenAI' => '/opt/pwn/lib/pwn/ai/open_ai.rb',
42
+ 'PWN::AI::Anthropic' => '/opt/pwn/lib/pwn/ai/anthropic.rb',
43
+ 'PWN::AI::Gemini' => '/opt/pwn/lib/pwn/ai/gemini.rb',
44
+ 'PWN::AI::Ollama' => '/opt/pwn/lib/pwn/ai/ollama.rb',
45
+ 'PWN::AI::OpenWebUI' => '/opt/pwn/lib/pwn/ai/open_web_ui.rb'
46
+ }.each do |name, path|
47
+ it "#{name} uses HttpRetry 180s / max_attempts and retries ReadTimeout" do
48
+ src = File.read(path)
49
+ expect(src).to include('HttpRetry')
50
+ expect(src).not_to match(/timeout \|\|= 900/)
51
+ expect(src).to match(/Exceptions::Timeout/)
52
+ expect(src).to match(/HttpRetry\.report_event/)
53
+ end
54
+ end
55
+ end