pwn 0.5.686 → 0.5.688
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.ruby-version +1 -1
- data/Gemfile +3 -3
- data/README.md +3 -4
- data/documentation/Agent-Tool-Registry.md +2 -2
- data/documentation/Configuration.md +5 -4
- data/documentation/General-PWN-Usage.md +1 -1
- data/documentation/Home.md +1 -1
- data/documentation/How-PWN-Works.md +1 -1
- data/documentation/Mistakes.md +1 -1
- data/documentation/Reinforcement-Learning.md +4 -6
- data/documentation/Troubleshooting.md +1 -1
- data/documentation/pwn-REPL.md +1 -1
- data/documentation/pwn-ai-Agent.md +2 -3
- data/lib/pwn/ai/agent/curriculum.rb +2 -2
- data/lib/pwn/ai/agent/dispatch.rb +1 -1
- data/lib/pwn/ai/agent/learning.rb +150 -32
- data/lib/pwn/ai/agent/loop.rb +291 -129
- data/lib/pwn/ai/agent/prompt_builder.rb +47 -29
- data/lib/pwn/ai/agent/registry.rb +2 -1
- data/lib/pwn/ai/agent/reward.rb +9 -0
- data/lib/pwn/ai/agent/swarm.rb +3 -1
- data/lib/pwn/ai/agent/task_summarizer.rb +23 -6
- data/lib/pwn/ai/agent/tool_guard.rb +109 -0
- data/lib/pwn/ai/agent/tools/ruby_eval.rb +44 -20
- data/lib/pwn/ai/agent/tools/shell.rb +17 -4
- data/lib/pwn/ai/agent/tools/skills.rb +63 -2
- data/lib/pwn/ai/anthropic.rb +25 -5
- data/lib/pwn/ai/gemini.rb +25 -5
- data/lib/pwn/ai/grok.rb +30 -13
- data/lib/pwn/ai/http_retry.rb +78 -0
- data/lib/pwn/ai/ollama.rb +25 -5
- data/lib/pwn/ai/open_ai.rb +26 -13
- data/lib/pwn/ai/open_web_ui.rb +25 -5
- data/lib/pwn/ai.rb +1 -0
- data/lib/pwn/config.rb +1 -1
- data/lib/pwn/plugins/log.rb +183 -41
- data/lib/pwn/plugins/repl.rb +50 -15
- data/lib/pwn/plugins/transparent_browser.rb +16 -2
- data/lib/pwn/sessions.rb +50 -1
- data/lib/pwn/version.rb +1 -1
- data/spec/integration/reinforced_feedback_loop_spec.rb +9 -11
- data/spec/lib/pwn/ai/agent/loop_spec.rb +180 -20
- data/spec/lib/pwn/ai/agent/prompt_builder_spec.rb +3 -0
- data/spec/lib/pwn/ai/agent/registry_spec.rb +1 -1
- data/spec/lib/pwn/ai/agent/reward_spec.rb +13 -0
- data/spec/lib/pwn/ai/agent/swarm_spec.rb +6 -0
- data/spec/lib/pwn/ai/agent/task_summarizer_spec.rb +36 -0
- data/spec/lib/pwn/ai/agent/tool_guard_spec.rb +46 -0
- data/spec/lib/pwn/ai/agent/tools/ruby_eval_spec.rb +24 -0
- data/spec/lib/pwn/ai/agent/tools/shell_spec.rb +10 -0
- data/spec/lib/pwn/ai/agent/tools/skills_spec.rb +29 -0
- data/spec/lib/pwn/ai/grok_spec.rb +22 -0
- data/spec/lib/pwn/ai/http_retry_spec.rb +55 -0
- data/spec/lib/pwn/plugins/log_spec.rb +170 -19
- data/spec/lib/pwn/plugins/repl_pwn_vault_spec.rb +14 -4
- data/spec/lib/pwn/plugins/transparent_browser_spec.rb +8 -0
- data/spec/lib/pwn/sessions_spec.rb +24 -0
- data/third_party/pwn_rdoc.jsonl +41 -5
- data/tmp_pwn_critic_skills_flaw.txt +9 -0
- metadata +11 -8
|
@@ -16,22 +16,21 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
describe 'RL-adjacent loop contracts' do # rubocop:disable Metrics/BlockLength
|
|
19
|
-
it '
|
|
19
|
+
it 'spins on engine HTTP wait and still dispatches on_tool (debug must not hide the TUI)' do
|
|
20
20
|
src = File.read(described_class.method(:run).source_location.first)
|
|
21
|
-
expect(src).
|
|
22
|
-
expect(src).to match(/
|
|
21
|
+
expect(src).to match(/spinner:\s*true/)
|
|
22
|
+
expect(src).to match(/on_tool&?\.call/)
|
|
23
|
+
expect(src).not_to match(/on_tool\s*=\s*nil/)
|
|
23
24
|
end
|
|
24
25
|
|
|
25
|
-
it '
|
|
26
|
+
it 'does not treat a round cap as a finished answer' do
|
|
26
27
|
src = File.read(described_class.method(:run).source_location.first)
|
|
27
|
-
|
|
28
|
-
expect(src).
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
expect(
|
|
32
|
-
expect(
|
|
33
|
-
# flush before return of final_msg
|
|
34
|
-
expect(exhaust).to match(/task_summary_flush!.*final_msg/m)
|
|
28
|
+
expect(src).not_to match(/iteration budget exhausted/)
|
|
29
|
+
expect(src).not_to match(/final budget_exhausted/)
|
|
30
|
+
expect(src).not_to match(/shape: :budget_exhausted/)
|
|
31
|
+
expect(src).not_to match(/max_iters\.times/)
|
|
32
|
+
expect(src).to match(/may_finalize\?/)
|
|
33
|
+
expect(src).to match(/loop do/)
|
|
35
34
|
end
|
|
36
35
|
|
|
37
36
|
it 'plan_first unifies ts_state plan after red_team (P2)' do
|
|
@@ -359,6 +358,128 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
359
358
|
expect(described_class.needs_host_work?(request: 'Write hello into /tmp/x.txt')).to eq true
|
|
360
359
|
end
|
|
361
360
|
|
|
361
|
+
it 'skips the implement compass on a browse/navigate request' do
|
|
362
|
+
req = 'Navigate to https://0dayinc.com using TransparentBrowser.open and list blog URLs'
|
|
363
|
+
expect(described_class.send(:request_need, request: req)).to eq :browse
|
|
364
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
365
|
+
expect(src).to match(/skip_compass = .*browse/)
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
it 'does not bounce a text-only turn for missing tool evidence' do
|
|
369
|
+
req = 'Navigate to https://0dayinc.com using TransparentBrowser.open and list blog URLs'
|
|
370
|
+
expect(described_class.needs_host_work?(request: req)).to eq true
|
|
371
|
+
expect(
|
|
372
|
+
described_class.send(
|
|
373
|
+
:request_unsatisfied?,
|
|
374
|
+
request: req,
|
|
375
|
+
messages: [{ role: 'assistant', content: 'SOUND — step 1 is the likely fail.', tool_calls: [] }]
|
|
376
|
+
)
|
|
377
|
+
).to eq true
|
|
378
|
+
Thread.current[:pwn_loop_no_tools] = true
|
|
379
|
+
expect(
|
|
380
|
+
described_class.send(
|
|
381
|
+
:request_unsatisfied?,
|
|
382
|
+
request: req,
|
|
383
|
+
messages: [{ role: 'assistant', content: 'SOUND — step 1 is the likely fail.', tool_calls: [] }]
|
|
384
|
+
)
|
|
385
|
+
).to eq false
|
|
386
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
387
|
+
expect(src).to match(/pwn_loop_no_tools/)
|
|
388
|
+
expect(src).to match(/skip_compass = .*no_tools/)
|
|
389
|
+
ensure
|
|
390
|
+
Thread.current[:pwn_loop_no_tools] = nil
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
it 'does not start post-answer review from a nested or tool-less Loop.run' do
|
|
394
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
395
|
+
expect(src).to match(/!nested && !no_tools && should_auto_introspect\?/)
|
|
396
|
+
expect(
|
|
397
|
+
described_class.send(
|
|
398
|
+
:should_auto_introspect?,
|
|
399
|
+
local: false,
|
|
400
|
+
turn_fails: {},
|
|
401
|
+
iter: 1
|
|
402
|
+
)
|
|
403
|
+
).to eq true
|
|
404
|
+
Thread.current[:pwn_loop_no_tools] = true
|
|
405
|
+
expect(
|
|
406
|
+
described_class.send(
|
|
407
|
+
:should_auto_introspect?,
|
|
408
|
+
local: false,
|
|
409
|
+
turn_fails: {},
|
|
410
|
+
iter: 1
|
|
411
|
+
)
|
|
412
|
+
).to eq false
|
|
413
|
+
ensure
|
|
414
|
+
Thread.current[:pwn_loop_no_tools] = nil
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
it 'writes full tool request/result to the debug log and timestamps Interrupt' do
|
|
418
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
419
|
+
expect(src).to match(/debug_tool_io!/)
|
|
420
|
+
expect(src).to match(/tool #\{name\} start/)
|
|
421
|
+
expect(src).to match(/rescue Interrupt/)
|
|
422
|
+
expect(src).to match(/note_interrupt!/)
|
|
423
|
+
expect(src).to match(/pwn_log_progress/)
|
|
424
|
+
expect(src).to match(/note_exception!/)
|
|
425
|
+
expect(src).to match(/rescue StandardError/)
|
|
426
|
+
expect(src).to include("debug_progress(msg: 'engine returned no message')")
|
|
427
|
+
expect(src).not_to match(/dispatch #\{name\} ok_len=/)
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
it 'records a reconstruct-first timeout mistake instead of treating success:true as ok' do
|
|
431
|
+
tmp = Dir.mktmpdir
|
|
432
|
+
stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
|
|
433
|
+
raw = JSON.generate(
|
|
434
|
+
success: true,
|
|
435
|
+
result: { stdout: '', error: 'timeout after 20s', hint: 'x', scenario: 'construction' }
|
|
436
|
+
)
|
|
437
|
+
tele = described_class.send(
|
|
438
|
+
:record_metrics,
|
|
439
|
+
name: 'pwn_eval',
|
|
440
|
+
started: Time.now,
|
|
441
|
+
raw: raw,
|
|
442
|
+
args: '{"code":"sleep 3"}'
|
|
443
|
+
)
|
|
444
|
+
expect(tele[:ok]).to eq false
|
|
445
|
+
expect(tele[:mistake]).to be_a(Hash)
|
|
446
|
+
expect(tele[:mistake][:error].to_s).to match(/reconstruct/)
|
|
447
|
+
expect(tele[:mistake][:shape].to_s).to eq('timeout')
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
it 'treats a skills-catalog ask as recall, not trivia or host-work' do
|
|
451
|
+
req = 'what skills are available?'
|
|
452
|
+
expect(described_class.catalog_lookup?(request: req)).to eq true
|
|
453
|
+
expect(described_class.world_knowledge?(request: req)).to eq false
|
|
454
|
+
expect(described_class.needs_host_work?(request: req)).to eq false
|
|
455
|
+
expect(described_class.send(:request_need, request: req)).to eq :read
|
|
456
|
+
expect(
|
|
457
|
+
described_class.send(
|
|
458
|
+
:request_unsatisfied?,
|
|
459
|
+
request: req,
|
|
460
|
+
messages: [{ role: 'assistant', content: '103 skills…', tool_calls: [] }]
|
|
461
|
+
)
|
|
462
|
+
).to eq true
|
|
463
|
+
expect(
|
|
464
|
+
described_class.send(
|
|
465
|
+
:request_unsatisfied?,
|
|
466
|
+
request: req,
|
|
467
|
+
messages: [
|
|
468
|
+
{
|
|
469
|
+
role: 'assistant',
|
|
470
|
+
content: '',
|
|
471
|
+
tool_calls: [
|
|
472
|
+
{ function: { name: 'skills_recall', arguments: '{}' } }
|
|
473
|
+
]
|
|
474
|
+
}
|
|
475
|
+
]
|
|
476
|
+
)
|
|
477
|
+
).to eq false
|
|
478
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
479
|
+
expect(src).to match(/skip_plan.*catalog/)
|
|
480
|
+
expect(src).to match(/force_plan && cal_state\[:cal\] && !skip_plan/)
|
|
481
|
+
end
|
|
482
|
+
|
|
362
483
|
it 'does not treat rspec/rubocop green as a finished write/docs ask' do
|
|
363
484
|
req = 'Update all markdown files to reflect operational changes and rebuild diagrams'
|
|
364
485
|
verified = [
|
|
@@ -415,7 +536,7 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
415
536
|
expect(src).not_to match(/blob\.match\?\(MUTATION_EVIDENCE_RX\)/)
|
|
416
537
|
end
|
|
417
538
|
|
|
418
|
-
it 'keeps tools
|
|
539
|
+
it 'keeps tools for any unfinished host-work request — no last-iter strip' do
|
|
419
540
|
req = 'Update all markdown files and rebuild diagrams'
|
|
420
541
|
verified = [
|
|
421
542
|
{ role: 'tool', name: 'shell', content: "12 examples, 0 failures\n" }
|
|
@@ -429,8 +550,8 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
429
550
|
)
|
|
430
551
|
).to eq true
|
|
431
552
|
src = File.read(described_class.method(:run).source_location.first)
|
|
432
|
-
expect(src).
|
|
433
|
-
expect(src).to match(/
|
|
553
|
+
expect(src).not_to match(/last_iter = want_last && !still_open/)
|
|
554
|
+
expect(src).to match(/call_engine\(messages: messages, tools: tools/)
|
|
434
555
|
end
|
|
435
556
|
|
|
436
557
|
it 'never accepts a text final while the original host-work request is unsatisfied' do
|
|
@@ -546,9 +667,9 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
546
667
|
).to eq false
|
|
547
668
|
end
|
|
548
669
|
|
|
549
|
-
it '
|
|
670
|
+
it 'does not strip tools on a last slot; leftover English tasks do not force required' do
|
|
550
671
|
src = File.read(described_class.method(:run).source_location.first)
|
|
551
|
-
expect(src).
|
|
672
|
+
expect(src).not_to match(/text_only_iters = 1/)
|
|
552
673
|
expect(src).not_to match(/english_open && hot/)
|
|
553
674
|
expect(src).to match(/has_tool_result \|\| !need_tools \? 'auto' : 'required'/)
|
|
554
675
|
expect(src).not_to match(/still_acting/)
|
|
@@ -671,6 +792,37 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
671
792
|
FileUtils.rm_rf(sess_tmp) if sess_tmp
|
|
672
793
|
end
|
|
673
794
|
|
|
795
|
+
it 'sends this session conversation as LLM messages without a recall-cue regex' do
|
|
796
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
797
|
+
expect(src).to match(/to_llm_messages|session_chat_history/)
|
|
798
|
+
expect(src).not_to match(/VAGUE_MEMORY_RX.*?to_llm_messages/m)
|
|
799
|
+
sess_tmp = Dir.mktmpdir('pwn-sess-hist')
|
|
800
|
+
stub_const('PWN::Sessions::SESSIONS_DIR', sess_tmp)
|
|
801
|
+
sid = 'hist_run_spec'
|
|
802
|
+
PWN::Sessions.create(id: sid, title: 'hist-run')
|
|
803
|
+
PWN::Sessions.append(session_id: sid, role: 'user', content: 'where is OpenGoal implemented?')
|
|
804
|
+
PWN::Sessions.append(session_id: sid, role: 'assistant', content: 'lib/pwn/ai/agent/open_goal.rb')
|
|
805
|
+
seen = []
|
|
806
|
+
allow(described_class).to receive(:should_auto_introspect?).and_return(false)
|
|
807
|
+
allow(described_class).to receive(:plan_first).and_return(nil)
|
|
808
|
+
allow(PWN::AI::Agent::TaskSummarizer).to receive(:enabled?).and_return(false) if defined?(PWN::AI::Agent::TaskSummarizer)
|
|
809
|
+
allow(described_class).to receive(:call_engine) do |opts|
|
|
810
|
+
seen = Array(opts[:messages])
|
|
811
|
+
{ role: 'assistant', content: 'in loop.rb Loop.run', tool_calls: [] }
|
|
812
|
+
end
|
|
813
|
+
out = described_class.run(
|
|
814
|
+
request: 'where is this logic implemented in /opt/pwn when using pwn-ai',
|
|
815
|
+
session_id: sid,
|
|
816
|
+
system_role_content: 'test system'
|
|
817
|
+
)
|
|
818
|
+
expect(out).to include('in loop.rb Loop.run')
|
|
819
|
+
expect(seen.map { |m| m[:content].to_s }).to include('where is OpenGoal implemented?')
|
|
820
|
+
expect(seen.map { |m| m[:content].to_s }).to include('lib/pwn/ai/agent/open_goal.rb')
|
|
821
|
+
expect(seen.map { |m| m[:content].to_s }).to include('where is this logic implemented in /opt/pwn when using pwn-ai')
|
|
822
|
+
ensure
|
|
823
|
+
FileUtils.rm_rf(sess_tmp) if sess_tmp
|
|
824
|
+
end
|
|
825
|
+
|
|
674
826
|
it 'classifies pure prior-turn recall and vague memory cues as :recall' do
|
|
675
827
|
expect(described_class.request_intent(request: 'what did I just say?')).to eq(:recall)
|
|
676
828
|
expect(described_class.request_intent(request: 'What did I just say')).to eq(:recall)
|
|
@@ -720,14 +872,14 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
720
872
|
it 'answer_howto does not register shell tool use in source path' do
|
|
721
873
|
src = File.read(described_class.method(:run).source_location.first)
|
|
722
874
|
# how-to return happens before task_summary_plan!
|
|
723
|
-
expect(src).to match(/if intent == :howto.*?
|
|
875
|
+
expect(src).to match(/if intent == :howto.*?txt = answer_howto/m)
|
|
724
876
|
end
|
|
725
877
|
|
|
726
878
|
it 'run short-circuits pure recall via answer_recall before plan_first' do
|
|
727
879
|
src = File.read(described_class.method(:run).source_location.first)
|
|
728
880
|
expect(src).to match(/RECALL_RX/)
|
|
729
881
|
expect(src).to match(/answer_recall/)
|
|
730
|
-
expect(src).to match(/intent == :recall.*?
|
|
882
|
+
expect(src).to match(/intent == :recall.*?txt = answer_recall/m)
|
|
731
883
|
expect(src).to match(/skip_plan/)
|
|
732
884
|
expect(src).not_to match(/request_kind/)
|
|
733
885
|
end
|
|
@@ -736,7 +888,7 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
736
888
|
src = File.read(described_class.method(:run).source_location.first)
|
|
737
889
|
expect(src).to match(/GREETING_RX/)
|
|
738
890
|
expect(src).to match(/answer_greeting/)
|
|
739
|
-
expect(src).to match(/intent == :greeting.*?
|
|
891
|
+
expect(src).to match(/intent == :greeting.*?txt = answer_greeting/m)
|
|
740
892
|
expect(src).to match(/skip_plan/)
|
|
741
893
|
expect(src).not_to match(/request_kind/)
|
|
742
894
|
end
|
|
@@ -1013,6 +1165,14 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
1013
1165
|
it 'streams request-processing progress through PWN::Plugins::Log when debug is on' do
|
|
1014
1166
|
src = File.read(described_class.method(:run).source_location.first)
|
|
1015
1167
|
expect(src).to match(/PWN::Plugins::Log\.start_debug/)
|
|
1168
|
+
expect(src).to match(/next_request_log!/)
|
|
1169
|
+
expect(src).to match(/finish_request_log!/)
|
|
1170
|
+
expect(src).to match(/skip_roll|user_path\?/)
|
|
1171
|
+
expect(src).to match(/bounce /)
|
|
1172
|
+
expect(src).to match(/plan_first=/)
|
|
1173
|
+
expect(src).to match(/final text/)
|
|
1174
|
+
expect(src).to match(/debug_tools_line|tools=.*\[/)
|
|
1175
|
+
expect(src).to match(/debug_msgs_line|msgs=.*\[/)
|
|
1016
1176
|
expect(src).to match(/PWN::Plugins::Log\.progress/)
|
|
1017
1177
|
expect(src).to match(/quiet_debug_tui!/)
|
|
1018
1178
|
expect(described_class).to respond_to(:debug_on?)
|
|
@@ -73,6 +73,9 @@ describe PWN::AI::Agent::PromptBuilder do
|
|
|
73
73
|
expect(src).to match(/learning_block/)
|
|
74
74
|
expect(src).to match(/KNOWN MISTAKES|mistakes_block/)
|
|
75
75
|
expect(src).to match(/skills_recall/)
|
|
76
|
+
expect(src).to match(/default_skill_names|SKILLS CATALOG/)
|
|
77
|
+
expect(src).to match(/HOST LOAD/)
|
|
78
|
+
expect(src).to match(/host_load|deadline_s/)
|
|
76
79
|
end
|
|
77
80
|
end
|
|
78
81
|
end
|
|
@@ -16,7 +16,7 @@ describe PWN::AI::Agent::Registry do
|
|
|
16
16
|
it 'CORE_TOOLS is recall-then-act including skills_recall before pwn_eval' do
|
|
17
17
|
expect(described_class.const_defined?(:ACT_PREFERENCE)).to eq false
|
|
18
18
|
expect(described_class::CORE_TOOLS).to eq(
|
|
19
|
-
%w[memory_recall session_recall skills_recall pwn_eval shell mistakes_record mistakes_resolve learning_note_outcome memory_remember]
|
|
19
|
+
%w[memory_recall session_recall skills_recall pwn_eval shell mistakes_record mistakes_resolve learning_note_outcome memory_remember skills_update]
|
|
20
20
|
)
|
|
21
21
|
expect(described_class::DEFAULT_PREFERENCE).to eq(described_class::CORE_TOOLS)
|
|
22
22
|
expect(described_class::DEFAULT_PREFERENCE).not_to include('sessions_view')
|
|
@@ -68,6 +68,19 @@ describe PWN::AI::Agent::Reward do
|
|
|
68
68
|
expect(r[:benign]).to be false
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
+
it 'treats handler timeout after Ns as not semantic_ok even when success is true' do
|
|
72
|
+
raw = '{"success":true,"result":{"stdout":"","error":"timeout after 20s","exit":null}}'
|
|
73
|
+
r = described_class.semantic_ok(name: 'pwn_eval', raw: raw)
|
|
74
|
+
expect(r[:semantic_ok]).to be false
|
|
75
|
+
expect(r[:shape]).to eq(:timeout)
|
|
76
|
+
r2 = described_class.semantic_ok(
|
|
77
|
+
name: 'shell',
|
|
78
|
+
raw: '{"success":true,"result":{"stdout":"","stderr":"","exit":null,"error":"timeout after 30s"}}'
|
|
79
|
+
)
|
|
80
|
+
expect(r2[:semantic_ok]).to be false
|
|
81
|
+
expect(r2[:shape]).to eq(:timeout)
|
|
82
|
+
end
|
|
83
|
+
|
|
71
84
|
it 'recoverable_shape classifies common failure classes (2.2)' do
|
|
72
85
|
expect(described_class.recoverable_shape(exit_code: 127, err: 'nmap: command not found')).to eq :exit127
|
|
73
86
|
expect(described_class.recoverable_shape(err: 'No such file or directory')).to eq :enoent
|
|
@@ -31,4 +31,10 @@ describe PWN::AI::Agent::Swarm do
|
|
|
31
31
|
ensure
|
|
32
32
|
Thread.current[:pwn_swarm_depth] = nil
|
|
33
33
|
end
|
|
34
|
+
|
|
35
|
+
it 'text_only / empty toolsets strip tools so a reviewer cannot open browsers' do
|
|
36
|
+
src = File.read(described_class.method(:ask).source_location.first)
|
|
37
|
+
expect(src).to match(/text_only/)
|
|
38
|
+
expect(src).to match(/core_only:\s*empty_tools/)
|
|
39
|
+
end
|
|
34
40
|
end
|
|
@@ -104,6 +104,42 @@ describe PWN::AI::Agent::TaskSummarizer do
|
|
|
104
104
|
expect(line).not_to match(/about to run shell:/i)
|
|
105
105
|
end
|
|
106
106
|
|
|
107
|
+
it 'does not label a TransparentBrowser navigate as apply code/host changes' do
|
|
108
|
+
line = described_class.about_to(
|
|
109
|
+
tools: [
|
|
110
|
+
{
|
|
111
|
+
name: 'pwn_eval',
|
|
112
|
+
args: {
|
|
113
|
+
'code' => "$browser_obj = PWN::Plugins::TransparentBrowser.open(browser_type: :chrome); $browser.goto('https://example.com')"
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
state: state,
|
|
118
|
+
request: 'Navigate to https://example.com and list blog URLs'
|
|
119
|
+
)
|
|
120
|
+
expect(line).to be_a(String)
|
|
121
|
+
expect(line).not_to match(%r{apply code/host changes}i)
|
|
122
|
+
expect(line).to match(/navigate|collect|browse|page/i)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
it 'advances off a Navigate task after one successful browse eval' do
|
|
126
|
+
st = described_class.fresh(request: 'Navigate to https://example.com and list posts')
|
|
127
|
+
st[:plan] = [
|
|
128
|
+
'Understand the request',
|
|
129
|
+
'Navigate to the example.com website',
|
|
130
|
+
'Collect every blog post URL',
|
|
131
|
+
'Close the browser'
|
|
132
|
+
]
|
|
133
|
+
st[:plan_idx] = 1
|
|
134
|
+
described_class.record!(
|
|
135
|
+
state: st,
|
|
136
|
+
name: 'pwn_eval',
|
|
137
|
+
args: { 'code' => "browser.goto('https://example.com')" },
|
|
138
|
+
result: '{"success":true,"result":{"value":"{url: \\"https://example.com/\\"}"},"effect":"browse"}'
|
|
139
|
+
)
|
|
140
|
+
expect(st[:plan_idx]).to be > 1
|
|
141
|
+
end
|
|
142
|
+
|
|
107
143
|
it 'keeps single-tool legacy about_to(name:, args:) as a high-level brief' do
|
|
108
144
|
line = described_class.about_to(
|
|
109
145
|
name: 'shell',
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'spec_helper'
|
|
4
|
+
require 'tmpdir'
|
|
4
5
|
|
|
5
6
|
describe PWN::AI::Agent::ToolGuard do
|
|
6
7
|
it 'should display information for authors' do
|
|
@@ -58,4 +59,49 @@ describe PWN::AI::Agent::ToolGuard do
|
|
|
58
59
|
expect(out[:stderr]).to include('missing required command')
|
|
59
60
|
end
|
|
60
61
|
end
|
|
62
|
+
|
|
63
|
+
describe '.host_load / .deadline_s' do
|
|
64
|
+
it 'reports load1, ncpu, and mem_avail_mb' do
|
|
65
|
+
snap = described_class.host_load
|
|
66
|
+
expect(snap[:ncpu].to_i).to be >= 1
|
|
67
|
+
expect(snap[:load1]).to be_a(Numeric)
|
|
68
|
+
expect(snap[:mem_avail_mb].to_i).to be >= 0
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
it 'clamps an explicit timeout and derives a default when omitted' do
|
|
72
|
+
expect(described_class.deadline_s(timeout: 1, kind: :eval)).to eq(1)
|
|
73
|
+
expect(described_class.deadline_s(timeout: 9_999, kind: :eval)).to eq(90)
|
|
74
|
+
expect(described_class.deadline_s(timeout: 9_999, kind: :shell)).to eq(180)
|
|
75
|
+
omitted = described_class.deadline_s(kind: :eval)
|
|
76
|
+
expect(omitted).to be_between(8, 90)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
describe '.timeout_lesson' do
|
|
81
|
+
it 'prefers reconstruct-the-payload before raising timeout' do
|
|
82
|
+
tmp = Dir.mktmpdir
|
|
83
|
+
stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
|
|
84
|
+
first = described_class.timeout_lesson(tool: 'pwn_eval', payload: 'sleep 3', timeout: 1)
|
|
85
|
+
expect(first[:scenario]).to eq(:construction)
|
|
86
|
+
expect(first[:hint]).to match(/reconstruct|generated differently|improperly/i)
|
|
87
|
+
expect(first[:hint]).to match(/do not first raise timeout/i)
|
|
88
|
+
expect(first[:error]).to match(/reconstruct/)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
it 'switches to deadline-too-short only after a prior timeout mistake' do
|
|
92
|
+
tmp = Dir.mktmpdir
|
|
93
|
+
stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
|
|
94
|
+
PWN::AI::Agent::Mistakes.record(
|
|
95
|
+
tool: 'shell',
|
|
96
|
+
error: 'shell timeout: reconstruct payload to same goal before raising timeout',
|
|
97
|
+
args: 'sleep 3',
|
|
98
|
+
source: :tool,
|
|
99
|
+
shape: :timeout
|
|
100
|
+
)
|
|
101
|
+
second = described_class.timeout_lesson(tool: 'shell', payload: 'sleep 3', timeout: 1)
|
|
102
|
+
expect(second[:scenario]).to eq(:deadline)
|
|
103
|
+
expect(second[:hint]).to match(/timeout was too short|deadline too short|raise a conservative/i)
|
|
104
|
+
expect(second[:error]).to match(/too short/)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
61
107
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'spec_helper'
|
|
4
4
|
require 'json'
|
|
5
|
+
require 'tmpdir'
|
|
5
6
|
|
|
6
7
|
describe 'PWN::AI::Agent::Tools ruby_eval' do
|
|
7
8
|
before(:all) do
|
|
@@ -29,6 +30,29 @@ describe 'PWN::AI::Agent::Tools ruby_eval' do
|
|
|
29
30
|
expect(result[:value]).to eq('2')
|
|
30
31
|
end
|
|
31
32
|
|
|
33
|
+
it 'keeps locals across pwn_eval calls so a browser_obj can be reused' do
|
|
34
|
+
entry = PWN::AI::Agent::Registry.lookup(name: 'pwn_eval')
|
|
35
|
+
first = entry.handler.call(code: 'pwn_eval_persist_probe = 41')
|
|
36
|
+
expect(first[:error]).to be_nil
|
|
37
|
+
second = entry.handler.call(code: 'pwn_eval_persist_probe + 1')
|
|
38
|
+
expect(second[:error]).to be_nil
|
|
39
|
+
expect(second[:value]).to eq('42')
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
it 'enforces a timeout on pwn_eval and reports timeout after Ns' do
|
|
43
|
+
tmp = Dir.mktmpdir
|
|
44
|
+
stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
|
|
45
|
+
entry = PWN::AI::Agent::Registry.lookup(name: 'pwn_eval')
|
|
46
|
+
schema = entry.schema
|
|
47
|
+
expect(schema.dig(:parameters, :properties, :timeout)).not_to be_nil
|
|
48
|
+
t0 = Time.now
|
|
49
|
+
result = entry.handler.call(code: 'sleep 3', timeout: 1)
|
|
50
|
+
expect(Time.now - t0).to be < 2.5
|
|
51
|
+
expect(result[:error].to_s).to match(/timeout after 1s/)
|
|
52
|
+
expect(result[:hint].to_s).to match(/reconstruct|generated differently|improperly/i)
|
|
53
|
+
expect(result[:scenario].to_s).to eq('construction')
|
|
54
|
+
end
|
|
55
|
+
|
|
32
56
|
it 'Dispatch.call JSON-wraps syntax errors instead of raising' do
|
|
33
57
|
out = PWN::AI::Agent::Dispatch.call(
|
|
34
58
|
tool_call: {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'spec_helper'
|
|
4
|
+
require 'tmpdir'
|
|
4
5
|
|
|
5
6
|
describe 'PWN::AI::Agent::Tools shell' do
|
|
6
7
|
before(:all) do
|
|
@@ -58,4 +59,13 @@ describe 'PWN::AI::Agent::Tools shell' do
|
|
|
58
59
|
expect(r[:error]).not_to eq('unauthorized_recon_blocked')
|
|
59
60
|
expect(r[:exit]).to eq(0)
|
|
60
61
|
end
|
|
62
|
+
|
|
63
|
+
it 'on timeout prefers reconstruct-the-command before raising timeout' do
|
|
64
|
+
tmp = Dir.mktmpdir
|
|
65
|
+
stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', File.join(tmp, 'mistakes.json'))
|
|
66
|
+
r = handler.call(command: 'sleep 3', timeout: 1)
|
|
67
|
+
expect(r[:error].to_s).to match(/timeout after 1s/)
|
|
68
|
+
expect(r[:hint].to_s).to match(/reconstruct|generated differently|improperly/i)
|
|
69
|
+
expect(r[:scenario].to_s).to eq('construction')
|
|
70
|
+
end
|
|
61
71
|
end
|
|
@@ -14,6 +14,10 @@ describe 'PWN::AI::Agent::Tools skills' do
|
|
|
14
14
|
expect(PWN::AI::Agent::Registry.lookup(name: 'skills_recall')).not_to be_nil
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
+
it 'registers the skills_update tool' do
|
|
18
|
+
expect(PWN::AI::Agent::Registry.lookup(name: 'skills_update')).not_to be_nil
|
|
19
|
+
end
|
|
20
|
+
|
|
17
21
|
it 'registers the skill_migrate_legacy tool' do
|
|
18
22
|
expect(PWN::AI::Agent::Registry.lookup(name: 'skill_migrate_legacy')).not_to be_nil
|
|
19
23
|
end
|
|
@@ -37,6 +41,7 @@ describe 'PWN::AI::Agent::Tools skills' do
|
|
|
37
41
|
let(:list) { PWN::AI::Agent::Registry.lookup(name: 'skill_list')[:handler] }
|
|
38
42
|
let(:view) { PWN::AI::Agent::Registry.lookup(name: 'skill_view')[:handler] }
|
|
39
43
|
let(:recall) { PWN::AI::Agent::Registry.lookup(name: 'skills_recall')[:handler] }
|
|
44
|
+
let(:update) { PWN::AI::Agent::Registry.lookup(name: 'skills_update')[:handler] }
|
|
40
45
|
|
|
41
46
|
it 'skills_recall ranks matching skills by query and skips non-matches' do
|
|
42
47
|
create.call(name: 'browser-dump', description: 'Dump links via TransparentBrowser.', content: "open once, dump_links, close once\n")
|
|
@@ -49,6 +54,30 @@ describe 'PWN::AI::Agent::Tools skills' do
|
|
|
49
54
|
expect(names).not_to include('unrelated')
|
|
50
55
|
end
|
|
51
56
|
|
|
57
|
+
it 'skills_recall with no query returns the bundled catalog, not a mixed dump' do
|
|
58
|
+
create.call(name: 'osint', description: 'Drive extro_osint.', content: "osint body\n")
|
|
59
|
+
create.call(name: 'third-party-dump', description: 'Not the pwn-ai catalog.', content: "dump\n")
|
|
60
|
+
hits = recall.call(query: '', limit: 50)
|
|
61
|
+
expect(hits).to be_a(Hash)
|
|
62
|
+
names = Array(hits[:catalog]).map { |h| h[:name].to_s }
|
|
63
|
+
expect(names).to include('osint')
|
|
64
|
+
expect(names).not_to include('third-party-dump')
|
|
65
|
+
expect(hits[:other_installed].to_i).to be >= 1
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
it 'skills_update folds an RL lesson into an existing skill without creating one' do
|
|
69
|
+
create.call(name: 'osint', description: 'Drive extro_osint.', content: "# OSINT\nCollect from public sources.\n")
|
|
70
|
+
out = update.call(name: 'osint', lesson: 'Prefer sitemap.xml over infinite scroll for Wix blogs.')
|
|
71
|
+
expect(out[:updated]).to eq true
|
|
72
|
+
body = File.read(out[:path])
|
|
73
|
+
expect(body).to include('## RL feedback')
|
|
74
|
+
expect(body).to include('Prefer sitemap.xml over infinite scroll')
|
|
75
|
+
expect(body).to include('Collect from public sources')
|
|
76
|
+
again = update.call(name: 'osint', lesson: 'Prefer sitemap.xml over infinite scroll for Wix blogs.')
|
|
77
|
+
expect(again[:updated]).to eq false
|
|
78
|
+
expect(again[:reason]).to eq('already folded')
|
|
79
|
+
end
|
|
80
|
+
|
|
52
81
|
it 'skill_create writes <name>/SKILL.md with required name+description frontmatter' do
|
|
53
82
|
out = create.call(name: 'Recon Quick_Scan', description: 'Fast host triage.', content: "# Recon\nnmap -T4 -F {t}\n", references: ['T1046'])
|
|
54
83
|
expect(out[:saved]).to be true
|
|
@@ -45,4 +45,26 @@ describe PWN::AI::Grok do
|
|
|
45
45
|
expect(src).to include('PWN::Plugins::TTYSpinner.stop(spin: spin)')
|
|
46
46
|
expect(src).not_to match(/spin\.stop if spinner/)
|
|
47
47
|
end
|
|
48
|
+
|
|
49
|
+
it 'does not nil+ on 429 without Retry-After and does not retry 429 forever' do
|
|
50
|
+
src = File.read(described_class.method(:chat_with_tools).source_location.first)
|
|
51
|
+
expect(src).not_to match(/headers\[:retry_after\]&.to_i\s*\|\|=/)
|
|
52
|
+
expect(src).to match(/HttpRetry\.retry_after_s|headers\[:retry_after\]\.to_i/)
|
|
53
|
+
expect(src).to match(/HttpRetry\.max_attempts|max_attempts/)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
it 'defaults chat REST timeout to 180s and retries ReadTimeout up to 5 times' do
|
|
57
|
+
src = File.read(described_class.method(:chat_with_tools).source_location.first)
|
|
58
|
+
expect(src).to match(/HttpRetry|timeout \|\|= 180/)
|
|
59
|
+
expect(src).not_to match(/timeout \|\|= 900/)
|
|
60
|
+
expect(src).to match(/HttpRetry\.max_attempts|max_attempts/)
|
|
61
|
+
inner = src[/retry_count = 0.*?rescue RestClient::Exceptions::Timeout/m]
|
|
62
|
+
expect(inner).to include('TooManyRequests')
|
|
63
|
+
expect(inner).to include('Exceptions::Timeout')
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
it 'tees grok timeout/429 warnings into the debug request log' do
|
|
67
|
+
src = File.read(described_class.method(:chat_with_tools).source_location.first)
|
|
68
|
+
expect(src).to match(/HttpRetry\.report_event|Log\.progress/)
|
|
69
|
+
end
|
|
48
70
|
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
require 'stringio'
|
|
5
|
+
|
|
6
|
+
describe PWN::AI::HttpRetry do
|
|
7
|
+
it 'defaults timeout to 180 and max_attempts to 5' do
|
|
8
|
+
expect(described_class.timeout_s).to eq(180)
|
|
9
|
+
expect(described_class.timeout_s(timeout: nil)).to eq(180)
|
|
10
|
+
expect(described_class.timeout_s(timeout: 60)).to eq(60)
|
|
11
|
+
expect(described_class.max_attempts).to eq(5)
|
|
12
|
+
expect(described_class.max_attempts(quiet: true)).to eq(1)
|
|
13
|
+
expect(described_class.max_attempts(timeout: 12)).to eq(1)
|
|
14
|
+
expect(described_class.max_attempts(max_attempts: 3)).to eq(3)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
it 'tees emit lines into the open debug request log' do
|
|
18
|
+
path = PWN::Plugins::Log.start_debug(
|
|
19
|
+
tee: StringIO.new,
|
|
20
|
+
path: "/tmp/pwn-ai-DEBUG-#{Process.pid}-httpretry.log"
|
|
21
|
+
)
|
|
22
|
+
err = RestClient::Exceptions::ReadTimeout.new('Timed out reading data from server')
|
|
23
|
+
described_class.report_event(
|
|
24
|
+
label: 'openai',
|
|
25
|
+
error: err,
|
|
26
|
+
http_method: :post,
|
|
27
|
+
rest_call: 'chat/completions',
|
|
28
|
+
extra: 'timeout=180s attempt=1/5',
|
|
29
|
+
quiet: true
|
|
30
|
+
)
|
|
31
|
+
body = File.read(path)
|
|
32
|
+
expect(body).to include('[pwn-ai/openai]')
|
|
33
|
+
expect(body).to include('Timed out reading data from server')
|
|
34
|
+
expect(body).to include('timeout=180s')
|
|
35
|
+
ensure
|
|
36
|
+
PWN::Plugins::Log.stop_debug if defined?(PWN::Plugins::Log)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
{
|
|
40
|
+
'PWN::AI::Grok' => '/opt/pwn/lib/pwn/ai/grok.rb',
|
|
41
|
+
'PWN::AI::OpenAI' => '/opt/pwn/lib/pwn/ai/open_ai.rb',
|
|
42
|
+
'PWN::AI::Anthropic' => '/opt/pwn/lib/pwn/ai/anthropic.rb',
|
|
43
|
+
'PWN::AI::Gemini' => '/opt/pwn/lib/pwn/ai/gemini.rb',
|
|
44
|
+
'PWN::AI::Ollama' => '/opt/pwn/lib/pwn/ai/ollama.rb',
|
|
45
|
+
'PWN::AI::OpenWebUI' => '/opt/pwn/lib/pwn/ai/open_web_ui.rb'
|
|
46
|
+
}.each do |name, path|
|
|
47
|
+
it "#{name} uses HttpRetry 180s / max_attempts and retries ReadTimeout" do
|
|
48
|
+
src = File.read(path)
|
|
49
|
+
expect(src).to include('HttpRetry')
|
|
50
|
+
expect(src).not_to match(/timeout \|\|= 900/)
|
|
51
|
+
expect(src).to match(/Exceptions::Timeout/)
|
|
52
|
+
expect(src).to match(/HttpRetry\.report_event/)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|