pwn 0.5.721 → 0.5.723
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -2
- data/bin/pwn_setup +5 -5
- data/documentation/AI-Integration.md +34 -1
- data/documentation/Policy-Benchmark.md +151 -0
- data/documentation/Reinforcement-Learning.md +27 -1
- data/etc/default_skills/pwn/ai/agent/engagement/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/agent/learning/SKILL.md +2 -0
- data/etc/default_skills/pwn/ai/agent/metrics/SKILL.md +4 -0
- data/etc/default_skills/pwn/ai/agent/mistakes/SKILL.md +2 -0
- data/etc/default_skills/pwn/ai/agent/policy/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/agent/reward/SKILL.md +2 -0
- data/etc/default_skills/pwn/ai/agent/swarm/SKILL.md +6 -0
- data/etc/default_skills/pwn/ai/agent/tools/capabilities/SKILL.md +45 -0
- data/etc/default_skills/pwn/ai/agent/tools/context/SKILL.md +45 -0
- data/etc/default_skills/pwn/ai/context/SKILL.md +50 -0
- data/etc/default_skills/pwn/ai/http_retry/SKILL.md +7 -0
- data/etc/default_skills/pwn/ai/http_retry/references/urls.md +4 -0
- data/etc/default_skills/pwn/ai/open_ai/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/open_ai/references/urls.md +1 -0
- data/etc/default_skills/pwn/plugins/exploit_dev/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/findings/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/gdb/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/gdbmi/SKILL.md +55 -0
- data/etc/default_skills/pwn/plugins/ghidra_headless/SKILL.md +49 -0
- data/etc/default_skills/pwn/plugins/jobs/SKILL.md +6 -0
- data/etc/default_skills/pwn/plugins/packet/SKILL.md +3 -0
- data/etc/default_skills/pwn/plugins/preflight_checker/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/radare2/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/transparent_browser/SKILL.md +3 -0
- data/etc/default_skills/pwn/reports/engagement/SKILL.md +3 -2
- data/lib/pwn/ai/agent/curriculum.rb +37 -45
- data/lib/pwn/ai/agent/dispatch.rb +9 -0
- data/lib/pwn/ai/agent/engagement.rb +59 -0
- data/lib/pwn/ai/agent/learning.rb +111 -44
- data/lib/pwn/ai/agent/loop.rb +50 -8
- data/lib/pwn/ai/agent/metrics.rb +70 -2
- data/lib/pwn/ai/agent/mistakes.rb +67 -1
- data/lib/pwn/ai/agent/policy.rb +142 -16
- data/lib/pwn/ai/agent/registry.rb +1 -1
- data/lib/pwn/ai/agent/reward.rb +153 -53
- data/lib/pwn/ai/agent/swarm.rb +235 -35
- data/lib/pwn/ai/agent/tool_guard.rb +13 -1
- data/lib/pwn/ai/agent/tools/artifacts.rb +50 -3
- data/lib/pwn/ai/agent/tools/capabilities.rb +19 -0
- data/lib/pwn/ai/agent/tools/context.rb +38 -0
- data/lib/pwn/ai/agent/tools/finding_record.rb +18 -0
- data/lib/pwn/ai/agent/tools/fuzz_campaign.rb +10 -1
- data/lib/pwn/ai/agent/tools/job_run.rb +32 -0
- data/lib/pwn/ai/agent/tools/learning.rb +5 -6
- data/lib/pwn/ai/agent/tools/metrics.rb +16 -0
- data/lib/pwn/ai/agent/tools/pty_session.rb +4 -4
- data/lib/pwn/ai/agent/tools/ruby_eval.rb +6 -5
- data/lib/pwn/ai/agent/tools/shell.rb +10 -1
- data/lib/pwn/ai/agent/tools/skills.rb +30 -0
- data/lib/pwn/ai/agent/tools/swarm.rb +8 -2
- data/lib/pwn/ai/context.rb +193 -0
- data/lib/pwn/ai/http_retry.rb +53 -7
- data/lib/pwn/ai/open_ai.rb +302 -45
- data/lib/pwn/ai.rb +1 -0
- data/lib/pwn/migrate.rb +10 -1
- data/lib/pwn/plugins/artifact_registry.rb +40 -17
- data/lib/pwn/plugins/binary_parser.rb +4 -1
- data/lib/pwn/plugins/exploit_dev.rb +15 -0
- data/lib/pwn/plugins/findings.rb +48 -8
- data/lib/pwn/plugins/gdb.rb +17 -0
- data/lib/pwn/plugins/gdbmi.rb +128 -0
- data/lib/pwn/plugins/ghidra_headless.rb +104 -0
- data/lib/pwn/plugins/jobs.rb +72 -0
- data/lib/pwn/plugins/packet.rb +51 -0
- data/lib/pwn/plugins/preflight_checker.rb +29 -0
- data/lib/pwn/plugins/process_tube.rb +24 -7
- data/lib/pwn/plugins/radare2.rb +14 -2
- data/lib/pwn/plugins/repl.rb +6 -1
- data/lib/pwn/plugins/transparent_browser.rb +64 -0
- data/lib/pwn/plugins.rb +2 -0
- data/lib/pwn/reports/engagement.rb +19 -0
- data/lib/pwn/sessions.rb +3 -1
- data/lib/pwn/version.rb +1 -1
- data/scripts/benchmark_policy.rb +276 -0
- data/spec/documentation/installation_md_spec.rb +18 -4
- data/spec/integration/reinforced_feedback_loop_spec.rb +20 -13
- data/spec/lib/pwn/ai/agent/curriculum_spec.rb +267 -0
- data/spec/lib/pwn/ai/agent/engagement_spec.rb +12 -0
- data/spec/lib/pwn/ai/agent/learning_spec.rb +81 -3
- data/spec/lib/pwn/ai/agent/loop_spec.rb +51 -0
- data/spec/lib/pwn/ai/agent/metrics_spec.rb +44 -0
- data/spec/lib/pwn/ai/agent/mistakes_spec.rb +10 -0
- data/spec/lib/pwn/ai/agent/policy_spec.rb +103 -0
- data/spec/lib/pwn/ai/agent/registry_spec.rb +62 -0
- data/spec/lib/pwn/ai/agent/reward_spec.rb +175 -12
- data/spec/lib/pwn/ai/agent/swarm_spec.rb +121 -1
- data/spec/lib/pwn/ai/agent/tool_guard_spec.rb +6 -0
- data/spec/lib/pwn/ai/agent/tools/capabilities_spec.rb +14 -0
- data/spec/lib/pwn/ai/agent/tools/context_spec.rb +14 -0
- data/spec/lib/pwn/ai/agent/tools/job_run_spec.rb +2 -0
- data/spec/lib/pwn/ai/agent/tools/learning_spec.rb +25 -0
- data/spec/lib/pwn/ai/context_spec.rb +48 -0
- data/spec/lib/pwn/ai/http_retry_spec.rb +27 -0
- data/spec/lib/pwn/ai/open_ai_oauth_transport_spec.rb +245 -0
- data/spec/lib/pwn/ai/open_ai_spec.rb +150 -0
- data/spec/lib/pwn/migrate_spec.rb +24 -0
- data/spec/lib/pwn/plugins/artifact_registry_spec.rb +19 -0
- data/spec/lib/pwn/plugins/findings_spec.rb +2 -0
- data/spec/lib/pwn/plugins/gdbmi_spec.rb +17 -0
- data/spec/lib/pwn/plugins/ghidra_headless_spec.rb +17 -0
- data/third_party/pwn_rdoc.jsonl +86 -2
- metadata +24 -5
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
RSpec.describe PWN::AI::OpenAI do
|
|
6
|
+
let(:engine) do
|
|
7
|
+
{
|
|
8
|
+
model: 'gpt-4o', base_uri: 'https://api.openai.com/v1', key: 'fake-api-key',
|
|
9
|
+
oauth: { bearer_token: 'fake-oauth-token', account_id: 'fake-account' }
|
|
10
|
+
}
|
|
11
|
+
end
|
|
12
|
+
let(:completed) do
|
|
13
|
+
{
|
|
14
|
+
id: 'resp_test', status: 'completed', usage: { input_tokens: 3, output_tokens: 2 },
|
|
15
|
+
output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'Hello' }] }]
|
|
16
|
+
}
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def event(type, fields = {})
|
|
20
|
+
"event: #{type}\ndata: #{fields.merge(type: type).to_json}\n\n"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
before do
|
|
24
|
+
stub_const('PWN::Env', { ai: { openai: engine } })
|
|
25
|
+
allow(PWN::Plugins::TransparentBrowser).to receive(:open).with(browser_type: :rest).and_return(browser: RestClient)
|
|
26
|
+
allow(PWN::Plugins::TTYSpinner).to receive(:stop)
|
|
27
|
+
allow(described_class).to receive(:obtain_oauth_bearer_token).and_raise('Unexpected enrollment')
|
|
28
|
+
allow(described_class).to receive(:refresh_oauth_bearer_token).and_raise('Unexpected refresh')
|
|
29
|
+
allow(RestClient::Request).to receive(:execute).and_raise('Unexpected HTTP request')
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
it 'raises exhausted subscription timeouts instead of returning a blank result' do
|
|
33
|
+
allow(PWN::AI::HttpRetry).to receive(:max_attempts).and_return(1)
|
|
34
|
+
allow(RestClient::Request).to receive(:execute).and_raise(RestClient::Exceptions::ReadTimeout)
|
|
35
|
+
expect { described_class.chat(request: 'Hi', quiet: true) }.to raise_error(RestClient::Exceptions::ReadTimeout)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
it 'rejects failed Responses events even in quiet mode' do
|
|
39
|
+
allow(RestClient::Request).to receive(:execute).and_return(
|
|
40
|
+
event('response.failed', response: { error: { message: 'Model unavailable' } })
|
|
41
|
+
)
|
|
42
|
+
expect { described_class.chat(request: 'Hi', quiet: true) }.to raise_error(RuntimeError, /Model unavailable/)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
it 'routes subscription text chat to streaming Responses even for older model names' do
|
|
46
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
47
|
+
expect(request[:url]).to eq('https://chatgpt.com/backend-api/codex/responses')
|
|
48
|
+
expect(request[:headers]).to include(authorization: 'Bearer fake-oauth-token', accept: 'text/event-stream')
|
|
49
|
+
expect(request[:headers]['ChatGPT-Account-Id']).to eq('fake-account')
|
|
50
|
+
body = JSON.parse(request[:payload], symbolize_names: true)
|
|
51
|
+
expect(body).to include(model: 'gpt-4o', input: [{ role: 'user', content: 'Hi' }], instructions: '', store: false, stream: true)
|
|
52
|
+
expect(body.keys & %i[messages temperature max_tokens max_completion_tokens max_output_tokens]).to be_empty
|
|
53
|
+
event('response.completed', response: completed)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
response = described_class.chat(request: 'Hi')
|
|
57
|
+
expect(response[:choices].last[:content]).to eq('Hello')
|
|
58
|
+
expect(response[:usage]).to eq(completed[:usage])
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
[
|
|
62
|
+
['error', { error: { message: 'Stream rejected' } }, /Stream rejected/],
|
|
63
|
+
['response.incomplete', { response: { incomplete_details: { reason: 'output limit' } } }, /output limit/],
|
|
64
|
+
['response.completed', { response: { status: 'failed', error: { message: 'Failed completion' } } }, /Failed completion/],
|
|
65
|
+
['response.completed', {}, /missing response/]
|
|
66
|
+
].each do |type, fields, message|
|
|
67
|
+
it "rejects invalid terminal stream event #{type} #{fields}" do
|
|
68
|
+
allow(RestClient::Request).to receive(:execute).and_return(event(type, fields))
|
|
69
|
+
expect { described_class.chat(request: 'Hi', quiet: true) }.to raise_error(RuntimeError, message)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it 'rejects streams ending without a completion rather than using partial text' do
|
|
74
|
+
allow(RestClient::Request).to receive(:execute).and_return(
|
|
75
|
+
"#{event('response.output_text.delta', delta: 'Partial')}data: [DONE]\n\n"
|
|
76
|
+
)
|
|
77
|
+
expect { described_class.chat(request: 'Hi', quiet: true) }.to raise_error(RuntimeError, /before response.completed/)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
it 'preserves native tool outputs from done events when completion has only metadata' do
|
|
81
|
+
reasoning = { type: 'reasoning', id: 'rs_1', encrypted_content: 'fake-encrypted-reasoning' }
|
|
82
|
+
call = { type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'shell', arguments: '{"cmd":"pwd"}' }
|
|
83
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
84
|
+
expect(request[:url]).to end_with('/codex/responses')
|
|
85
|
+
body = JSON.parse(request[:payload], symbolize_names: true)
|
|
86
|
+
expect(body[:instructions]).to eq('Use tools')
|
|
87
|
+
expect(body[:tools].first).to include(type: 'function', name: 'shell')
|
|
88
|
+
expect(body[:tool_choice]).to eq(type: 'function', name: 'shell')
|
|
89
|
+
event('response.output_item.done', output_index: 1, item: call) +
|
|
90
|
+
event('response.output_item.done', output_index: 0, item: reasoning) +
|
|
91
|
+
event('response.completed', response: completed.except(:output))
|
|
92
|
+
end
|
|
93
|
+
result = described_class.chat_with_tools(
|
|
94
|
+
messages: [{ role: 'system', content: 'Use tools' }, { role: 'user', content: 'Hi' }],
|
|
95
|
+
tools: [{ type: 'function', function: { name: 'shell' } }],
|
|
96
|
+
tool_choice: { type: 'function', function: { name: 'shell' } }
|
|
97
|
+
)
|
|
98
|
+
expect(result[:assistant_message][:_native_content]).to eq([reasoning, call])
|
|
99
|
+
expected_calls = [{ id: 'call_1', type: 'function', function: { name: 'shell', arguments: '{"cmd":"pwd"}' } }]
|
|
100
|
+
expect(result[:assistant_message][:tool_calls]).to eq(expected_calls)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
it 'uses Platform when a failed OAuth refresh falls back to an API key' do
|
|
104
|
+
engine[:base_uri] = 'https://chatgpt.com/backend-api/codex/'
|
|
105
|
+
engine[:oauth].merge!(expires_at: 1, refresh_token: 'fake-refresh')
|
|
106
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
107
|
+
expect(request[:url]).to eq('https://api.openai.com/v1/chat/completions')
|
|
108
|
+
expect(request[:headers][:authorization]).to eq('Bearer fake-api-key')
|
|
109
|
+
expect(request[:headers]).not_to have_key('ChatGPT-Account-Id')
|
|
110
|
+
expect(JSON.parse(request[:payload])).to eq('model' => 'gpt-4o', 'temperature' => 0.5, 'messages' => [])
|
|
111
|
+
'{}'
|
|
112
|
+
end
|
|
113
|
+
described_class.send(:open_ai_rest_call,
|
|
114
|
+
rest_call: 'chat/completions', http_method: :post, non_interactive: true,
|
|
115
|
+
http_body: { model: 'gpt-4o', temperature: 0.5, messages: [] })
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
it 'preserves an explicitly configured HTTPS compatible proxy without duplicate slashes' do
|
|
119
|
+
engine[:base_uri] = 'https://openai-proxy.example/codex/'
|
|
120
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
121
|
+
expect(request[:url]).to eq('https://openai-proxy.example/codex/responses')
|
|
122
|
+
expect(request[:verify_ssl]).to eq(true)
|
|
123
|
+
event('response.completed', response: completed)
|
|
124
|
+
end
|
|
125
|
+
described_class.chat(request: 'Hi')
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
it 'refuses to send subscription credentials to an insecure custom endpoint' do
|
|
129
|
+
engine[:base_uri] = 'http://openai-proxy.example/codex'
|
|
130
|
+
expect(RestClient::Request).not_to receive(:execute)
|
|
131
|
+
expect { described_class.chat(request: 'Hi', quiet: true) }.to raise_error(ArgumentError, /HTTPS/)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
it 'raises subscription HTTP failures instead of returning a blank answer' do
|
|
135
|
+
allow(RestClient::Request).to receive(:execute).and_raise(RestClient::BadRequest)
|
|
136
|
+
expect { described_class.chat(request: 'Hi') }.to raise_error(RestClient::BadRequest)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
it 'does not follow subscription redirects to another auth endpoint' do
|
|
140
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
141
|
+
expect(request[:max_redirects]).to eq(0)
|
|
142
|
+
event('response.completed', response: completed)
|
|
143
|
+
end
|
|
144
|
+
described_class.chat(request: 'Hi')
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
it 'round-trips encrypted reasoning and function items through the agent loop on a stateless tool continuation' do
|
|
148
|
+
PWN::Env[:ai][:active] = :openai
|
|
149
|
+
native = [
|
|
150
|
+
{ type: 'reasoning', id: 'rs_1', encrypted_content: 'fake-encrypted-reasoning' },
|
|
151
|
+
{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'shell', arguments: '{}' }
|
|
152
|
+
]
|
|
153
|
+
messages = [{ role: 'user', content: 'Hi' }]
|
|
154
|
+
tools = [{ type: 'function', function: { name: 'shell' } }]
|
|
155
|
+
expect(RestClient::Request).to receive(:execute).ordered do |request|
|
|
156
|
+
body = JSON.parse(request[:payload], symbolize_names: true)
|
|
157
|
+
expect(body[:include]).to include('reasoning.encrypted_content')
|
|
158
|
+
event('response.completed', response: completed.merge(output: native))
|
|
159
|
+
end
|
|
160
|
+
first = described_class.chat_with_tools(messages: messages, tools: tools)
|
|
161
|
+
messages += [first[:assistant_message], { role: 'tool', tool_call_id: 'call_1', content: 'ok' }]
|
|
162
|
+
expect(RestClient::Request).to receive(:execute).ordered do |request|
|
|
163
|
+
body = JSON.parse(request[:payload], symbolize_names: true)
|
|
164
|
+
expected_input = [
|
|
165
|
+
{ role: 'user', content: 'Hi' }, *native,
|
|
166
|
+
{ type: 'function_call_output', call_id: 'call_1', output: 'ok' }
|
|
167
|
+
]
|
|
168
|
+
expect(body[:input]).to eq(expected_input)
|
|
169
|
+
event('response.completed', response: completed)
|
|
170
|
+
end
|
|
171
|
+
expect(PWN::AI::Agent::Loop.send(:call_engine, messages: messages, tools: tools)[:content]).to eq('Hello')
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
it 'routes a newly enrolled credential to subscription Responses immediately' do
|
|
175
|
+
engine[:oauth] = { enroll: true }
|
|
176
|
+
allow(described_class).to receive(:obtain_oauth_bearer_token) do |oauth|
|
|
177
|
+
oauth[:account_id] = 'new-account'
|
|
178
|
+
'new-fake-token'
|
|
179
|
+
end
|
|
180
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
181
|
+
expect(request[:url]).to eq('https://chatgpt.com/backend-api/codex/responses')
|
|
182
|
+
expect(request[:headers]).to include(authorization: 'Bearer new-fake-token', 'ChatGPT-Account-Id' => 'new-account')
|
|
183
|
+
event('response.completed', response: completed)
|
|
184
|
+
end
|
|
185
|
+
described_class.chat(request: 'Hi')
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
it 'uses a refreshed OAuth credential even when a Platform key is configured' do
|
|
189
|
+
engine[:oauth].merge!(expires_at: 1, refresh_token: 'fake-refresh')
|
|
190
|
+
allow(described_class).to receive(:refresh_oauth_bearer_token).and_return('refreshed-fake-token')
|
|
191
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
192
|
+
expect(request[:url]).to eq('https://chatgpt.com/backend-api/codex/responses')
|
|
193
|
+
expect(request[:headers][:authorization]).to eq('Bearer refreshed-fake-token')
|
|
194
|
+
event('response.completed', response: completed)
|
|
195
|
+
end
|
|
196
|
+
described_class.chat(request: 'Hi')
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
it 'leaves API-key text chat on Chat Completions with its original parameters' do
|
|
200
|
+
engine[:oauth] = {}
|
|
201
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
202
|
+
expect(request[:url]).to eq('https://api.openai.com/v1/chat/completions')
|
|
203
|
+
expect(request[:headers]).not_to have_key('ChatGPT-Account-Id')
|
|
204
|
+
body = JSON.parse(request[:payload], symbolize_names: true)
|
|
205
|
+
expect(body).to include(temperature: 0.5, max_completion_tokens: 16_384)
|
|
206
|
+
expect(body.keys & %i[stream store input include]).to be_empty
|
|
207
|
+
{ choices: [{ message: { role: 'assistant', content: 'API answer' } }] }.to_json
|
|
208
|
+
end
|
|
209
|
+
expect(described_class.chat(request: 'Hi', temp: 0.5)[:choices].last[:content]).to eq('API answer')
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
%w[gpt-4o gpt-5.5].each do |model|
|
|
213
|
+
it "leaves API-key tool routing and token caps unchanged for #{model}" do
|
|
214
|
+
engine[:oauth] = {}
|
|
215
|
+
expect(RestClient::Request).to receive(:execute) do |request|
|
|
216
|
+
body = JSON.parse(request[:payload], symbolize_names: true)
|
|
217
|
+
expect(body.keys & %i[stream store include]).to be_empty
|
|
218
|
+
if model == 'gpt-4o'
|
|
219
|
+
expect(request[:url]).to eq('https://api.openai.com/v1/chat/completions')
|
|
220
|
+
expect(body[:max_completion_tokens]).to eq(16_384)
|
|
221
|
+
expect(body[:messages].first).not_to have_key(:_native_content)
|
|
222
|
+
{ choices: [{ message: { role: 'assistant', content: 'API answer' } }] }.to_json
|
|
223
|
+
else
|
|
224
|
+
expect(request[:url]).to eq('https://api.openai.com/v1/responses')
|
|
225
|
+
expect(body[:max_output_tokens]).to eq(16_384)
|
|
226
|
+
completed.to_json
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
described_class.chat_with_tools(model: model, messages: [{ role: 'user', content: 'Hi', _native_content: [] }])
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
it 'collects all completed text parts from a CRLF multiline SSE event' do
|
|
234
|
+
completed[:output].first[:content] << { type: 'output_text', text: ' world' }
|
|
235
|
+
payload = JSON.pretty_generate(type: 'response.completed', response: completed)
|
|
236
|
+
sse = ": heartbeat\r\n\r\nevent: response.completed\r\n#{payload.lines.map { |line| "data: #{line.chomp}\r\n" }.join}\r\n"
|
|
237
|
+
allow(RestClient::Request).to receive(:execute).and_return(sse)
|
|
238
|
+
expect(described_class.chat(request: 'Hi')[:choices].last[:content]).to eq('Hello world')
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
it 'rejects a truncated final SSE frame even when its JSON is complete' do
|
|
242
|
+
allow(RestClient::Request).to receive(:execute).and_return(event('response.completed', response: completed).rstrip)
|
|
243
|
+
expect { described_class.chat(request: 'Hi', quiet: true) }.to raise_error(RuntimeError, /before response.completed/)
|
|
244
|
+
end
|
|
245
|
+
end
|
|
@@ -1,8 +1,98 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'spec_helper'
|
|
4
|
+
require 'tmpdir'
|
|
5
|
+
require 'rest-client'
|
|
4
6
|
|
|
5
7
|
describe PWN::AI::OpenAI do
|
|
8
|
+
describe 'OAuth enrollment persistence' do
|
|
9
|
+
before do
|
|
10
|
+
@oauth_home = Dir.mktmpdir('pwn-openai-oauth')
|
|
11
|
+
stub_const('PWN::Env', { ai: { openai: { oauth: {} } }, driver_opts: { pwn_env_path: File.join(@oauth_home, 'pwn.yaml') } })
|
|
12
|
+
allow(described_class).to receive(:sleep)
|
|
13
|
+
@tokens = { access_token: 'fixture-access', refresh_token: 'fixture-refresh', expires_in: 3600 }
|
|
14
|
+
allow(RestClient).to receive(:post).and_return(
|
|
15
|
+
double(body: { device_auth_id: 'fixture-device', user_code: 'fixture-code', interval: 1 }.to_json),
|
|
16
|
+
double(body: { authorization_code: 'fixture-auth-code', code_verifier: 'fixture-verifier' }.to_json),
|
|
17
|
+
double(body: @tokens.to_json)
|
|
18
|
+
)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
after { FileUtils.remove_entry(@oauth_home) }
|
|
22
|
+
|
|
23
|
+
it 'syncs a standalone enrollment into the live environment and attempts vault persistence' do
|
|
24
|
+
expect(described_class).to receive(:persist_oauth_to_vault).with(oauth: hash_including(bearer_token: 'fixture-access', refresh_token: 'fixture-refresh')).and_return(true)
|
|
25
|
+
expect(described_class.obtain_oauth_bearer_token).to eq('fixture-access')
|
|
26
|
+
expect(PWN::Env.dig(:ai, :openai, :oauth)).to include(bearer_token: 'fixture-access', refresh_token: 'fixture-refresh')
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
it 'does not print bearer or refresh tokens when enrollment succeeds without a vault' do
|
|
30
|
+
output = StringIO.new
|
|
31
|
+
original = $stdout
|
|
32
|
+
$stdout = output
|
|
33
|
+
described_class.obtain_oauth_bearer_token
|
|
34
|
+
expect(output.string).not_to include('fixture-access', 'fixture-refresh')
|
|
35
|
+
expect(output.string).to include('session only')
|
|
36
|
+
ensure
|
|
37
|
+
$stdout = original
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
context 'with an existing encrypted vault' do
|
|
41
|
+
before do
|
|
42
|
+
@vault_path = PWN::Env.dig(:driver_opts, :pwn_env_path)
|
|
43
|
+
@decryptor_path = File.join(@oauth_home, 'custom.decryptor')
|
|
44
|
+
PWN::Env[:driver_opts][:pwn_dec_path] = @decryptor_path
|
|
45
|
+
cipher = OpenSSL::Cipher.new('aes-256-cbc')
|
|
46
|
+
@vault_key = Base64.strict_encode64(cipher.random_key)
|
|
47
|
+
@vault_iv = Base64.strict_encode64(cipher.random_iv)
|
|
48
|
+
File.write(@decryptor_path, YAML.dump({ key: @vault_key, iv: @vault_iv }))
|
|
49
|
+
File.write(@vault_path, YAML.dump({ unrelated: 'preserved', ai: { openai: { model: 'test-model', oauth: { refresh_token: 'previous-refresh' } } } }))
|
|
50
|
+
PWN::Plugins::Vault.encrypt(file: @vault_path, key: @vault_key, iv: @vault_iv)
|
|
51
|
+
@original_ciphertext = File.binread(@vault_path)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
it 'leaves the original vault encrypted and unchanged if re-encryption fails' do
|
|
55
|
+
allow(PWN::Plugins::Vault).to receive(:encrypt).and_raise(IOError, 'fixture write failure')
|
|
56
|
+
expect(described_class.send(:persist_oauth_to_vault, oauth: { bearer_token: 'fixture-access' })).to be(false)
|
|
57
|
+
expect(File.binread(@vault_path)).to eq(@original_ciphertext)
|
|
58
|
+
expect(Dir.children(@oauth_home)).to match_array(%w[pwn.yaml custom.decryptor])
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
it 'does not expose vault contents through persistence exception messages' do
|
|
62
|
+
allow(PWN::Plugins::Vault).to receive(:encrypt).and_raise(IOError, 'fixture-access in parser context')
|
|
63
|
+
expect do
|
|
64
|
+
described_class.send(:persist_oauth_to_vault, oauth: { bearer_token: 'fixture-access' })
|
|
65
|
+
end.to output(/vault persistence failed.*IOError/).to_stderr
|
|
66
|
+
expect do
|
|
67
|
+
described_class.send(:persist_oauth_to_vault, oauth: { bearer_token: 'fixture-access' })
|
|
68
|
+
end.not_to output(/fixture-access/).to_stderr
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
it 'persists enrollment with the same decryptor, restrictive permissions and unrelated settings intact' do
|
|
72
|
+
decryptor_before = File.binread(@decryptor_path)
|
|
73
|
+
oauth = { account_id: 'fixture-account' }
|
|
74
|
+
described_class.obtain_oauth_bearer_token(oauth)
|
|
75
|
+
expect(PWN::Plugins::Vault.file_encrypted?(file: @vault_path)).to be(true)
|
|
76
|
+
expect(File.stat(@vault_path).mode & 0o777).to eq(0o600)
|
|
77
|
+
expect(File.binread(@decryptor_path)).to eq(decryptor_before)
|
|
78
|
+
persisted = PWN::Plugins::Vault.dump(file: @vault_path, key: @vault_key, iv: @vault_iv)
|
|
79
|
+
expect(persisted[:unrelated]).to eq('preserved')
|
|
80
|
+
expect(persisted.dig(:ai, :openai, :model)).to eq('test-model')
|
|
81
|
+
expect(persisted.dig(:ai, :openai, :oauth)).to include(oauth)
|
|
82
|
+
expect(persisted.dig(:ai, :openai, :oauth, :expires_at)).to be > Time.now.to_i
|
|
83
|
+
expect(Dir.children(@oauth_home)).to match_array(%w[pwn.yaml custom.decryptor])
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
it 'persists rotated refresh credentials as well as first enrollment' do
|
|
87
|
+
allow(RestClient).to receive(:post).and_return(double(body: { access_token: 'rotated-access', refresh_token: 'rotated-refresh', expires_in: 3600 }.to_json))
|
|
88
|
+
expect(described_class.refresh_oauth_bearer_token(refresh_token: 'previous-refresh')).to eq('rotated-access')
|
|
89
|
+
persisted = PWN::Plugins::Vault.dump(file: @vault_path, key: @vault_key, iv: @vault_iv)
|
|
90
|
+
expect(persisted.dig(:ai, :openai, :oauth)).to include(bearer_token: 'rotated-access', refresh_token: 'rotated-refresh')
|
|
91
|
+
expect(PWN::Env.dig(:ai, :openai, :oauth)).to include(bearer_token: 'rotated-access', refresh_token: 'rotated-refresh')
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
6
96
|
it 'should display information for authors' do
|
|
7
97
|
authors_response = PWN::AI::OpenAI
|
|
8
98
|
expect(authors_response).to respond_to :authors
|
|
@@ -36,4 +126,64 @@ describe PWN::AI::OpenAI do
|
|
|
36
126
|
expect(src).to include('prompt_cache_key')
|
|
37
127
|
expect(src).to include('enabled?(engine: :openai)')
|
|
38
128
|
end
|
|
129
|
+
|
|
130
|
+
it 'treats gpt-5 family as reasoning models' do
|
|
131
|
+
expect(described_class.send(:reasoning_model?, model: 'gpt-5.5')).to eq(true)
|
|
132
|
+
expect(described_class.send(:reasoning_model?, model: 'gpt-5-mini')).to eq(true)
|
|
133
|
+
expect(described_class.send(:reasoning_model?, model: 'gpt-6-astra')).to eq(true)
|
|
134
|
+
expect(described_class.send(:reasoning_model?, model: 'gpt-4o')).to eq(false)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
it 'routes gpt-6-astra and gpt-5.4+ tool calls to responses, gpt-4o to chat/completions' do
|
|
138
|
+
tools = [{ type: 'function', function: { name: 'shell' } }]
|
|
139
|
+
expect(described_class.api_endpoint(model: 'gpt-6-astra', tools: tools)).to eq('responses')
|
|
140
|
+
expect(described_class.api_endpoint(model: 'gpt-5.5', tools: tools)).to eq('responses')
|
|
141
|
+
expect(described_class.api_endpoint(model: 'gpt-5.6-terra', tools: tools)).to eq('responses')
|
|
142
|
+
expect(described_class.api_endpoint(model: 'gpt-4o', tools: tools)).to eq('chat/completions')
|
|
143
|
+
expect(described_class.api_endpoint(model: 'gpt-5-mini', tools: tools)).to eq('chat/completions')
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
it 'converts chat tools and messages into Responses function items' do
|
|
147
|
+
tools = [{
|
|
148
|
+
type: 'function',
|
|
149
|
+
function: { name: 'shell', description: 'run', parameters: { type: 'object', properties: {} } }
|
|
150
|
+
}]
|
|
151
|
+
msgs = [
|
|
152
|
+
{ role: 'system', content: 'sys' },
|
|
153
|
+
{ role: 'user', content: 'hi' },
|
|
154
|
+
{ role: 'assistant', tool_calls: [{ id: 'c1', type: 'function', function: { name: 'shell', arguments: '{}' } }] },
|
|
155
|
+
{ role: 'tool', tool_call_id: 'c1', content: 'ok' }
|
|
156
|
+
]
|
|
157
|
+
body = described_class.send(:responses_http_body, model: 'gpt-6-astra', messages: msgs, tools: tools, max_tokens: 128)
|
|
158
|
+
expect(body[:model]).to eq('gpt-6-astra')
|
|
159
|
+
expect(body[:instructions]).to include('sys')
|
|
160
|
+
expect(body[:tools].first[:name]).to eq('shell')
|
|
161
|
+
expect(body[:tools].first).not_to have_key(:function)
|
|
162
|
+
expect(body[:input]).to include(hash_including(type: 'function_call_output', call_id: 'c1'))
|
|
163
|
+
expect(body).not_to have_key(:messages)
|
|
164
|
+
expect(body).not_to have_key(:reasoning_effort)
|
|
165
|
+
expect(body[:max_output_tokens]).to eq(128)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
it 'parses Responses function_call output into Loop assistant_message tool_calls' do
|
|
169
|
+
raw = {
|
|
170
|
+
output_text: '',
|
|
171
|
+
output: [
|
|
172
|
+
{ type: 'function_call', call_id: 'c1', name: 'shell', arguments: '{"cmd":"id"}' },
|
|
173
|
+
{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: '' }] }
|
|
174
|
+
]
|
|
175
|
+
}
|
|
176
|
+
out = described_class.send(:parse_responses, raw: raw)
|
|
177
|
+
msg = out[:assistant_message]
|
|
178
|
+
expect(msg[:tool_calls].first[:id]).to eq('c1')
|
|
179
|
+
expect(msg[:tool_calls].first[:function][:name]).to eq('shell')
|
|
180
|
+
expect(msg[:_native_content]).to eq(raw[:output])
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
it 'caps chat_with_tools completion tokens and does not retry quota 429s' do
|
|
184
|
+
src = File.read(described_class.method(:chat_with_tools).source_location.first)
|
|
185
|
+
expect(src).to include('max_completion_tokens')
|
|
186
|
+
rest = File.read(described_class.method(:chat).source_location.first)
|
|
187
|
+
expect(rest).to include('quota_exhausted?')
|
|
188
|
+
end
|
|
39
189
|
end
|
|
@@ -68,6 +68,30 @@ describe PWN::Migrate do
|
|
|
68
68
|
end
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
+
it 'schema 2 patches stock escalator/scribe toolsets and leaves engines alone' do
|
|
72
|
+
File.write(File.join(@tmp, 'agents.yml'), <<~YML)
|
|
73
|
+
---
|
|
74
|
+
scribe:
|
|
75
|
+
role: report
|
|
76
|
+
engine: grok
|
|
77
|
+
toolsets: [memory, skills, learning, sessions]
|
|
78
|
+
escalator:
|
|
79
|
+
role: hint
|
|
80
|
+
engine: grok
|
|
81
|
+
toolsets: [terminal, pwn, memory]
|
|
82
|
+
custom_coach:
|
|
83
|
+
role: keep my tools
|
|
84
|
+
engine: grok
|
|
85
|
+
toolsets: [terminal, pwn, memory]
|
|
86
|
+
YML
|
|
87
|
+
PWN::Migrate.run(fix: false, backup: false, io: io)
|
|
88
|
+
raw = YAML.safe_load_file(File.join(@tmp, 'agents.yml'), permitted_classes: [Symbol], symbolize_names: true)
|
|
89
|
+
expect(Array(raw[:escalator][:toolsets] || raw[:escalator]['toolsets'])).to eq([])
|
|
90
|
+
expect(Array(raw[:scribe][:toolsets] || raw[:scribe]['toolsets'])).to include('pwn')
|
|
91
|
+
expect(raw[:escalator][:engine] || raw[:escalator]['engine']).to eq('grok')
|
|
92
|
+
expect(Array(raw[:custom_coach][:toolsets] || raw[:custom_coach]['toolsets'])).to include('terminal')
|
|
93
|
+
end
|
|
94
|
+
|
|
71
95
|
it 'detects & autofixes a corrupt JSON store (quarantine)' do
|
|
72
96
|
File.write(File.join(@tmp, 'metrics.json'), '{{{ not json')
|
|
73
97
|
st = PWN::Migrate.status
|
|
@@ -22,4 +22,23 @@ describe PWN::Plugins::ArtifactRegistry do
|
|
|
22
22
|
expect(rows.first[:kind]).to eq('loot')
|
|
23
23
|
end
|
|
24
24
|
end
|
|
25
|
+
|
|
26
|
+
it 'greps an artifact by regex' do
|
|
27
|
+
Dir.mktmpdir do |dir|
|
|
28
|
+
src = File.join(dir, 'dump.txt')
|
|
29
|
+
File.write(src, "nop\ncall system\nret\n")
|
|
30
|
+
hits = described_class.read_page(path: src, grep: 'call.*system')
|
|
31
|
+
expect(hits[:matches].first[:text]).to include('call system')
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
it 'round-trips put then get by sha256' do
|
|
36
|
+
Dir.mktmpdir do |dir|
|
|
37
|
+
stub_const('PWN::Plugins::ArtifactRegistry::ROOT', File.join(dir, 'art'))
|
|
38
|
+
stored = described_class.put(bytes: 'pcap-bytes', kind: 'pcap', tags: ['net'])
|
|
39
|
+
got = described_class.get(sha256: stored[:sha256])
|
|
40
|
+
expect(got[:body]).to include('pcap-bytes')
|
|
41
|
+
expect(stored[:tags]).to include('net')
|
|
42
|
+
end
|
|
43
|
+
end
|
|
25
44
|
end
|
|
@@ -37,6 +37,8 @@ describe PWN::Plugins::Findings do
|
|
|
37
37
|
child = described_class.chain(parent_id: parent[:id], title: 'account-takeover', poc_artifacts: [poc], severity: 'high')
|
|
38
38
|
expect(child[:chain_parent_id]).to eq(parent[:id])
|
|
39
39
|
expect(child[:composite_severity]).to eq('high')
|
|
40
|
+
score = described_class.chain_score(ids: [parent[:id], child[:id]])
|
|
41
|
+
expect(score[:combined_severity]).to eq('critical')
|
|
40
42
|
end
|
|
41
43
|
end
|
|
42
44
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
describe PWN::Plugins::GDBMI do
|
|
6
|
+
it 'should display information for authors' do
|
|
7
|
+
expect(described_class).to respond_to :authors
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
it 'should display information for existing help method' do
|
|
11
|
+
expect(described_class).to respond_to :help
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
it 'break requires a location' do
|
|
15
|
+
expect { described_class.break(location: '') }.to raise_error(/location/)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
describe PWN::Plugins::GhidraHeadless do
|
|
6
|
+
it 'should display information for authors' do
|
|
7
|
+
expect(described_class).to respond_to :authors
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
it 'should display information for existing help method' do
|
|
11
|
+
expect(described_class).to respond_to :help
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
it 'analyze requires a binary path' do
|
|
15
|
+
expect { described_class.analyze(bin: '') }.to raise_error(/bin is required/)
|
|
16
|
+
end
|
|
17
|
+
end
|