pwn 0.5.705 → 0.5.707
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/documentation/Reporting.md +1 -0
- data/etc/default_skills/pwn/reports/SKILL.md +4 -2
- data/etc/default_skills/pwn/reports/csv/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/html/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/json/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/markdown/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/pdf/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/xml/SKILL.md +47 -0
- data/lib/pwn/ai/agent/curriculum.rb +22 -20
- data/lib/pwn/ai/agent/learning.rb +3 -10
- data/lib/pwn/ai/agent/loop.rb +243 -52
- data/lib/pwn/ai/agent/policy.rb +23 -5
- data/lib/pwn/ai/agent/reward.rb +11 -23
- data/lib/pwn/ai/agent/turn_finalizer.rb +0 -1
- data/lib/pwn/reports/ai_red_team.rb +1 -1
- data/lib/pwn/reports/csv.rb +38 -0
- data/lib/pwn/reports/fuzz.rb +1 -1
- data/lib/pwn/reports/html.rb +58 -0
- data/lib/pwn/reports/json.rb +32 -0
- data/lib/pwn/reports/markdown.rb +40 -0
- data/lib/pwn/reports/pdf.rb +93 -0
- data/lib/pwn/reports/phone.rb +1 -1
- data/lib/pwn/reports/sast.rb +1 -1
- data/lib/pwn/reports/uri_buster.rb +1 -1
- data/lib/pwn/reports/xml.rb +44 -0
- data/lib/pwn/reports.rb +54 -6
- data/lib/pwn/version.rb +1 -1
- data/spec/integration/reinforced_feedback_loop_spec.rb +19 -4
- data/spec/lib/pwn/ai/agent/loop_spec.rb +166 -10
- data/spec/lib/pwn/ai/agent/policy_spec.rb +12 -2
- data/spec/lib/pwn/ai/agent/reward_spec.rb +72 -0
- data/spec/lib/pwn/reports/csv_spec.rb +19 -0
- data/spec/lib/pwn/reports/formats_spec.rb +90 -0
- data/spec/lib/pwn/reports/html_spec.rb +19 -0
- data/spec/lib/pwn/reports/json_spec.rb +19 -0
- data/spec/lib/pwn/reports/markdown_spec.rb +19 -0
- data/spec/lib/pwn/reports/pdf_spec.rb +19 -0
- data/spec/lib/pwn/reports/xml_spec.rb +19 -0
- data/third_party/pwn_rdoc.jsonl +46 -1
- metadata +22 -3
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PWN
|
|
4
|
+
module Reports
|
|
5
|
+
# Generic HTML report writer for pentest / findings payloads.
|
|
6
|
+
module HTML
|
|
7
|
+
public_class_method def self.generate(opts = {})
|
|
8
|
+
out = PWN::Reports.resolve_path(opts.merge(ext: 'html'))
|
|
9
|
+
payload = PWN::Reports.report_payload(opts)
|
|
10
|
+
rows = payload[:findings].map do |row|
|
|
11
|
+
"<tr><td>#{h(text: row['id'])}</td><td>#{h(text: row['title'])}</td><td>#{h(text: row['severity'])}</td><td>#{h(text: row['cvss'])}</td><td>#{h(text: row['description'])}</td><td>#{h(text: row['poc'])}</td><td>#{h(text: row['recommendation'])}</td></tr>"
|
|
12
|
+
end
|
|
13
|
+
body = <<~HTML
|
|
14
|
+
<!DOCTYPE html>
|
|
15
|
+
<html lang="en">
|
|
16
|
+
<head>
|
|
17
|
+
<meta charset="utf-8">
|
|
18
|
+
<title>#{h(text: payload[:title])}</title>
|
|
19
|
+
</head>
|
|
20
|
+
<body>
|
|
21
|
+
<h1>#{h(text: payload[:title])}</h1>
|
|
22
|
+
#{summary_html(text: payload[:executive_summary])}
|
|
23
|
+
<table>
|
|
24
|
+
<thead>
|
|
25
|
+
<tr><th>id</th><th>title</th><th>severity</th><th>cvss</th><th>description</th><th>poc</th><th>recommendation</th></tr>
|
|
26
|
+
</thead>
|
|
27
|
+
<tbody>
|
|
28
|
+
#{rows.join("\n")}
|
|
29
|
+
</tbody>
|
|
30
|
+
</table>
|
|
31
|
+
</body>
|
|
32
|
+
</html>
|
|
33
|
+
HTML
|
|
34
|
+
File.write(out, body)
|
|
35
|
+
out
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private_class_method def self.summary_html(opts = {})
|
|
39
|
+
text = opts[:text].to_s
|
|
40
|
+
return if text.empty?
|
|
41
|
+
|
|
42
|
+
"<p>#{h(text: text)}</p>"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private_class_method def self.h(opts = {})
|
|
46
|
+
opts[:text].to_s.gsub('&', '&').gsub('<', '<').gsub('>', '>').gsub('"', '"')
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
public_class_method def self.authors
|
|
50
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
public_class_method def self.help
|
|
54
|
+
puts "USAGE:\n #{self}.generate(\n path: '/tmp/report.html',\n results_hash: {}\n )\n\n #{self}.authors\n"
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module PWN
|
|
6
|
+
module Reports
|
|
7
|
+
# Generic JSON report writer for pentest / findings payloads.
|
|
8
|
+
module JSON
|
|
9
|
+
public_class_method def self.generate(opts = {})
|
|
10
|
+
out = PWN::Reports.resolve_path(opts.merge(ext: 'json'))
|
|
11
|
+
payload = PWN::Reports.report_payload(opts)
|
|
12
|
+
File.write(
|
|
13
|
+
out,
|
|
14
|
+
::JSON.pretty_generate(
|
|
15
|
+
'title' => payload[:title],
|
|
16
|
+
'executive_summary' => payload[:executive_summary],
|
|
17
|
+
'findings' => payload[:findings]
|
|
18
|
+
)
|
|
19
|
+
)
|
|
20
|
+
out
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
public_class_method def self.authors
|
|
24
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
public_class_method def self.help
|
|
28
|
+
puts "USAGE:\n #{self}.generate(\n path: '/tmp/report.json',\n results_hash: {}\n )\n\n #{self}.authors\n"
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PWN
|
|
4
|
+
module Reports
|
|
5
|
+
# Generic Markdown report writer for pentest / findings payloads.
|
|
6
|
+
module Markdown
|
|
7
|
+
public_class_method def self.generate(opts = {})
|
|
8
|
+
out = PWN::Reports.resolve_path(opts.merge(ext: 'md'))
|
|
9
|
+
payload = PWN::Reports.report_payload(opts)
|
|
10
|
+
lines = ["# #{payload[:title]}", '']
|
|
11
|
+
lines += ['## Executive summary', '', payload[:executive_summary].to_s, ''] unless payload[:executive_summary].to_s.empty?
|
|
12
|
+
lines += ['## Findings', '']
|
|
13
|
+
if payload[:findings].empty?
|
|
14
|
+
lines << '_No findings._'
|
|
15
|
+
else
|
|
16
|
+
payload[:findings].each do |row|
|
|
17
|
+
lines << "### #{row['id'].to_s.empty? ? row['title'] : "#{row['id']}: #{row['title']}"}"
|
|
18
|
+
lines << ''
|
|
19
|
+
row.each do |key, val|
|
|
20
|
+
next if %w[id title].include?(key.to_s)
|
|
21
|
+
|
|
22
|
+
lines << "- **#{key}**: #{val}"
|
|
23
|
+
end
|
|
24
|
+
lines << ''
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
File.write(out, "#{lines.join("\n").rstrip}\n")
|
|
28
|
+
out
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
public_class_method def self.authors
|
|
32
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
public_class_method def self.help
|
|
36
|
+
puts "USAGE:\n #{self}.generate(\n path: '/tmp/report.md',\n results_hash: {}\n )\n\n #{self}.authors\n"
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PWN
|
|
4
|
+
module Reports
|
|
5
|
+
# Generic PDF report writer for pentest / findings payloads.
|
|
6
|
+
# Emits a minimal PDF 1.4 document (no wkhtmltopdf).
|
|
7
|
+
module PDF
|
|
8
|
+
public_class_method def self.generate(opts = {})
|
|
9
|
+
out = PWN::Reports.resolve_path(opts.merge(ext: 'pdf'))
|
|
10
|
+
payload = PWN::Reports.report_payload(opts)
|
|
11
|
+
File.binwrite(out, render_pdf(payload: payload))
|
|
12
|
+
out
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private_class_method def self.render_pdf(opts = {})
|
|
16
|
+
payload = opts[:payload]
|
|
17
|
+
lines = [payload[:title].to_s]
|
|
18
|
+
lines << ''
|
|
19
|
+
unless payload[:executive_summary].to_s.empty?
|
|
20
|
+
lines << 'Executive summary'
|
|
21
|
+
lines.concat(wrap_line(text: payload[:executive_summary].to_s))
|
|
22
|
+
lines << ''
|
|
23
|
+
end
|
|
24
|
+
payload[:findings].each do |row|
|
|
25
|
+
heading = [row['id'], row['title']].compact.map(&:to_s).reject(&:empty?).join(': ')
|
|
26
|
+
lines.concat(wrap_line(text: heading))
|
|
27
|
+
row.each do |key, val|
|
|
28
|
+
next if %w[id title].include?(key.to_s)
|
|
29
|
+
|
|
30
|
+
lines.concat(wrap_line(text: "#{key}: #{val}"))
|
|
31
|
+
end
|
|
32
|
+
lines << ''
|
|
33
|
+
end
|
|
34
|
+
content = pdf_stream(lines: lines)
|
|
35
|
+
objects = [
|
|
36
|
+
'<< /Type /Catalog /Pages 2 0 R >>',
|
|
37
|
+
'<< /Type /Pages /Kids [3 0 R] /Count 1 >>',
|
|
38
|
+
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>',
|
|
39
|
+
"<< /Length #{content.bytesize} >>\nstream\n#{content}\nendstream",
|
|
40
|
+
'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'
|
|
41
|
+
]
|
|
42
|
+
assemble_pdf(objects: objects)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private_class_method def self.pdf_stream(opts = {})
|
|
46
|
+
lines = Array(opts[:lines])
|
|
47
|
+
chunks = ['BT', '/F1 11 Tf', '72 720 Td']
|
|
48
|
+
lines.each_with_index do |line, idx|
|
|
49
|
+
chunks << '0 -14 Td' unless idx.zero?
|
|
50
|
+
chunks << "(#{pdf_escape(text: line)}) Tj"
|
|
51
|
+
end
|
|
52
|
+
chunks << 'ET'
|
|
53
|
+
"#{chunks.join("\n")}\n"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private_class_method def self.pdf_escape(opts = {})
|
|
57
|
+
opts[:text].to_s.encode('UTF-8', invalid: :replace, undef: :replace).gsub('\\', '\\\\').gsub('(', '\\(').gsub(')', '\\)')
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private_class_method def self.wrap_line(opts = {})
|
|
61
|
+
text = opts[:text].to_s.tr("\r", '')
|
|
62
|
+
return [''] if text.empty?
|
|
63
|
+
|
|
64
|
+
text.scan(/.{1,90}(?:\s+|$)|.{1,90}/).map(&:strip)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private_class_method def self.assemble_pdf(opts = {})
|
|
68
|
+
objects = Array(opts[:objects])
|
|
69
|
+
out = +"%PDF-1.4\n"
|
|
70
|
+
offsets = [0]
|
|
71
|
+
objects.each_with_index do |body, idx|
|
|
72
|
+
offsets << out.bytesize
|
|
73
|
+
out << "#{idx + 1} 0 obj\n#{body}\nendobj\n"
|
|
74
|
+
end
|
|
75
|
+
xref_at = out.bytesize
|
|
76
|
+
out << "xref\n0 #{objects.length + 1}\n"
|
|
77
|
+
out << "0000000000 65535 f \n"
|
|
78
|
+
offsets[1..].each { |off| out << format("%010d 00000 n \n", off) }
|
|
79
|
+
out << "trailer\n<< /Size #{objects.length + 1} /Root 1 0 R >>\n"
|
|
80
|
+
out << "startxref\n#{xref_at}\n%%EOF\n"
|
|
81
|
+
out
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
public_class_method def self.authors
|
|
85
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
public_class_method def self.help
|
|
89
|
+
puts "USAGE:\n #{self}.generate(\n path: '/tmp/report.pdf',\n results_hash: {}\n )\n\n #{self}.authors\n"
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
data/lib/pwn/reports/phone.rb
CHANGED
data/lib/pwn/reports/sast.rb
CHANGED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rexml/document'
|
|
4
|
+
|
|
5
|
+
module PWN
|
|
6
|
+
module Reports
|
|
7
|
+
# Generic XML report writer for pentest / findings payloads.
|
|
8
|
+
module XML
|
|
9
|
+
public_class_method def self.generate(opts = {})
|
|
10
|
+
out = PWN::Reports.resolve_path(opts.merge(ext: 'xml'))
|
|
11
|
+
payload = PWN::Reports.report_payload(opts)
|
|
12
|
+
doc = REXML::Document.new
|
|
13
|
+
doc << REXML::XMLDecl.new('1.0', 'UTF-8')
|
|
14
|
+
root = doc.add_element('report')
|
|
15
|
+
root.add_element('title').text = payload[:title]
|
|
16
|
+
root.add_element('executive_summary').text = payload[:executive_summary]
|
|
17
|
+
findings = root.add_element('findings')
|
|
18
|
+
payload[:findings].each do |row|
|
|
19
|
+
node = findings.add_element('finding')
|
|
20
|
+
row.each do |key, val|
|
|
21
|
+
node.add_element(safe_tag(name: key)).text = val.to_s
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
File.open(out, 'w') { |io| doc.write(io, 2) }
|
|
25
|
+
out
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private_class_method def self.safe_tag(opts = {})
|
|
29
|
+
name = opts[:name].to_s
|
|
30
|
+
name = 'field' if name.empty?
|
|
31
|
+
name = "f_#{name}" unless name.match?(/\A[A-Za-z_]/)
|
|
32
|
+
name.gsub(/[^A-Za-z0-9_\-.]/, '_')
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
public_class_method def self.authors
|
|
36
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
public_class_method def self.help
|
|
40
|
+
puts "USAGE:\n #{self}.generate(\n path: '/tmp/report.xml',\n results_hash: {}\n )\n\n #{self}.authors\n"
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
data/lib/pwn/reports.rb
CHANGED
|
@@ -1,25 +1,73 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
|
|
3
5
|
module PWN
|
|
4
6
|
# This file, using the autoload directive loads Report modules
|
|
5
7
|
# into memory only when they're needed. For more information, see:
|
|
6
8
|
# http://www.rubyinside.com/ruby-techniques-revealed-autoload-1652.html
|
|
7
9
|
module Reports
|
|
8
|
-
# autoload :HTML, 'pwn/reports/html'
|
|
9
|
-
# autoload :JSON, 'pwn/reports/json'
|
|
10
|
-
# autoload :PDF, 'pwn/reports/pdf'
|
|
11
10
|
autoload :AIRedTeam, 'pwn/reports/ai_red_team'
|
|
11
|
+
autoload :CSV, 'pwn/reports/csv'
|
|
12
12
|
autoload :Fuzz, 'pwn/reports/fuzz'
|
|
13
|
+
autoload :HTML, 'pwn/reports/html'
|
|
13
14
|
autoload :HTMLFooter, 'pwn/reports/html_footer'
|
|
14
15
|
autoload :HTMLHeader, 'pwn/reports/html_header'
|
|
16
|
+
autoload :JSON, 'pwn/reports/json'
|
|
17
|
+
autoload :Markdown, 'pwn/reports/markdown'
|
|
18
|
+
autoload :PDF, 'pwn/reports/pdf'
|
|
15
19
|
autoload :Phone, 'pwn/reports/phone'
|
|
16
20
|
autoload :SAST, 'pwn/reports/sast'
|
|
17
21
|
autoload :URIBuster, 'pwn/reports/uri_buster'
|
|
18
|
-
|
|
22
|
+
autoload :XML, 'pwn/reports/xml'
|
|
23
|
+
|
|
24
|
+
public_class_method def self.resolve_path(opts = {})
|
|
25
|
+
path = opts[:path].to_s
|
|
26
|
+
ext = opts[:ext].to_s.sub(/\A\./, '')
|
|
27
|
+
unless path.empty?
|
|
28
|
+
FileUtils.mkdir_p(File.dirname(path)) unless File.dirname(path).to_s.empty? || File.dirname(path) == '.'
|
|
29
|
+
return path
|
|
30
|
+
end
|
|
19
31
|
|
|
20
|
-
|
|
32
|
+
dir = opts[:dir_path].to_s
|
|
33
|
+
dir = '.' if dir.empty?
|
|
34
|
+
FileUtils.mkdir_p(dir)
|
|
35
|
+
name = opts[:report_name].to_s
|
|
36
|
+
name = File.basename(Dir.pwd) if name.empty?
|
|
37
|
+
File.join(dir, "#{name}.#{ext}")
|
|
38
|
+
end
|
|
21
39
|
|
|
22
|
-
|
|
40
|
+
public_class_method def self.report_payload(opts = {})
|
|
41
|
+
raw = opts[:results_hash]
|
|
42
|
+
raw = {} unless raw.is_a?(Hash)
|
|
43
|
+
title = (
|
|
44
|
+
opts[:title] ||
|
|
45
|
+
raw[:title] || raw['title'] ||
|
|
46
|
+
raw[:report_name] || raw['report_name'] ||
|
|
47
|
+
'PWN Report'
|
|
48
|
+
).to_s
|
|
49
|
+
summary = (
|
|
50
|
+
opts[:executive_summary] ||
|
|
51
|
+
raw[:executive_summary] || raw['executive_summary']
|
|
52
|
+
).to_s
|
|
53
|
+
findings = raw[:findings] || raw['findings'] || raw[:data] || raw['data'] || []
|
|
54
|
+
findings = [] unless findings.is_a?(Array)
|
|
55
|
+
{
|
|
56
|
+
title: title,
|
|
57
|
+
executive_summary: summary,
|
|
58
|
+
findings: findings.map { |row| stringify_keys(hash: row) },
|
|
59
|
+
raw: raw
|
|
60
|
+
}
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private_class_method def self.stringify_keys(opts = {})
|
|
64
|
+
hash = opts[:hash]
|
|
65
|
+
return { 'value' => hash.to_s } unless hash.is_a?(Hash)
|
|
66
|
+
|
|
67
|
+
hash.each_with_object({}) do |(key, val), acc|
|
|
68
|
+
acc[key.to_s] = val
|
|
69
|
+
end
|
|
70
|
+
end
|
|
23
71
|
|
|
24
72
|
public_class_method def self.authors
|
|
25
73
|
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
data/lib/pwn/version.rb
CHANGED
|
@@ -628,11 +628,11 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
|
|
|
628
628
|
expect(metrics.calibration(engine: :ollama)[:n]).to eq 1
|
|
629
629
|
end
|
|
630
630
|
|
|
631
|
-
it 'a critic :flaw caps
|
|
631
|
+
it 'a critic :flaw caps a weak judge score ≤ 0.3 and triggers HER on failure' do
|
|
632
632
|
@agent_cfg[:auto_introspect] = true
|
|
633
633
|
@agent_cfg[:critic] = true
|
|
634
634
|
allow(curriculum).to receive(:critic).and_return(verdict: :flaw, flaw: 'wrong CVE')
|
|
635
|
-
allow(reward).to receive(:judge).and_return(score: 0.
|
|
635
|
+
allow(reward).to receive(:judge).and_return(score: 0.45, verdict: :partial, rationale: '', success: false)
|
|
636
636
|
allow(reward).to receive(:sentinel).and_return(status: :insufficient)
|
|
637
637
|
expect(curriculum).to receive(:hindsight)
|
|
638
638
|
|
|
@@ -642,6 +642,21 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
|
|
|
642
642
|
expect(row[:success]).to be false
|
|
643
643
|
expect(row[:score]).to be <= 0.3
|
|
644
644
|
end
|
|
645
|
+
|
|
646
|
+
it 'does not let critic :flaw floor a high-evidence judge score' do
|
|
647
|
+
@agent_cfg[:auto_introspect] = true
|
|
648
|
+
@agent_cfg[:critic] = true
|
|
649
|
+
allow(curriculum).to receive(:critic).and_return(verdict: :flaw, flaw: 'plan_cover_low')
|
|
650
|
+
allow(reward).to receive(:judge).and_return(score: 0.87, verdict: :solved, rationale: 'evidence=0.87', success: true)
|
|
651
|
+
allow(reward).to receive(:sentinel).and_return(status: :insufficient)
|
|
652
|
+
allow(curriculum).to receive(:hindsight)
|
|
653
|
+
|
|
654
|
+
s = PWN::Sessions.create(title: 'e2e_critic_keep')
|
|
655
|
+
learning.auto_introspect(session_id: s[:id], request: 'x', final: 'path-backed complete answer')
|
|
656
|
+
row = learning.outcomes.first
|
|
657
|
+
expect(row[:success]).to be true
|
|
658
|
+
expect(row[:score]).to be >= 0.6
|
|
659
|
+
end
|
|
645
660
|
end
|
|
646
661
|
|
|
647
662
|
# ═══════════════════════════════════════════════════════════════════════
|
|
@@ -1009,7 +1024,7 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
|
|
|
1009
1024
|
)
|
|
1010
1025
|
expect(prompts.length).to eq 4
|
|
1011
1026
|
blob = prompts.join(' ').downcase
|
|
1012
|
-
expect(blob).to match(/
|
|
1027
|
+
expect(blob).to match(/poc|findings|severity|reports::json/i)
|
|
1013
1028
|
end
|
|
1014
1029
|
|
|
1015
1030
|
it 'practice sorts budget fingerprints ahead of shell noise' do
|
|
@@ -1167,7 +1182,7 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
|
|
|
1167
1182
|
mistake: { tool: 'agent_loop', error: 'iteration budget exhausted' },
|
|
1168
1183
|
count: 3
|
|
1169
1184
|
)
|
|
1170
|
-
expect(prompts.join(' ')).to match(/
|
|
1185
|
+
expect(prompts.join(' ')).to match(/PoC|findings|severity|Reports::JSON/i)
|
|
1171
1186
|
end
|
|
1172
1187
|
|
|
1173
1188
|
it 'practice refuses resolve when holdouts ok but trace weak on budget target' do
|
|
@@ -40,14 +40,12 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
40
40
|
expect(src).to match(/red_team_plan/)
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
it '
|
|
43
|
+
it 'does not pass TUI plan into auto_introspect' do
|
|
44
44
|
src = File.read(described_class.method(:run).source_location.first)
|
|
45
|
-
# every auto_introspect in Loop.run should pass plan:
|
|
46
45
|
calls = src.scan(/Learning\.auto_introspect\([^)]*\)/m)
|
|
47
|
-
# also multi-line form collapsed into one-liners already
|
|
48
46
|
expect(calls.length).to be >= 2
|
|
49
47
|
calls.each do |c|
|
|
50
|
-
expect(c).
|
|
48
|
+
expect(c).not_to match(/plan:/), "auto_introspect must not take TUI plan: #{c[0, 120]}"
|
|
51
49
|
end
|
|
52
50
|
end
|
|
53
51
|
|
|
@@ -93,7 +91,7 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
93
91
|
parked_n = [a, b].count do |m|
|
|
94
92
|
PWN::AI::Agent::Mistakes.find(signature: m[:signature])[:parked]
|
|
95
93
|
end
|
|
96
|
-
expect(parked_n).to
|
|
94
|
+
expect(parked_n).to eq(0)
|
|
97
95
|
ensure
|
|
98
96
|
FileUtils.remove_entry(tmp) if tmp && Dir.exist?(tmp)
|
|
99
97
|
end
|
|
@@ -451,12 +449,19 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
451
449
|
]
|
|
452
450
|
8.times do |n|
|
|
453
451
|
msgs << { role: 'assistant', content: '', tool_calls: [{ id: "c#{n}" }] }
|
|
454
|
-
msgs << { role: 'tool', tool_call_id: "c#{n}", name: 'shell', content: ('x' * 4_000) }
|
|
452
|
+
msgs << { role: 'tool', tool_call_id: "c#{n}", name: 'shell', content: ('x' * 4_000) + n.to_s }
|
|
455
453
|
end
|
|
456
454
|
out = described_class.send(:compact_history!, messages: msgs)
|
|
457
455
|
tool_bodies = out.select { |m| m[:role].to_s == 'tool' }
|
|
458
456
|
expect(tool_bodies.length).to be <= 6
|
|
459
|
-
|
|
457
|
+
last = tool_bodies.last(2)
|
|
458
|
+
expect(last.map { |m| m[:content].to_s.length }.min).to eq(4_001)
|
|
459
|
+
older = tool_bodies[0...-2]
|
|
460
|
+
expect(older).not_to be_empty
|
|
461
|
+
older.each do |m|
|
|
462
|
+
expect(m[:content].to_s).to include('[compacted path=')
|
|
463
|
+
expect(m[:content].to_s.length).to be < 500
|
|
464
|
+
end
|
|
460
465
|
end
|
|
461
466
|
|
|
462
467
|
it 'compacts again after a transient engine hop before retrying' do
|
|
@@ -523,10 +528,10 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
523
528
|
expect(out.map { |m| m[:tool_call_id] }).not_to include('toolu_orphan')
|
|
524
529
|
end
|
|
525
530
|
|
|
526
|
-
it '
|
|
531
|
+
it 'checkpoints a repeated identical payload instead of extinguishing the tool' do
|
|
527
532
|
src = File.read(described_class.method(:run).source_location.first)
|
|
528
|
-
expect(src).to match(/
|
|
529
|
-
expect(src).not_to match(/pwn_extinguished\[
|
|
533
|
+
expect(src).to match(/checkpoint_result/)
|
|
534
|
+
expect(src).not_to match(/pwn_extinguished\[sig\] = true/)
|
|
530
535
|
end
|
|
531
536
|
|
|
532
537
|
it 'records a timeout increment mistake instead of treating success:true as ok' do
|
|
@@ -910,6 +915,157 @@ describe PWN::AI::Agent::Loop do # rubocop:disable Metrics/BlockLength
|
|
|
910
915
|
Thread.current[:pwn_loop_t0] = nil
|
|
911
916
|
end
|
|
912
917
|
|
|
918
|
+
it 'keeps LLM-declared deliverable paths unsatisfied until those files exist' do
|
|
919
|
+
path = "/tmp/pwn-deliv-#{Process.pid}.bin"
|
|
920
|
+
FileUtils.rm_f(path)
|
|
921
|
+
Thread.current[:pwn_loop_deliverables] = [path]
|
|
922
|
+
req = 'Exhaustively analyze the docker app. 100% coverage. Store the PDF in /tmp/container_pentest-p4-PWN-122B.pdf'
|
|
923
|
+
recap = 'Good! Let me continue the penetration test with more aggressive testing now.'
|
|
924
|
+
msgs = [
|
|
925
|
+
{ role: 'user', content: req },
|
|
926
|
+
{
|
|
927
|
+
role: 'assistant',
|
|
928
|
+
tool_calls: [{ function: { name: 'pwn_eval', arguments: '{"code":"File.write(\"/tmp/pentest_report.json\",\"x\")"}' } }]
|
|
929
|
+
},
|
|
930
|
+
{
|
|
931
|
+
role: 'tool',
|
|
932
|
+
name: 'pwn_eval',
|
|
933
|
+
content: '{"success":true,"result":{"stdout":"Report saved to /tmp/pentest_report.json"},"effect":"write"}'
|
|
934
|
+
},
|
|
935
|
+
{
|
|
936
|
+
role: 'assistant',
|
|
937
|
+
tool_calls: [{ function: { name: 'shell', arguments: '{"command":"curl -s http://127.0.0.1:5000/auth/login"}' } }]
|
|
938
|
+
},
|
|
939
|
+
{
|
|
940
|
+
role: 'tool',
|
|
941
|
+
name: 'shell',
|
|
942
|
+
content: '{"success":true,"result":{"stdout":"<form>"},"effect":"read"}'
|
|
943
|
+
},
|
|
944
|
+
{ role: 'assistant', content: recap }
|
|
945
|
+
]
|
|
946
|
+
expect(described_class.send(:request_unsatisfied?, request: req, messages: msgs)).to eq(true)
|
|
947
|
+
expect(described_class.send(:may_finalize?, request: req, messages: msgs, text: recap)).to eq(false)
|
|
948
|
+
File.write(path, 'any bytes')
|
|
949
|
+
expect(described_class.send(:request_unsatisfied?, request: req, messages: msgs)).to eq(false)
|
|
950
|
+
ensure
|
|
951
|
+
FileUtils.rm_f(path)
|
|
952
|
+
Thread.current[:pwn_loop_deliverables] = nil
|
|
953
|
+
end
|
|
954
|
+
|
|
955
|
+
it 'parses a full acceptance contract from one engine JSON hop' do
|
|
956
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
957
|
+
expect(src).not_to match(/NAMED_PATH_RX/)
|
|
958
|
+
expect(src).not_to include('%PDF')
|
|
959
|
+
expect(src).to include('PWN::Reports::PDF.generate')
|
|
960
|
+
payload = {
|
|
961
|
+
paths: ['/tmp/out.json'],
|
|
962
|
+
min_seconds: 28_800,
|
|
963
|
+
skills: ['penetration-testing'],
|
|
964
|
+
proofs: ['/tmp/poc.sh'],
|
|
965
|
+
hosts: ['127.0.0.1'],
|
|
966
|
+
techniques: ['T1059'],
|
|
967
|
+
issue_work: true
|
|
968
|
+
}
|
|
969
|
+
allow(described_class).to receive(:call_engine).and_return(payload.to_json)
|
|
970
|
+
Thread.current[:pwn_loop_active] = true
|
|
971
|
+
Thread.current[:pwn_loop_deliverables] = nil
|
|
972
|
+
paths = described_class.send(:declared_deliverables, request: 'write a report somewhere')
|
|
973
|
+
expect(paths).to eq(['/tmp/out.json'])
|
|
974
|
+
contract = described_class.send(:declared_contract, request: 'write a report somewhere')
|
|
975
|
+
expect(contract[:min_seconds]).to eq(28_800)
|
|
976
|
+
expect(contract[:skills]).to eq(['penetration-testing'])
|
|
977
|
+
expect(contract[:proofs]).to eq(['/tmp/poc.sh'])
|
|
978
|
+
expect(contract[:hosts]).to eq(['127.0.0.1'])
|
|
979
|
+
expect(contract[:techniques]).to eq(['T1059'])
|
|
980
|
+
expect(contract[:issue_work]).to eq(true)
|
|
981
|
+
expect(described_class).to have_received(:call_engine).once
|
|
982
|
+
ensure
|
|
983
|
+
Thread.current[:pwn_loop_active] = nil
|
|
984
|
+
Thread.current[:pwn_loop_deliverables] = nil
|
|
985
|
+
end
|
|
986
|
+
|
|
987
|
+
it 'keeps issue_work unsatisfied until proofs exist, without Crit/High regex' do
|
|
988
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
989
|
+
expect(src).not_to include('Crit/High')
|
|
990
|
+
Thread.current[:pwn_loop_deliverables] = {
|
|
991
|
+
paths: [],
|
|
992
|
+
min_seconds: 0,
|
|
993
|
+
skills: [],
|
|
994
|
+
proofs: [],
|
|
995
|
+
hosts: [],
|
|
996
|
+
techniques: ['T1059'],
|
|
997
|
+
issue_work: true
|
|
998
|
+
}
|
|
999
|
+
msgs = [{ role: 'tool', content: 'no techniques yet' }]
|
|
1000
|
+
expect(
|
|
1001
|
+
described_class.send(:declared_contract_unsatisfied?, request: 'unique hunt', messages: msgs)
|
|
1002
|
+
).to eq(true)
|
|
1003
|
+
poc = "/tmp/pwn-issue-#{Process.pid}.poc"
|
|
1004
|
+
File.write(poc, 'working poc')
|
|
1005
|
+
Thread.current[:pwn_loop_deliverables][:proofs] = [poc]
|
|
1006
|
+
msgs = [{ role: 'tool', content: 'used T1059 on host' }]
|
|
1007
|
+
expect(
|
|
1008
|
+
described_class.send(:declared_contract_unsatisfied?, request: 'unique hunt', messages: msgs)
|
|
1009
|
+
).to eq(false)
|
|
1010
|
+
ensure
|
|
1011
|
+
FileUtils.rm_f(poc) if defined?(poc)
|
|
1012
|
+
Thread.current[:pwn_loop_deliverables] = nil
|
|
1013
|
+
end
|
|
1014
|
+
|
|
1015
|
+
it 'does not append PLAN: onto the model wire from plan_first' do
|
|
1016
|
+
src = File.read(described_class.method(:plan_first).source_location.first)
|
|
1017
|
+
expect(src).not_to include('messages << { role: \'assistant\', content: "PLAN:')
|
|
1018
|
+
expect(src).not_to include('messages << { role: \'user\', content: rt }')
|
|
1019
|
+
end
|
|
1020
|
+
|
|
1021
|
+
it 'nested host hunts keep CORE_TOOLS by clearing enabled_toolsets' do
|
|
1022
|
+
src = File.read(described_class.method(:run).source_location.first)
|
|
1023
|
+
expect(src).to match(/nested && needs_host_work\?/)
|
|
1024
|
+
expect(src).to match(/opts\[:enabled_toolsets\] = nil/)
|
|
1025
|
+
end
|
|
1026
|
+
|
|
1027
|
+
it 'verifies LLM-declared duration, skills, proofs, and hosts in the world' do
|
|
1028
|
+
proof = "/tmp/pwn-proof-#{Process.pid}.txt"
|
|
1029
|
+
FileUtils.rm_f(proof)
|
|
1030
|
+
req = 'black-box test 127.0.0.1 using penetration-testing'
|
|
1031
|
+
recap = 'done'
|
|
1032
|
+
msgs = [
|
|
1033
|
+
{ role: 'user', content: req },
|
|
1034
|
+
{
|
|
1035
|
+
role: 'assistant',
|
|
1036
|
+
tool_calls: [{ function: { name: 'pwn_eval', arguments: '{"code":"1"}' } }]
|
|
1037
|
+
},
|
|
1038
|
+
{
|
|
1039
|
+
role: 'tool',
|
|
1040
|
+
name: 'pwn_eval',
|
|
1041
|
+
content: '{"success":true,"result":{"value":"1","stdout":"scanned 10.0.0.1"},"effect":"eval"}'
|
|
1042
|
+
},
|
|
1043
|
+
{ role: 'assistant', content: recap }
|
|
1044
|
+
]
|
|
1045
|
+
Thread.current[:pwn_loop_t0] = Time.now
|
|
1046
|
+
Thread.current[:pwn_loop_deliverables] = {
|
|
1047
|
+
paths: [],
|
|
1048
|
+
min_seconds: 60,
|
|
1049
|
+
skills: ['no-such-bundled-skill'],
|
|
1050
|
+
proofs: [proof],
|
|
1051
|
+
hosts: ['127.0.0.1']
|
|
1052
|
+
}
|
|
1053
|
+
expect(described_class.send(:request_unsatisfied?, request: req, messages: msgs)).to eq(true)
|
|
1054
|
+
Thread.current[:pwn_loop_deliverables][:min_seconds] = 0
|
|
1055
|
+
expect(described_class.send(:request_unsatisfied?, request: req, messages: msgs)).to eq(true)
|
|
1056
|
+
Thread.current[:pwn_loop_deliverables][:skills] = []
|
|
1057
|
+
expect(described_class.send(:request_unsatisfied?, request: req, messages: msgs)).to eq(true)
|
|
1058
|
+
File.write(proof, 'poc')
|
|
1059
|
+
expect(described_class.send(:request_unsatisfied?, request: req, messages: msgs)).to eq(true)
|
|
1060
|
+
msgs[2][:content] = '{"success":true,"result":{"value":"1","stdout":"open 127.0.0.1:5000"},"effect":"eval"}'
|
|
1061
|
+
expect(described_class.send(:request_unsatisfied?, request: req, messages: msgs)).to eq(false)
|
|
1062
|
+
expect(described_class.send(:may_finalize?, request: req, messages: msgs, text: recap)).to eq(true)
|
|
1063
|
+
ensure
|
|
1064
|
+
FileUtils.rm_f(proof)
|
|
1065
|
+
Thread.current[:pwn_loop_t0] = nil
|
|
1066
|
+
Thread.current[:pwn_loop_deliverables] = nil
|
|
1067
|
+
end
|
|
1068
|
+
|
|
913
1069
|
it 'forces tool_choice required on host-work before any tool result, for every engine' do
|
|
914
1070
|
src = File.read(described_class.method(:run).source_location.first)
|
|
915
1071
|
expect(src).to match(/tool_choice/)
|