pwn 0.5.613 → 0.5.615
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +10 -10
- data/README.md +110 -24
- data/documentation/AI-Integration.md +36 -0
- data/documentation/BurpSuite.md +26 -0
- data/documentation/Contributing.md +31 -0
- data/documentation/Diagrams.md +57 -0
- data/documentation/Drivers.md +25 -0
- data/documentation/General-PWN-Usage.md +40 -0
- data/documentation/Home.md +51 -0
- data/documentation/How-PWN-Works.md +60 -0
- data/documentation/Installation.md +50 -0
- data/documentation/NmapIt.md +18 -0
- data/documentation/Plugins.md +59 -0
- data/documentation/Reporting.md +28 -0
- data/documentation/SAST.md +32 -0
- data/documentation/Skills-Memory-Learning.md +44 -0
- data/documentation/Transparent-Browser.md +32 -0
- data/documentation/Troubleshooting.md +38 -0
- data/documentation/What-is-PWN.md +34 -0
- data/documentation/Why-PWN.md +18 -0
- data/documentation/diagrams/ai-integration-tool-calling.svg +134 -0
- data/documentation/diagrams/burp-vs-zap-preference.svg +84 -0
- data/documentation/diagrams/code-scanning-sast.svg +112 -0
- data/documentation/diagrams/dot/ai-integration-tool-calling.dot +23 -0
- data/documentation/diagrams/dot/burp-vs-zap-preference.dot +15 -0
- data/documentation/diagrams/dot/code-scanning-sast.dot +21 -0
- data/documentation/diagrams/dot/driver-framework.dot +17 -0
- data/documentation/diagrams/dot/fuzzing-workflow.dot +27 -0
- data/documentation/diagrams/dot/history-to-drivers.dot +20 -0
- data/documentation/diagrams/dot/memory-skills-detailed.dot +28 -0
- data/documentation/diagrams/dot/network-infra-testing.dot +22 -0
- data/documentation/diagrams/dot/overall-pwn-architecture.dot +49 -0
- data/documentation/diagrams/dot/penetration-testing-workflow.dot +29 -0
- data/documentation/diagrams/dot/plugin-ecosystem.dot +31 -0
- data/documentation/diagrams/dot/pwn-ai-feedback-learning-loop.dot +57 -0
- data/documentation/diagrams/dot/pwn-repl-prototyping.dot +37 -0
- data/documentation/diagrams/dot/reporting-pipeline.dot +17 -0
- data/documentation/diagrams/dot/reverse-engineering-flow.dot +21 -0
- data/documentation/diagrams/dot/sessions-cron-automation.dot +17 -0
- data/documentation/diagrams/dot/web-application-testing.dot +27 -0
- data/documentation/diagrams/dot/zero-day-research-flow.dot +27 -0
- data/documentation/diagrams/driver-framework.svg +90 -0
- data/documentation/diagrams/fuzzing-workflow.svg +147 -0
- data/documentation/diagrams/history-to-drivers.svg +117 -0
- data/documentation/diagrams/memory-skills-detailed.svg +154 -0
- data/documentation/diagrams/network-infra-testing.svg +121 -0
- data/documentation/diagrams/overall-pwn-architecture.svg +234 -0
- data/documentation/diagrams/penetration-testing-workflow.svg +148 -0
- data/documentation/diagrams/plugin-ecosystem.svg +172 -0
- data/documentation/diagrams/pwn-ai-feedback-learning-loop.svg +234 -0
- data/documentation/diagrams/pwn-repl-prototyping.svg +179 -0
- data/documentation/diagrams/reporting-pipeline.svg +91 -0
- data/documentation/diagrams/reverse-engineering-flow.svg +114 -0
- data/documentation/diagrams/sessions-cron-automation.svg +92 -0
- data/documentation/diagrams/web-application-testing.svg +137 -0
- data/documentation/diagrams/zero-day-research-flow.svg +149 -0
- data/documentation/pwn-REPL.md +33 -0
- data/documentation/pwn-ai-Agent.md +43 -0
- data/lib/pwn/ai/agent/learning.rb +377 -0
- data/lib/pwn/ai/agent/loop.rb +18 -3
- data/lib/pwn/ai/agent/metrics.rb +155 -0
- data/lib/pwn/ai/agent/prompt_builder.rb +23 -3
- data/lib/pwn/ai/agent/tools/learning.rb +113 -0
- data/lib/pwn/ai/agent/tools/skills.rb +94 -8
- data/lib/pwn/ai/agent.rb +2 -0
- data/lib/pwn/ai/grok.rb +249 -106
- data/lib/pwn/config.rb +86 -13
- data/lib/pwn/plugins/monkey_patch.rb +2 -2
- data/lib/pwn/plugins/repl.rb +68 -3
- data/lib/pwn/version.rb +1 -1
- data/spec/lib/pwn/ai/agent/learning_spec.rb +68 -0
- data/spec/lib/pwn/ai/agent/metrics_spec.rb +27 -0
- data/spec/lib/pwn/ai/agent/tools/learning_spec.rb +30 -0
- metadata +84 -34
- data/documentation/PWN.png +0 -0
- data/documentation/PWN_Contributors_and_Users.png +0 -0
- data/documentation/fax-spectrogram.png +0 -0
- data/documentation/fax-waveform.png +0 -0
- data/documentation/frozen_string_literal_files_changed_from_false_to_true.txt +0 -54
- data/documentation/lifecycle_authz_replay.example.yaml +0 -27
- data/documentation/pwn_android_war_dialer_session.png +0 -0
- data/documentation/pwn_install.png +0 -0
- data/documentation/pwn_wallpaper.jpg +0 -0
- data/documentation/ringing-spectrogram.png +0 -0
- data/documentation/ringing-waveform.png +0 -0
- data/documentation/vulnerability_report_template.md +0 -37
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# The pwn REPL
|
|
2
|
+
|
|
3
|
+
The `pwn` command launches a Pry-based interactive Ruby shell with the entire `PWN` namespace pre-loaded.
|
|
4
|
+
|
|
5
|
+
## Starting
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pwn
|
|
9
|
+
pwn[v0.5.613]:001 >>>
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Key Features
|
|
13
|
+
|
|
14
|
+
- Direct access to every `PWN::` constant and plugin.
|
|
15
|
+
- Tab completion for classes and methods.
|
|
16
|
+
- Multi-line support.
|
|
17
|
+
- Easy prototyping of security workflows.
|
|
18
|
+
- `PWN.help` and inspection of any object.
|
|
19
|
+
|
|
20
|
+
## Useful REPL Commands
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
pwn[v0.5.613]:001 >>> PWN::Plugins.constants.sort
|
|
24
|
+
pwn[v0.5.613]:001 >>> PWN::Plugins::BurpSuite.methods(false).sort
|
|
25
|
+
pwn[v0.5.613]:001 >>> PWN::SAST.constants
|
|
26
|
+
pwn[v0.5.613]:001 >>> pwn-ai
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
See:
|
|
30
|
+
- [pwn-ai Agent](pwn-ai-Agent.md) — the killer feature inside the REPL
|
|
31
|
+
- [General Usage](General-PWN-Usage.md)
|
|
32
|
+
|
|
33
|
+
[[Diagrams]]
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# pwn-ai Autonomous Agent
|
|
2
|
+
|
|
3
|
+
The most powerful way to use PWN is via the `pwn-ai` command inside the REPL.
|
|
4
|
+
|
|
5
|
+
## Activation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pwn[v0.5.613]:001 >>> pwn-ai
|
|
9
|
+
[*] pwn-ai agent TUI activated...
|
|
10
|
+
>
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Capabilities
|
|
14
|
+
|
|
15
|
+
- Natural language instruction of complex multi-step security tasks.
|
|
16
|
+
- Full tool calling access to:
|
|
17
|
+
- All `PWN::Plugins` (BurpSuite preferred, TransparentBrowser, NmapIt, Shodan, Metasploit, etc.)
|
|
18
|
+
- `PWN::SAST`
|
|
19
|
+
- `PWN::Reports`
|
|
20
|
+
- Shell execution
|
|
21
|
+
- Memory recall / remember
|
|
22
|
+
- Skill usage and distillation
|
|
23
|
+
- Learning / introspection loop
|
|
24
|
+
- Persistent context across interactions via memory and skills.
|
|
25
|
+
|
|
26
|
+
## Tips
|
|
27
|
+
|
|
28
|
+
- Use `SHIFT+ENTER` to insert newlines.
|
|
29
|
+
- `ENTER` submits the prompt.
|
|
30
|
+
- Type `back` or `exit` to return to normal REPL.
|
|
31
|
+
- Example prompt:
|
|
32
|
+
```
|
|
33
|
+
Scan https://target with NmapIt + TransparentBrowser (via BurpSuite),
|
|
34
|
+
run relevant SAST if source present, exploit any findings, generate report.
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Self-Improvement
|
|
38
|
+
|
|
39
|
+
Successful workflows can be distilled into reusable **Skills** (see [Skills-Memory-Learning](Skills-Memory-Learning.md)).
|
|
40
|
+
|
|
41
|
+
See also [AI Integration](AI-Integration.md).
|
|
42
|
+
|
|
43
|
+
[[Diagrams]]
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'time'
|
|
6
|
+
require 'digest'
|
|
7
|
+
|
|
8
|
+
module PWN
|
|
9
|
+
module AI
|
|
10
|
+
module Agent
|
|
11
|
+
# PWN::AI::Agent::Learning is the self-improvement engine that closes
|
|
12
|
+
# the pwn-ai feedback loop. It captures task outcomes, mines session
|
|
13
|
+
# transcripts for durable lessons, promotes successful workflows into
|
|
14
|
+
# reusable skills, and prunes / consolidates persistent memory so the
|
|
15
|
+
# agent gets sharper over time instead of accumulating noise.
|
|
16
|
+
#
|
|
17
|
+
# Data flows:
|
|
18
|
+
# Loop.run --(tool telemetry)--> Metrics.record
|
|
19
|
+
# Loop.run --(final answer)----> Learning.auto_reflect (opt-in)
|
|
20
|
+
# model --(tool calls)------> learning_note_outcome / _distill_skill
|
|
21
|
+
# PromptBuilder <----------------- Learning.to_context + Metrics.to_context
|
|
22
|
+
#
|
|
23
|
+
# Everything is file-backed under ~/.pwn so it survives across REPL
|
|
24
|
+
# restarts and is shared by every future session.
|
|
25
|
+
module Learning
|
|
26
|
+
LEARNING_FILE = File.join(Dir.home, '.pwn', 'learning.jsonl')
|
|
27
|
+
MAX_MEMORY_ENTRIES = 200
|
|
28
|
+
|
|
29
|
+
# Supported Method Parameters::
|
|
30
|
+
# entry = PWN::AI::Agent::Learning.note_outcome(
|
|
31
|
+
# task: 'required - short description of what was attempted',
|
|
32
|
+
# success: 'required - Boolean, did the attempt achieve its goal',
|
|
33
|
+
# details: 'optional - free-form notes / error / evidence',
|
|
34
|
+
# session_id: 'optional - PWN::Sessions id this outcome belongs to',
|
|
35
|
+
# tags: 'optional - Array of String labels for later retrieval'
|
|
36
|
+
# )
|
|
37
|
+
|
|
38
|
+
public_class_method def self.note_outcome(opts = {})
|
|
39
|
+
task = opts[:task].to_s
|
|
40
|
+
success = opts[:success] ? true : false
|
|
41
|
+
raise 'ERROR: task is required' if task.strip.empty?
|
|
42
|
+
|
|
43
|
+
entry = {
|
|
44
|
+
id: Digest::SHA256.hexdigest("#{task}-#{Time.now.to_f}")[0, 12],
|
|
45
|
+
task: task,
|
|
46
|
+
success: success,
|
|
47
|
+
details: opts[:details].to_s[0, 2_000],
|
|
48
|
+
session_id: opts[:session_id],
|
|
49
|
+
tags: Array(opts[:tags]).map(&:to_s),
|
|
50
|
+
timestamp: Time.now.utc.iso8601
|
|
51
|
+
}
|
|
52
|
+
FileUtils.mkdir_p(File.dirname(LEARNING_FILE))
|
|
53
|
+
File.open(LEARNING_FILE, 'a') { |f| f.puts(JSON.generate(entry)) }
|
|
54
|
+
|
|
55
|
+
key = :"lesson_#{entry[:id]}"
|
|
56
|
+
cat = :lesson
|
|
57
|
+
val = "#{success ? 'SUCCESS' : 'FAILURE'}: #{task} — #{opts[:details].to_s.strip[0, 200]}"
|
|
58
|
+
PWN::Memory.remember(key: key, value: val, category: cat) if defined?(PWN::Memory)
|
|
59
|
+
|
|
60
|
+
entry
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Supported Method Parameters::
|
|
64
|
+
# rows = PWN::AI::Agent::Learning.outcomes(
|
|
65
|
+
# limit: 'optional - max entries returned newest-first (default 50)',
|
|
66
|
+
# success: 'optional - filter by Boolean outcome',
|
|
67
|
+
# tag: 'optional - filter by tag substring'
|
|
68
|
+
# )
|
|
69
|
+
|
|
70
|
+
public_class_method def self.outcomes(opts = {})
|
|
71
|
+
limit = opts[:limit] || 50
|
|
72
|
+
want_ok = opts.key?(:success) ? !opts[:success].nil? && opts[:success] != false : nil
|
|
73
|
+
tag = opts[:tag].to_s.downcase
|
|
74
|
+
return [] unless File.exist?(LEARNING_FILE)
|
|
75
|
+
|
|
76
|
+
rows = File.readlines(LEARNING_FILE).map do |l|
|
|
77
|
+
JSON.parse(l, symbolize_names: true)
|
|
78
|
+
rescue StandardError
|
|
79
|
+
nil
|
|
80
|
+
end
|
|
81
|
+
rows.compact!
|
|
82
|
+
rows.select! { |r| r[:success] == want_ok } unless want_ok.nil?
|
|
83
|
+
rows.select! { |r| Array(r[:tags]).any? { |t| t.to_s.downcase.include?(tag) } } unless tag.empty?
|
|
84
|
+
rows.reverse.first(limit)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Supported Method Parameters::
|
|
88
|
+
# stats = PWN::AI::Agent::Learning.stats
|
|
89
|
+
|
|
90
|
+
public_class_method def self.stats
|
|
91
|
+
rows = outcomes(limit: 10_000)
|
|
92
|
+
total = rows.length
|
|
93
|
+
ok = rows.count { |r| r[:success] }
|
|
94
|
+
skills = defined?(PWN::Skills) && PWN::Skills.is_a?(Hash) ? PWN::Skills.keys.length : 0
|
|
95
|
+
mem = defined?(PWN::Memory) ? PWN::Memory.load.keys.length : 0
|
|
96
|
+
{
|
|
97
|
+
total_outcomes: total,
|
|
98
|
+
successes: ok,
|
|
99
|
+
failures: total - ok,
|
|
100
|
+
success_rate: total.positive? ? (ok.to_f / total).round(3) : 0.0,
|
|
101
|
+
skills_known: skills,
|
|
102
|
+
memory_entries: mem,
|
|
103
|
+
tool_metrics: (Metrics.summary(limit: 5) if defined?(Metrics))
|
|
104
|
+
}
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Supported Method Parameters::
|
|
108
|
+
# ctx = PWN::AI::Agent::Learning.to_context(
|
|
109
|
+
# limit: 'optional - number of recent outcomes to surface (default 5)'
|
|
110
|
+
# )
|
|
111
|
+
|
|
112
|
+
public_class_method def self.to_context(opts = {})
|
|
113
|
+
limit = opts[:limit] || 5
|
|
114
|
+
rows = outcomes(limit: limit)
|
|
115
|
+
return '' if rows.empty?
|
|
116
|
+
|
|
117
|
+
lines = rows.map do |r|
|
|
118
|
+
flag = r[:success] ? '✓' : '✗'
|
|
119
|
+
" #{flag} #{r[:task][0, 100]} (#{r[:timestamp]})"
|
|
120
|
+
end
|
|
121
|
+
s = stats
|
|
122
|
+
hdr = "RECENT OUTCOMES (success_rate=#{(s[:success_rate] * 100).round(1)}% over #{s[:total_outcomes]} attempts)"
|
|
123
|
+
"#{hdr}\n#{lines.join("\n")}\n\n"
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Supported Method Parameters::
|
|
127
|
+
# skill = PWN::AI::Agent::Learning.distill_skill(
|
|
128
|
+
# name: 'required - snake_case name for the new skill',
|
|
129
|
+
# session_id: 'optional - PWN::Sessions id to mine (uses its transcript)',
|
|
130
|
+
# content: 'optional - explicit markdown body; overrides transcript mining',
|
|
131
|
+
# references: 'optional - Array of reference URLs / CWE / CVE / ATT&CK ids'
|
|
132
|
+
# )
|
|
133
|
+
|
|
134
|
+
public_class_method def self.distill_skill(opts = {})
|
|
135
|
+
name = opts[:name].to_s.gsub(/[^a-z0-9_-]/i, '_')
|
|
136
|
+
raise 'ERROR: name is required' if name.empty?
|
|
137
|
+
|
|
138
|
+
body = opts[:content].to_s
|
|
139
|
+
body = build_skill_from_session(session_id: opts[:session_id], name: name) if body.strip.empty? && opts[:session_id]
|
|
140
|
+
raise 'ERROR: content or session_id is required' if body.strip.empty?
|
|
141
|
+
|
|
142
|
+
refs = Array(opts[:references]).map(&:to_s).map(&:strip).reject(&:empty?).uniq
|
|
143
|
+
unless refs.empty?
|
|
144
|
+
body = "---\nreferences:\n#{refs.map { |r| " - #{r}" }.join("\n")}\n---\n#{body}" unless body.start_with?("---\n")
|
|
145
|
+
body = "#{body.rstrip}\n\n## References\n#{refs.map { |r| "- #{r}" }.join("\n")}\n" unless body =~ /^\#{1,3}\s*References\s*$/i
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
dir = skills_dir
|
|
149
|
+
FileUtils.mkdir_p(dir)
|
|
150
|
+
path = File.join(dir, "#{name}.md")
|
|
151
|
+
File.write(path, body)
|
|
152
|
+
PWN::Config.load_skills(pwn_skills_path: dir) if defined?(PWN::Config) && PWN::Config.respond_to?(:load_skills)
|
|
153
|
+
note_outcome(task: "distill_skill:#{name}", success: true, details: "Saved #{path}", tags: %w[skill auto])
|
|
154
|
+
{ saved: true, name: name, path: path, bytes: body.bytesize, references: refs }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Supported Method Parameters::
|
|
158
|
+
# report = PWN::AI::Agent::Learning.reflect(
|
|
159
|
+
# session_id: 'required - PWN::Sessions id to analyse',
|
|
160
|
+
# dry_run: 'optional - when true, do not write to Memory/Skills (default false)'
|
|
161
|
+
# )
|
|
162
|
+
#
|
|
163
|
+
# Uses PWN::AI::Introspection (when available) to LLM-summarise the
|
|
164
|
+
# session into structured lessons. Falls back to a heuristic
|
|
165
|
+
# extractor when introspection is disabled so learning never stops.
|
|
166
|
+
|
|
167
|
+
public_class_method def self.reflect(opts = {})
|
|
168
|
+
session_id = opts[:session_id]
|
|
169
|
+
dry_run = opts[:dry_run] ? true : false
|
|
170
|
+
raise 'ERROR: session_id is required' if session_id.to_s.empty?
|
|
171
|
+
|
|
172
|
+
transcript = PWN::Sessions.load(session_id: session_id)
|
|
173
|
+
return { session_id: session_id, lessons: [], reason: 'empty transcript' } if transcript.empty?
|
|
174
|
+
|
|
175
|
+
lessons = introspective_lessons(transcript: transcript)
|
|
176
|
+
lessons = heuristic_lessons(transcript: transcript) if lessons.empty?
|
|
177
|
+
|
|
178
|
+
saved = []
|
|
179
|
+
lessons.each do |l|
|
|
180
|
+
next if l.to_s.strip.empty?
|
|
181
|
+
|
|
182
|
+
key = :"reflect_#{session_id}_#{Digest::SHA256.hexdigest(l)[0, 8]}"
|
|
183
|
+
PWN::Memory.remember(key: key, value: l, category: :lesson) unless dry_run
|
|
184
|
+
saved << { key: key, lesson: l }
|
|
185
|
+
end
|
|
186
|
+
consolidate unless dry_run
|
|
187
|
+
|
|
188
|
+
{ session_id: session_id, lessons: saved, count: saved.length, dry_run: dry_run }
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Supported Method Parameters::
|
|
192
|
+
# PWN::AI::Agent::Learning.auto_reflect(
|
|
193
|
+
# session_id: 'required - id of the just-completed session',
|
|
194
|
+
# request: 'optional - original user request (for outcome logging)',
|
|
195
|
+
# final: 'optional - final assistant answer (for outcome logging)'
|
|
196
|
+
# )
|
|
197
|
+
#
|
|
198
|
+
# Called by Loop.run when PWN::Env[:ai][:agent][:auto_reflect] is
|
|
199
|
+
# truthy. Never raises — learning must not break the primary loop.
|
|
200
|
+
|
|
201
|
+
public_class_method def self.auto_reflect(opts = {})
|
|
202
|
+
session_id = opts[:session_id]
|
|
203
|
+
return unless session_id
|
|
204
|
+
return unless auto_reflect_enabled?
|
|
205
|
+
|
|
206
|
+
note_outcome(
|
|
207
|
+
task: opts[:request].to_s[0, 120],
|
|
208
|
+
success: !opts[:final].to_s.strip.empty?,
|
|
209
|
+
details: opts[:final].to_s[0, 300],
|
|
210
|
+
session_id: session_id,
|
|
211
|
+
tags: %w[auto loop]
|
|
212
|
+
)
|
|
213
|
+
reflect(session_id: session_id)
|
|
214
|
+
rescue StandardError => e
|
|
215
|
+
warn "[pwn-ai/learning] auto_reflect swallowed: #{e.class}: #{e.message}"
|
|
216
|
+
nil
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Supported Method Parameters::
|
|
220
|
+
# removed = PWN::AI::Agent::Learning.consolidate(
|
|
221
|
+
# max_entries: 'optional - hard cap on PWN::Memory size (default MAX_MEMORY_ENTRIES)'
|
|
222
|
+
# )
|
|
223
|
+
#
|
|
224
|
+
# Deduplicates near-identical lesson values and prunes the oldest
|
|
225
|
+
# entries once the cap is exceeded so the injected MEMORY block
|
|
226
|
+
# stays high-signal.
|
|
227
|
+
|
|
228
|
+
public_class_method def self.consolidate(opts = {})
|
|
229
|
+
cap = opts[:max_entries] || MAX_MEMORY_ENTRIES
|
|
230
|
+
return { removed: 0 } unless defined?(PWN::Memory)
|
|
231
|
+
|
|
232
|
+
mem = PWN::Memory.load
|
|
233
|
+
seen = {}
|
|
234
|
+
removed = []
|
|
235
|
+
mem.each do |k, v|
|
|
236
|
+
sig = Digest::SHA256.hexdigest(v[:value].to_s.strip.downcase)[0, 16]
|
|
237
|
+
if seen[sig]
|
|
238
|
+
removed << k
|
|
239
|
+
else
|
|
240
|
+
seen[sig] = k
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
removed.each { |k| mem.delete(k) }
|
|
244
|
+
|
|
245
|
+
if mem.size > cap
|
|
246
|
+
sorted = mem.sort_by { |_k, v| v[:timestamp].to_s }
|
|
247
|
+
drop = sorted.first(mem.size - cap).map(&:first)
|
|
248
|
+
drop.each { |k| mem.delete(k) }
|
|
249
|
+
removed.concat(drop)
|
|
250
|
+
end
|
|
251
|
+
PWN::Memory.save(mem: mem)
|
|
252
|
+
{ removed: removed.length, remaining: mem.size }
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Supported Method Parameters::
|
|
256
|
+
# PWN::AI::Agent::Learning.reset
|
|
257
|
+
|
|
258
|
+
public_class_method def self.reset
|
|
259
|
+
FileUtils.rm_f(LEARNING_FILE)
|
|
260
|
+
{ cleared: true }
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# -------------------------------------------------------------
|
|
264
|
+
# privates
|
|
265
|
+
# -------------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
private_class_method def self.auto_reflect_enabled?
|
|
268
|
+
return false unless defined?(PWN::Env) && PWN::Env.is_a?(Hash)
|
|
269
|
+
|
|
270
|
+
PWN::Env.dig(:ai, :agent, :auto_reflect) ? true : false
|
|
271
|
+
rescue StandardError
|
|
272
|
+
false
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
private_class_method def self.skills_dir
|
|
276
|
+
if defined?(PWN::Config) && PWN::Config.respond_to?(:pwn_skills_path)
|
|
277
|
+
PWN::Config.pwn_skills_path
|
|
278
|
+
else
|
|
279
|
+
File.join(Dir.home, '.pwn', 'skills')
|
|
280
|
+
end
|
|
281
|
+
rescue StandardError
|
|
282
|
+
File.join(Dir.home, '.pwn', 'skills')
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
private_class_method def self.transcript_text(opts = {})
|
|
286
|
+
transcript = opts[:transcript] || []
|
|
287
|
+
transcript.map do |e|
|
|
288
|
+
"[#{e[:role]}] #{e[:content].to_s.gsub(/\s+/, ' ')[0, 400]}"
|
|
289
|
+
end.join("\n")
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
private_class_method def self.introspective_lessons(opts = {})
|
|
293
|
+
transcript = opts[:transcript] || []
|
|
294
|
+
return [] unless defined?(PWN::AI::Introspection)
|
|
295
|
+
return [] unless defined?(PWN::Env) && PWN::Env.is_a?(Hash) && PWN::Env.dig(:ai, :introspection)
|
|
296
|
+
|
|
297
|
+
req = "Analyse this pwn-ai session transcript and emit up to 5 durable, generalizable lessons (one per line, no numbering, imperative voice) that would make future runs faster or more reliable. Focus on tool selection, error recovery, and target-agnostic technique. Ignore trivia.\n\nTRANSCRIPT:\n#{transcript_text(transcript: transcript)}"
|
|
298
|
+
resp = PWN::AI::Introspection.reflect_on(request: req, suppress_pii_warning: true)
|
|
299
|
+
resp.to_s.lines.map(&:strip).reject(&:empty?).first(5)
|
|
300
|
+
rescue StandardError
|
|
301
|
+
[]
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
private_class_method def self.heuristic_lessons(opts = {})
|
|
305
|
+
transcript = opts[:transcript] || []
|
|
306
|
+
lessons = []
|
|
307
|
+
transcript.each do |e|
|
|
308
|
+
c = e[:content].to_s
|
|
309
|
+
next unless e[:role].to_s == 'tool'
|
|
310
|
+
|
|
311
|
+
if c.include?('"success":false') || c.match?(/error|Error|ERROR/)
|
|
312
|
+
tool = c[/^(\w+) →/, 1] || c[/"error":"([^"]{5,120})"/, 1] || 'tool'
|
|
313
|
+
lessons << "Avoid repeating failure pattern from #{tool}: #{c[0, 160]}"
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
if lessons.empty?
|
|
317
|
+
asst = transcript.rfind { |e| e[:role].to_s == 'assistant' }
|
|
318
|
+
lessons << "Approach that worked: #{asst[:content].to_s.strip[0, 200]}" if asst && !asst[:content].to_s.strip.empty?
|
|
319
|
+
end
|
|
320
|
+
lessons.uniq.first(5)
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
private_class_method def self.build_skill_from_session(opts = {})
|
|
324
|
+
session_id = opts[:session_id]
|
|
325
|
+
name = opts[:name]
|
|
326
|
+
transcript = PWN::Sessions.load(session_id: session_id)
|
|
327
|
+
steps = transcript.select { |e| %w[tool assistant].include?(e[:role].to_s) }
|
|
328
|
+
.map { |e| "- **#{e[:role]}**: #{e[:content].to_s.strip[0, 300]}" }
|
|
329
|
+
user = transcript.find { |e| e[:role].to_s == 'user' }
|
|
330
|
+
goal = user ? user[:content].to_s.strip[0, 200] : name
|
|
331
|
+
<<~MD
|
|
332
|
+
# #{name.tr('_-', ' ').capitalize}
|
|
333
|
+
|
|
334
|
+
_Auto-distilled by PWN::AI::Agent::Learning from session `#{session_id}` on #{Time.now.utc.iso8601}._
|
|
335
|
+
|
|
336
|
+
## Goal
|
|
337
|
+
#{goal}
|
|
338
|
+
|
|
339
|
+
## Observed Procedure
|
|
340
|
+
#{steps.join("\n")}
|
|
341
|
+
|
|
342
|
+
## Notes
|
|
343
|
+
Refine this skill by editing #{name}.md under ~/.pwn/skills.
|
|
344
|
+
MD
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# Author(s):: 0day Inc. <support@0dayinc.com>
|
|
348
|
+
|
|
349
|
+
public_class_method def self.authors
|
|
350
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# Display Usage for this Module
|
|
354
|
+
|
|
355
|
+
public_class_method def self.help
|
|
356
|
+
puts <<~USAGE
|
|
357
|
+
USAGE:
|
|
358
|
+
PWN::AI::Agent::Learning.note_outcome(task: 'nmap sweep 10.0.0.0/24', success: true, details: '12 hosts up')
|
|
359
|
+
PWN::AI::Agent::Learning.outcomes(limit: 20, success: false)
|
|
360
|
+
PWN::AI::Agent::Learning.reflect(session_id: sid) # LLM or heuristic → PWN::Memory
|
|
361
|
+
PWN::AI::Agent::Learning.auto_reflect(session_id: sid, request: req, final: text)
|
|
362
|
+
PWN::AI::Agent::Learning.distill_skill(name: 'quick_recon', session_id: sid)
|
|
363
|
+
PWN::AI::Agent::Learning.consolidate(max_entries: 200) # dedupe + prune Memory
|
|
364
|
+
PWN::AI::Agent::Learning.to_context(limit: 5) # injected by PromptBuilder
|
|
365
|
+
PWN::AI::Agent::Learning.stats
|
|
366
|
+
PWN::AI::Agent::Learning.reset
|
|
367
|
+
|
|
368
|
+
Enable end-of-run auto-learning with:
|
|
369
|
+
PWN::Env[:ai][:agent][:auto_reflect] = true
|
|
370
|
+
|
|
371
|
+
#{self}.authors
|
|
372
|
+
USAGE
|
|
373
|
+
end
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
end
|
|
377
|
+
end
|
data/lib/pwn/ai/agent/loop.rb
CHANGED
|
@@ -49,6 +49,18 @@ module PWN
|
|
|
49
49
|
DEFAULT_MAX_ITERS
|
|
50
50
|
end
|
|
51
51
|
|
|
52
|
+
private_class_method def self.record_metrics(opts = {})
|
|
53
|
+
name = opts[:name]
|
|
54
|
+
started = opts[:started]
|
|
55
|
+
raw = opts[:raw].to_s
|
|
56
|
+
ok = raw.include?('"success":true')
|
|
57
|
+
err = raw[/"error":"([^"]{1,300})"/, 1]
|
|
58
|
+
dur = started ? (Time.now - started) : 0.0
|
|
59
|
+
Metrics.record(name: name, success: ok, duration: dur, error: err) if defined?(Metrics)
|
|
60
|
+
rescue StandardError
|
|
61
|
+
nil
|
|
62
|
+
end
|
|
63
|
+
|
|
52
64
|
private_class_method def self.append_session(opts = {})
|
|
53
65
|
session_id = opts[:session_id]
|
|
54
66
|
return unless session_id && defined?(PWN::Sessions)
|
|
@@ -163,13 +175,16 @@ module PWN
|
|
|
163
175
|
if calls.empty?
|
|
164
176
|
text = msg[:content].to_s
|
|
165
177
|
append_session(session_id: session_id, role: 'assistant', content: text)
|
|
178
|
+
Learning.auto_reflect(session_id: session_id, request: request, final: text) if defined?(Learning)
|
|
166
179
|
return text
|
|
167
180
|
end
|
|
168
181
|
|
|
169
182
|
calls.each do |tc|
|
|
170
|
-
name
|
|
171
|
-
entry
|
|
172
|
-
|
|
183
|
+
name = tc.dig(:function, :name).to_s
|
|
184
|
+
entry = Registry.lookup(name: name)
|
|
185
|
+
started = Time.now
|
|
186
|
+
raw = Dispatch.call(tool_call: tc)
|
|
187
|
+
record_metrics(name: name, started: started, raw: raw)
|
|
173
188
|
result = Result.condition(content: raw, entry: entry)
|
|
174
189
|
|
|
175
190
|
on_tool&.call(name, tc.dig(:function, :arguments), result)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'time'
|
|
6
|
+
|
|
7
|
+
module PWN
|
|
8
|
+
module AI
|
|
9
|
+
module Agent
|
|
10
|
+
# PWN::AI::Agent::Metrics is the telemetry layer of the pwn-ai
|
|
11
|
+
# learning loop. Every tool dispatch performed by
|
|
12
|
+
# PWN::AI::Agent::Loop is recorded here (name, success, duration,
|
|
13
|
+
# last error) and persisted to ~/.pwn/metrics.json.
|
|
14
|
+
#
|
|
15
|
+
# PromptBuilder re-injects a compact effectiveness summary into the
|
|
16
|
+
# system prompt on every turn, so the model gains awareness of which
|
|
17
|
+
# tools historically succeed vs. fail on THIS host and can adapt its
|
|
18
|
+
# tool selection accordingly. This is one half of the closed
|
|
19
|
+
# feedback loop that lets pwn-ai continuously make itself smarter
|
|
20
|
+
# (the other half is PWN::AI::Agent::Learning).
|
|
21
|
+
module Metrics
|
|
22
|
+
METRICS_FILE = File.join(Dir.home, '.pwn', 'metrics.json')
|
|
23
|
+
|
|
24
|
+
# Supported Method Parameters::
|
|
25
|
+
# metrics = PWN::AI::Agent::Metrics.load
|
|
26
|
+
|
|
27
|
+
public_class_method def self.load
|
|
28
|
+
FileUtils.mkdir_p(File.dirname(METRICS_FILE))
|
|
29
|
+
return { tools: {}, updated_at: nil } unless File.exist?(METRICS_FILE)
|
|
30
|
+
|
|
31
|
+
JSON.parse(File.read(METRICS_FILE), symbolize_names: true)
|
|
32
|
+
rescue StandardError
|
|
33
|
+
{ tools: {}, updated_at: nil }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Supported Method Parameters::
|
|
37
|
+
# PWN::AI::Agent::Metrics.save(
|
|
38
|
+
# metrics: 'required - Hash returned by .load / mutated in place'
|
|
39
|
+
# )
|
|
40
|
+
|
|
41
|
+
public_class_method def self.save(opts = {})
|
|
42
|
+
metrics = opts[:metrics] ||= { tools: {} }
|
|
43
|
+
metrics[:updated_at] = Time.now.utc.iso8601
|
|
44
|
+
FileUtils.mkdir_p(File.dirname(METRICS_FILE))
|
|
45
|
+
File.write(METRICS_FILE, JSON.pretty_generate(metrics))
|
|
46
|
+
metrics
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Supported Method Parameters::
|
|
50
|
+
# PWN::AI::Agent::Metrics.record(
|
|
51
|
+
# name: 'required - tool name that was dispatched',
|
|
52
|
+
# success: 'required - Boolean, did the handler complete without error',
|
|
53
|
+
# duration: 'optional - Float seconds the dispatch took',
|
|
54
|
+
# error: 'optional - String error message when success is false'
|
|
55
|
+
# )
|
|
56
|
+
|
|
57
|
+
public_class_method def self.record(opts = {})
|
|
58
|
+
name = opts[:name].to_s
|
|
59
|
+
success = opts[:success] ? true : false
|
|
60
|
+
duration = opts[:duration].to_f
|
|
61
|
+
error = opts[:error]
|
|
62
|
+
return if name.empty?
|
|
63
|
+
|
|
64
|
+
metrics = load
|
|
65
|
+
metrics[:tools] ||= {}
|
|
66
|
+
key = name.to_sym
|
|
67
|
+
t = metrics[:tools][key] ||= {
|
|
68
|
+
calls: 0, ok: 0, fail: 0, total_duration: 0.0,
|
|
69
|
+
last_error: nil, last_at: nil
|
|
70
|
+
}
|
|
71
|
+
t[:calls] += 1
|
|
72
|
+
t[:ok] += 1 if success
|
|
73
|
+
t[:fail] += 1 unless success
|
|
74
|
+
t[:total_duration] += duration
|
|
75
|
+
t[:last_error] = error.to_s[0, 300] if error && !success
|
|
76
|
+
t[:last_at] = Time.now.utc.iso8601
|
|
77
|
+
save(metrics: metrics)
|
|
78
|
+
t
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Supported Method Parameters::
|
|
82
|
+
# rows = PWN::AI::Agent::Metrics.summary(
|
|
83
|
+
# limit: 'optional - cap number of tools returned (default 25)'
|
|
84
|
+
# )
|
|
85
|
+
|
|
86
|
+
public_class_method def self.summary(opts = {})
|
|
87
|
+
limit = opts[:limit] || 25
|
|
88
|
+
tools = load[:tools] || {}
|
|
89
|
+
rows = tools.map do |name, t|
|
|
90
|
+
calls = t[:calls].to_i
|
|
91
|
+
ok = t[:ok].to_i
|
|
92
|
+
rate = calls.positive? ? (ok.to_f / calls).round(3) : 0.0
|
|
93
|
+
avg = calls.positive? ? (t[:total_duration].to_f / calls).round(3) : 0.0
|
|
94
|
+
{
|
|
95
|
+
name: name.to_s,
|
|
96
|
+
calls: calls,
|
|
97
|
+
success_rate: rate,
|
|
98
|
+
avg_duration: avg,
|
|
99
|
+
last_error: t[:last_error],
|
|
100
|
+
last_at: t[:last_at]
|
|
101
|
+
}
|
|
102
|
+
end
|
|
103
|
+
rows.sort_by { |r| [-r[:calls], -r[:success_rate]] }.first(limit)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Supported Method Parameters::
|
|
107
|
+
# ctx = PWN::AI::Agent::Metrics.to_context(
|
|
108
|
+
# limit: 'optional - cap number of tools included (default 8)'
|
|
109
|
+
# )
|
|
110
|
+
|
|
111
|
+
public_class_method def self.to_context(opts = {})
|
|
112
|
+
limit = opts[:limit] || 8
|
|
113
|
+
rows = summary(limit: limit)
|
|
114
|
+
return '' if rows.empty?
|
|
115
|
+
|
|
116
|
+
lines = rows.map do |r|
|
|
117
|
+
err = r[:last_error] ? " last_err=#{r[:last_error][0, 60]}" : ''
|
|
118
|
+
" - #{r[:name]}: calls=#{r[:calls]} success=#{(r[:success_rate] * 100).round(1)}% avg=#{r[:avg_duration]}s#{err}"
|
|
119
|
+
end
|
|
120
|
+
"TOOL EFFECTIVENESS (historical, adapt tool choice accordingly)\n#{lines.join("\n")}\n\n"
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Supported Method Parameters::
|
|
124
|
+
# PWN::AI::Agent::Metrics.reset
|
|
125
|
+
|
|
126
|
+
public_class_method def self.reset
|
|
127
|
+
FileUtils.rm_f(METRICS_FILE)
|
|
128
|
+
{ tools: {}, updated_at: nil }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Author(s):: 0day Inc. <support@0dayinc.com>
|
|
132
|
+
|
|
133
|
+
public_class_method def self.authors
|
|
134
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Display Usage for this Module
|
|
138
|
+
|
|
139
|
+
public_class_method def self.help
|
|
140
|
+
puts <<~USAGE
|
|
141
|
+
USAGE:
|
|
142
|
+
PWN::AI::Agent::Metrics.record(name: 'shell', success: true, duration: 0.42)
|
|
143
|
+
PWN::AI::Agent::Metrics.summary(limit: 10)
|
|
144
|
+
PWN::AI::Agent::Metrics.to_context(limit: 8) # injected by PromptBuilder
|
|
145
|
+
PWN::AI::Agent::Metrics.reset
|
|
146
|
+
PWN::AI::Agent::Metrics.load
|
|
147
|
+
PWN::AI::Agent::Metrics.save(metrics: hash)
|
|
148
|
+
|
|
149
|
+
#{self}.authors
|
|
150
|
+
USAGE
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|