pwn 0.5.613 → 0.5.615

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +10 -10
  3. data/README.md +110 -24
  4. data/documentation/AI-Integration.md +36 -0
  5. data/documentation/BurpSuite.md +26 -0
  6. data/documentation/Contributing.md +31 -0
  7. data/documentation/Diagrams.md +57 -0
  8. data/documentation/Drivers.md +25 -0
  9. data/documentation/General-PWN-Usage.md +40 -0
  10. data/documentation/Home.md +51 -0
  11. data/documentation/How-PWN-Works.md +60 -0
  12. data/documentation/Installation.md +50 -0
  13. data/documentation/NmapIt.md +18 -0
  14. data/documentation/Plugins.md +59 -0
  15. data/documentation/Reporting.md +28 -0
  16. data/documentation/SAST.md +32 -0
  17. data/documentation/Skills-Memory-Learning.md +44 -0
  18. data/documentation/Transparent-Browser.md +32 -0
  19. data/documentation/Troubleshooting.md +38 -0
  20. data/documentation/What-is-PWN.md +34 -0
  21. data/documentation/Why-PWN.md +18 -0
  22. data/documentation/diagrams/ai-integration-tool-calling.svg +134 -0
  23. data/documentation/diagrams/burp-vs-zap-preference.svg +84 -0
  24. data/documentation/diagrams/code-scanning-sast.svg +112 -0
  25. data/documentation/diagrams/dot/ai-integration-tool-calling.dot +23 -0
  26. data/documentation/diagrams/dot/burp-vs-zap-preference.dot +15 -0
  27. data/documentation/diagrams/dot/code-scanning-sast.dot +21 -0
  28. data/documentation/diagrams/dot/driver-framework.dot +17 -0
  29. data/documentation/diagrams/dot/fuzzing-workflow.dot +27 -0
  30. data/documentation/diagrams/dot/history-to-drivers.dot +20 -0
  31. data/documentation/diagrams/dot/memory-skills-detailed.dot +28 -0
  32. data/documentation/diagrams/dot/network-infra-testing.dot +22 -0
  33. data/documentation/diagrams/dot/overall-pwn-architecture.dot +49 -0
  34. data/documentation/diagrams/dot/penetration-testing-workflow.dot +29 -0
  35. data/documentation/diagrams/dot/plugin-ecosystem.dot +31 -0
  36. data/documentation/diagrams/dot/pwn-ai-feedback-learning-loop.dot +57 -0
  37. data/documentation/diagrams/dot/pwn-repl-prototyping.dot +37 -0
  38. data/documentation/diagrams/dot/reporting-pipeline.dot +17 -0
  39. data/documentation/diagrams/dot/reverse-engineering-flow.dot +21 -0
  40. data/documentation/diagrams/dot/sessions-cron-automation.dot +17 -0
  41. data/documentation/diagrams/dot/web-application-testing.dot +27 -0
  42. data/documentation/diagrams/dot/zero-day-research-flow.dot +27 -0
  43. data/documentation/diagrams/driver-framework.svg +90 -0
  44. data/documentation/diagrams/fuzzing-workflow.svg +147 -0
  45. data/documentation/diagrams/history-to-drivers.svg +117 -0
  46. data/documentation/diagrams/memory-skills-detailed.svg +154 -0
  47. data/documentation/diagrams/network-infra-testing.svg +121 -0
  48. data/documentation/diagrams/overall-pwn-architecture.svg +234 -0
  49. data/documentation/diagrams/penetration-testing-workflow.svg +148 -0
  50. data/documentation/diagrams/plugin-ecosystem.svg +172 -0
  51. data/documentation/diagrams/pwn-ai-feedback-learning-loop.svg +234 -0
  52. data/documentation/diagrams/pwn-repl-prototyping.svg +179 -0
  53. data/documentation/diagrams/reporting-pipeline.svg +91 -0
  54. data/documentation/diagrams/reverse-engineering-flow.svg +114 -0
  55. data/documentation/diagrams/sessions-cron-automation.svg +92 -0
  56. data/documentation/diagrams/web-application-testing.svg +137 -0
  57. data/documentation/diagrams/zero-day-research-flow.svg +149 -0
  58. data/documentation/pwn-REPL.md +33 -0
  59. data/documentation/pwn-ai-Agent.md +43 -0
  60. data/lib/pwn/ai/agent/learning.rb +377 -0
  61. data/lib/pwn/ai/agent/loop.rb +18 -3
  62. data/lib/pwn/ai/agent/metrics.rb +155 -0
  63. data/lib/pwn/ai/agent/prompt_builder.rb +23 -3
  64. data/lib/pwn/ai/agent/tools/learning.rb +113 -0
  65. data/lib/pwn/ai/agent/tools/skills.rb +94 -8
  66. data/lib/pwn/ai/agent.rb +2 -0
  67. data/lib/pwn/ai/grok.rb +249 -106
  68. data/lib/pwn/config.rb +86 -13
  69. data/lib/pwn/plugins/monkey_patch.rb +2 -2
  70. data/lib/pwn/plugins/repl.rb +68 -3
  71. data/lib/pwn/version.rb +1 -1
  72. data/spec/lib/pwn/ai/agent/learning_spec.rb +68 -0
  73. data/spec/lib/pwn/ai/agent/metrics_spec.rb +27 -0
  74. data/spec/lib/pwn/ai/agent/tools/learning_spec.rb +30 -0
  75. metadata +84 -34
  76. data/documentation/PWN.png +0 -0
  77. data/documentation/PWN_Contributors_and_Users.png +0 -0
  78. data/documentation/fax-spectrogram.png +0 -0
  79. data/documentation/fax-waveform.png +0 -0
  80. data/documentation/frozen_string_literal_files_changed_from_false_to_true.txt +0 -54
  81. data/documentation/lifecycle_authz_replay.example.yaml +0 -27
  82. data/documentation/pwn_android_war_dialer_session.png +0 -0
  83. data/documentation/pwn_install.png +0 -0
  84. data/documentation/pwn_wallpaper.jpg +0 -0
  85. data/documentation/ringing-spectrogram.png +0 -0
  86. data/documentation/ringing-waveform.png +0 -0
  87. data/documentation/vulnerability_report_template.md +0 -37
@@ -0,0 +1,33 @@
1
+ # The pwn REPL
2
+
3
+ The `pwn` command launches a Pry-based interactive Ruby shell with the entire `PWN` namespace pre-loaded.
4
+
5
+ ## Starting
6
+
7
+ ```bash
8
+ pwn
9
+ pwn[v0.5.613]:001 >>>
10
+ ```
11
+
12
+ ## Key Features
13
+
14
+ - Direct access to every `PWN::` constant and plugin.
15
+ - Tab completion for classes and methods.
16
+ - Multi-line support.
17
+ - Easy prototyping of security workflows.
18
+ - `PWN.help` and inspection of any object.
19
+
20
+ ## Useful REPL Commands
21
+
22
+ ```
23
+ pwn[v0.5.613]:001 >>> PWN::Plugins.constants.sort
24
+ pwn[v0.5.613]:001 >>> PWN::Plugins::BurpSuite.methods(false).sort
25
+ pwn[v0.5.613]:001 >>> PWN::SAST.constants
26
+ pwn[v0.5.613]:001 >>> pwn-ai
27
+ ```
28
+
29
+ See:
30
+ - [pwn-ai Agent](pwn-ai-Agent.md) — the killer feature inside the REPL
31
+ - [General Usage](General-PWN-Usage.md)
32
+
33
+ [[Diagrams]]
@@ -0,0 +1,43 @@
1
+ # pwn-ai Autonomous Agent
2
+
3
+ The most powerful way to use PWN is via the `pwn-ai` command inside the REPL.
4
+
5
+ ## Activation
6
+
7
+ ```bash
8
+ pwn[v0.5.613]:001 >>> pwn-ai
9
+ [*] pwn-ai agent TUI activated...
10
+ >
11
+ ```
12
+
13
+ ## Capabilities
14
+
15
+ - Natural language instruction of complex multi-step security tasks.
16
+ - Full tool calling access to:
17
+ - All `PWN::Plugins` (BurpSuite preferred, TransparentBrowser, NmapIt, Shodan, Metasploit, etc.)
18
+ - `PWN::SAST`
19
+ - `PWN::Reports`
20
+ - Shell execution
21
+ - Memory recall / remember
22
+ - Skill usage and distillation
23
+ - Learning / introspection loop
24
+ - Persistent context across interactions via memory and skills.
25
+
26
+ ## Tips
27
+
28
+ - Use `SHIFT+ENTER` to insert newlines.
29
+ - `ENTER` submits the prompt.
30
+ - Type `back` or `exit` to return to normal REPL.
31
+ - Example prompt:
32
+ ```
33
+ Scan https://target with NmapIt + TransparentBrowser (via BurpSuite),
34
+ run relevant SAST if source present, exploit any findings, generate report.
35
+ ```
36
+
37
+ ## Self-Improvement
38
+
39
+ Successful workflows can be distilled into reusable **Skills** (see [Skills-Memory-Learning](Skills-Memory-Learning.md)).
40
+
41
+ See also [AI Integration](AI-Integration.md).
42
+
43
+ [[Diagrams]]
@@ -0,0 +1,377 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'fileutils'
5
+ require 'time'
6
+ require 'digest'
7
+
8
+ module PWN
9
+ module AI
10
+ module Agent
11
+ # PWN::AI::Agent::Learning is the self-improvement engine that closes
12
+ # the pwn-ai feedback loop. It captures task outcomes, mines session
13
+ # transcripts for durable lessons, promotes successful workflows into
14
+ # reusable skills, and prunes / consolidates persistent memory so the
15
+ # agent gets sharper over time instead of accumulating noise.
16
+ #
17
+ # Data flows:
18
+ # Loop.run --(tool telemetry)--> Metrics.record
19
+ # Loop.run --(final answer)----> Learning.auto_reflect (opt-in)
20
+ # model --(tool calls)------> learning_note_outcome / _distill_skill
21
+ # PromptBuilder <----------------- Learning.to_context + Metrics.to_context
22
+ #
23
+ # Everything is file-backed under ~/.pwn so it survives across REPL
24
+ # restarts and is shared by every future session.
25
+ module Learning
26
+ LEARNING_FILE = File.join(Dir.home, '.pwn', 'learning.jsonl')
27
+ MAX_MEMORY_ENTRIES = 200
28
+
29
+ # Supported Method Parameters::
30
+ # entry = PWN::AI::Agent::Learning.note_outcome(
31
+ # task: 'required - short description of what was attempted',
32
+ # success: 'required - Boolean, did the attempt achieve its goal',
33
+ # details: 'optional - free-form notes / error / evidence',
34
+ # session_id: 'optional - PWN::Sessions id this outcome belongs to',
35
+ # tags: 'optional - Array of String labels for later retrieval'
36
+ # )
37
+
38
+ public_class_method def self.note_outcome(opts = {})
39
+ task = opts[:task].to_s
40
+ success = opts[:success] ? true : false
41
+ raise 'ERROR: task is required' if task.strip.empty?
42
+
43
+ entry = {
44
+ id: Digest::SHA256.hexdigest("#{task}-#{Time.now.to_f}")[0, 12],
45
+ task: task,
46
+ success: success,
47
+ details: opts[:details].to_s[0, 2_000],
48
+ session_id: opts[:session_id],
49
+ tags: Array(opts[:tags]).map(&:to_s),
50
+ timestamp: Time.now.utc.iso8601
51
+ }
52
+ FileUtils.mkdir_p(File.dirname(LEARNING_FILE))
53
+ File.open(LEARNING_FILE, 'a') { |f| f.puts(JSON.generate(entry)) }
54
+
55
+ key = :"lesson_#{entry[:id]}"
56
+ cat = :lesson
57
+ val = "#{success ? 'SUCCESS' : 'FAILURE'}: #{task} — #{opts[:details].to_s.strip[0, 200]}"
58
+ PWN::Memory.remember(key: key, value: val, category: cat) if defined?(PWN::Memory)
59
+
60
+ entry
61
+ end
62
+
63
+ # Supported Method Parameters::
64
+ # rows = PWN::AI::Agent::Learning.outcomes(
65
+ # limit: 'optional - max entries returned newest-first (default 50)',
66
+ # success: 'optional - filter by Boolean outcome',
67
+ # tag: 'optional - filter by tag substring'
68
+ # )
69
+
70
+ public_class_method def self.outcomes(opts = {})
71
+ limit = opts[:limit] || 50
72
+ want_ok = opts.key?(:success) ? !opts[:success].nil? && opts[:success] != false : nil
73
+ tag = opts[:tag].to_s.downcase
74
+ return [] unless File.exist?(LEARNING_FILE)
75
+
76
+ rows = File.readlines(LEARNING_FILE).map do |l|
77
+ JSON.parse(l, symbolize_names: true)
78
+ rescue StandardError
79
+ nil
80
+ end
81
+ rows.compact!
82
+ rows.select! { |r| r[:success] == want_ok } unless want_ok.nil?
83
+ rows.select! { |r| Array(r[:tags]).any? { |t| t.to_s.downcase.include?(tag) } } unless tag.empty?
84
+ rows.reverse.first(limit)
85
+ end
86
+
87
+ # Supported Method Parameters::
88
+ # stats = PWN::AI::Agent::Learning.stats
89
+
90
+ public_class_method def self.stats
91
+ rows = outcomes(limit: 10_000)
92
+ total = rows.length
93
+ ok = rows.count { |r| r[:success] }
94
+ skills = defined?(PWN::Skills) && PWN::Skills.is_a?(Hash) ? PWN::Skills.keys.length : 0
95
+ mem = defined?(PWN::Memory) ? PWN::Memory.load.keys.length : 0
96
+ {
97
+ total_outcomes: total,
98
+ successes: ok,
99
+ failures: total - ok,
100
+ success_rate: total.positive? ? (ok.to_f / total).round(3) : 0.0,
101
+ skills_known: skills,
102
+ memory_entries: mem,
103
+ tool_metrics: (Metrics.summary(limit: 5) if defined?(Metrics))
104
+ }
105
+ end
106
+
107
+ # Supported Method Parameters::
108
+ # ctx = PWN::AI::Agent::Learning.to_context(
109
+ # limit: 'optional - number of recent outcomes to surface (default 5)'
110
+ # )
111
+
112
+ public_class_method def self.to_context(opts = {})
113
+ limit = opts[:limit] || 5
114
+ rows = outcomes(limit: limit)
115
+ return '' if rows.empty?
116
+
117
+ lines = rows.map do |r|
118
+ flag = r[:success] ? '✓' : '✗'
119
+ " #{flag} #{r[:task][0, 100]} (#{r[:timestamp]})"
120
+ end
121
+ s = stats
122
+ hdr = "RECENT OUTCOMES (success_rate=#{(s[:success_rate] * 100).round(1)}% over #{s[:total_outcomes]} attempts)"
123
+ "#{hdr}\n#{lines.join("\n")}\n\n"
124
+ end
125
+
126
+ # Supported Method Parameters::
127
+ # skill = PWN::AI::Agent::Learning.distill_skill(
128
+ # name: 'required - snake_case name for the new skill',
129
+ # session_id: 'optional - PWN::Sessions id to mine (uses its transcript)',
130
+ # content: 'optional - explicit markdown body; overrides transcript mining',
131
+ # references: 'optional - Array of reference URLs / CWE / CVE / ATT&CK ids'
132
+ # )
133
+
134
+ public_class_method def self.distill_skill(opts = {})
135
+ name = opts[:name].to_s.gsub(/[^a-z0-9_-]/i, '_')
136
+ raise 'ERROR: name is required' if name.empty?
137
+
138
+ body = opts[:content].to_s
139
+ body = build_skill_from_session(session_id: opts[:session_id], name: name) if body.strip.empty? && opts[:session_id]
140
+ raise 'ERROR: content or session_id is required' if body.strip.empty?
141
+
142
+ refs = Array(opts[:references]).map(&:to_s).map(&:strip).reject(&:empty?).uniq
143
+ unless refs.empty?
144
+ body = "---\nreferences:\n#{refs.map { |r| " - #{r}" }.join("\n")}\n---\n#{body}" unless body.start_with?("---\n")
145
+ body = "#{body.rstrip}\n\n## References\n#{refs.map { |r| "- #{r}" }.join("\n")}\n" unless body =~ /^\#{1,3}\s*References\s*$/i
146
+ end
147
+
148
+ dir = skills_dir
149
+ FileUtils.mkdir_p(dir)
150
+ path = File.join(dir, "#{name}.md")
151
+ File.write(path, body)
152
+ PWN::Config.load_skills(pwn_skills_path: dir) if defined?(PWN::Config) && PWN::Config.respond_to?(:load_skills)
153
+ note_outcome(task: "distill_skill:#{name}", success: true, details: "Saved #{path}", tags: %w[skill auto])
154
+ { saved: true, name: name, path: path, bytes: body.bytesize, references: refs }
155
+ end
156
+
157
+ # Supported Method Parameters::
158
+ # report = PWN::AI::Agent::Learning.reflect(
159
+ # session_id: 'required - PWN::Sessions id to analyse',
160
+ # dry_run: 'optional - when true, do not write to Memory/Skills (default false)'
161
+ # )
162
+ #
163
+ # Uses PWN::AI::Introspection (when available) to LLM-summarise the
164
+ # session into structured lessons. Falls back to a heuristic
165
+ # extractor when introspection is disabled so learning never stops.
166
+
167
+ public_class_method def self.reflect(opts = {})
168
+ session_id = opts[:session_id]
169
+ dry_run = opts[:dry_run] ? true : false
170
+ raise 'ERROR: session_id is required' if session_id.to_s.empty?
171
+
172
+ transcript = PWN::Sessions.load(session_id: session_id)
173
+ return { session_id: session_id, lessons: [], reason: 'empty transcript' } if transcript.empty?
174
+
175
+ lessons = introspective_lessons(transcript: transcript)
176
+ lessons = heuristic_lessons(transcript: transcript) if lessons.empty?
177
+
178
+ saved = []
179
+ lessons.each do |l|
180
+ next if l.to_s.strip.empty?
181
+
182
+ key = :"reflect_#{session_id}_#{Digest::SHA256.hexdigest(l)[0, 8]}"
183
+ PWN::Memory.remember(key: key, value: l, category: :lesson) unless dry_run
184
+ saved << { key: key, lesson: l }
185
+ end
186
+ consolidate unless dry_run
187
+
188
+ { session_id: session_id, lessons: saved, count: saved.length, dry_run: dry_run }
189
+ end
190
+
191
+ # Supported Method Parameters::
192
+ # PWN::AI::Agent::Learning.auto_reflect(
193
+ # session_id: 'required - id of the just-completed session',
194
+ # request: 'optional - original user request (for outcome logging)',
195
+ # final: 'optional - final assistant answer (for outcome logging)'
196
+ # )
197
+ #
198
+ # Called by Loop.run when PWN::Env[:ai][:agent][:auto_reflect] is
199
+ # truthy. Never raises — learning must not break the primary loop.
200
+
201
+ public_class_method def self.auto_reflect(opts = {})
202
+ session_id = opts[:session_id]
203
+ return unless session_id
204
+ return unless auto_reflect_enabled?
205
+
206
+ note_outcome(
207
+ task: opts[:request].to_s[0, 120],
208
+ success: !opts[:final].to_s.strip.empty?,
209
+ details: opts[:final].to_s[0, 300],
210
+ session_id: session_id,
211
+ tags: %w[auto loop]
212
+ )
213
+ reflect(session_id: session_id)
214
+ rescue StandardError => e
215
+ warn "[pwn-ai/learning] auto_reflect swallowed: #{e.class}: #{e.message}"
216
+ nil
217
+ end
218
+
219
+ # Supported Method Parameters::
220
+ # removed = PWN::AI::Agent::Learning.consolidate(
221
+ # max_entries: 'optional - hard cap on PWN::Memory size (default MAX_MEMORY_ENTRIES)'
222
+ # )
223
+ #
224
+ # Deduplicates near-identical lesson values and prunes the oldest
225
+ # entries once the cap is exceeded so the injected MEMORY block
226
+ # stays high-signal.
227
+
228
+ public_class_method def self.consolidate(opts = {})
229
+ cap = opts[:max_entries] || MAX_MEMORY_ENTRIES
230
+ return { removed: 0 } unless defined?(PWN::Memory)
231
+
232
+ mem = PWN::Memory.load
233
+ seen = {}
234
+ removed = []
235
+ mem.each do |k, v|
236
+ sig = Digest::SHA256.hexdigest(v[:value].to_s.strip.downcase)[0, 16]
237
+ if seen[sig]
238
+ removed << k
239
+ else
240
+ seen[sig] = k
241
+ end
242
+ end
243
+ removed.each { |k| mem.delete(k) }
244
+
245
+ if mem.size > cap
246
+ sorted = mem.sort_by { |_k, v| v[:timestamp].to_s }
247
+ drop = sorted.first(mem.size - cap).map(&:first)
248
+ drop.each { |k| mem.delete(k) }
249
+ removed.concat(drop)
250
+ end
251
+ PWN::Memory.save(mem: mem)
252
+ { removed: removed.length, remaining: mem.size }
253
+ end
254
+
255
+ # Supported Method Parameters::
256
+ # PWN::AI::Agent::Learning.reset
257
+
258
+ public_class_method def self.reset
259
+ FileUtils.rm_f(LEARNING_FILE)
260
+ { cleared: true }
261
+ end
262
+
263
+ # -------------------------------------------------------------
264
+ # privates
265
+ # -------------------------------------------------------------
266
+
267
+ private_class_method def self.auto_reflect_enabled?
268
+ return false unless defined?(PWN::Env) && PWN::Env.is_a?(Hash)
269
+
270
+ PWN::Env.dig(:ai, :agent, :auto_reflect) ? true : false
271
+ rescue StandardError
272
+ false
273
+ end
274
+
275
+ private_class_method def self.skills_dir
276
+ if defined?(PWN::Config) && PWN::Config.respond_to?(:pwn_skills_path)
277
+ PWN::Config.pwn_skills_path
278
+ else
279
+ File.join(Dir.home, '.pwn', 'skills')
280
+ end
281
+ rescue StandardError
282
+ File.join(Dir.home, '.pwn', 'skills')
283
+ end
284
+
285
+ private_class_method def self.transcript_text(opts = {})
286
+ transcript = opts[:transcript] || []
287
+ transcript.map do |e|
288
+ "[#{e[:role]}] #{e[:content].to_s.gsub(/\s+/, ' ')[0, 400]}"
289
+ end.join("\n")
290
+ end
291
+
292
+ private_class_method def self.introspective_lessons(opts = {})
293
+ transcript = opts[:transcript] || []
294
+ return [] unless defined?(PWN::AI::Introspection)
295
+ return [] unless defined?(PWN::Env) && PWN::Env.is_a?(Hash) && PWN::Env.dig(:ai, :introspection)
296
+
297
+ req = "Analyse this pwn-ai session transcript and emit up to 5 durable, generalizable lessons (one per line, no numbering, imperative voice) that would make future runs faster or more reliable. Focus on tool selection, error recovery, and target-agnostic technique. Ignore trivia.\n\nTRANSCRIPT:\n#{transcript_text(transcript: transcript)}"
298
+ resp = PWN::AI::Introspection.reflect_on(request: req, suppress_pii_warning: true)
299
+ resp.to_s.lines.map(&:strip).reject(&:empty?).first(5)
300
+ rescue StandardError
301
+ []
302
+ end
303
+
304
+ private_class_method def self.heuristic_lessons(opts = {})
305
+ transcript = opts[:transcript] || []
306
+ lessons = []
307
+ transcript.each do |e|
308
+ c = e[:content].to_s
309
+ next unless e[:role].to_s == 'tool'
310
+
311
+ if c.include?('"success":false') || c.match?(/error|Error|ERROR/)
312
+ tool = c[/^(\w+) →/, 1] || c[/"error":"([^"]{5,120})"/, 1] || 'tool'
313
+ lessons << "Avoid repeating failure pattern from #{tool}: #{c[0, 160]}"
314
+ end
315
+ end
316
+ if lessons.empty?
317
+ asst = transcript.rfind { |e| e[:role].to_s == 'assistant' }
318
+ lessons << "Approach that worked: #{asst[:content].to_s.strip[0, 200]}" if asst && !asst[:content].to_s.strip.empty?
319
+ end
320
+ lessons.uniq.first(5)
321
+ end
322
+
323
+ private_class_method def self.build_skill_from_session(opts = {})
324
+ session_id = opts[:session_id]
325
+ name = opts[:name]
326
+ transcript = PWN::Sessions.load(session_id: session_id)
327
+ steps = transcript.select { |e| %w[tool assistant].include?(e[:role].to_s) }
328
+ .map { |e| "- **#{e[:role]}**: #{e[:content].to_s.strip[0, 300]}" }
329
+ user = transcript.find { |e| e[:role].to_s == 'user' }
330
+ goal = user ? user[:content].to_s.strip[0, 200] : name
331
+ <<~MD
332
+ # #{name.tr('_-', ' ').capitalize}
333
+
334
+ _Auto-distilled by PWN::AI::Agent::Learning from session `#{session_id}` on #{Time.now.utc.iso8601}._
335
+
336
+ ## Goal
337
+ #{goal}
338
+
339
+ ## Observed Procedure
340
+ #{steps.join("\n")}
341
+
342
+ ## Notes
343
+ Refine this skill by editing #{name}.md under ~/.pwn/skills.
344
+ MD
345
+ end
346
+
347
+ # Author(s):: 0day Inc. <support@0dayinc.com>
348
+
349
+ public_class_method def self.authors
350
+ "AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
351
+ end
352
+
353
+ # Display Usage for this Module
354
+
355
+ public_class_method def self.help
356
+ puts <<~USAGE
357
+ USAGE:
358
+ PWN::AI::Agent::Learning.note_outcome(task: 'nmap sweep 10.0.0.0/24', success: true, details: '12 hosts up')
359
+ PWN::AI::Agent::Learning.outcomes(limit: 20, success: false)
360
+ PWN::AI::Agent::Learning.reflect(session_id: sid) # LLM or heuristic → PWN::Memory
361
+ PWN::AI::Agent::Learning.auto_reflect(session_id: sid, request: req, final: text)
362
+ PWN::AI::Agent::Learning.distill_skill(name: 'quick_recon', session_id: sid)
363
+ PWN::AI::Agent::Learning.consolidate(max_entries: 200) # dedupe + prune Memory
364
+ PWN::AI::Agent::Learning.to_context(limit: 5) # injected by PromptBuilder
365
+ PWN::AI::Agent::Learning.stats
366
+ PWN::AI::Agent::Learning.reset
367
+
368
+ Enable end-of-run auto-learning with:
369
+ PWN::Env[:ai][:agent][:auto_reflect] = true
370
+
371
+ #{self}.authors
372
+ USAGE
373
+ end
374
+ end
375
+ end
376
+ end
377
+ end
@@ -49,6 +49,18 @@ module PWN
49
49
  DEFAULT_MAX_ITERS
50
50
  end
51
51
 
52
+ private_class_method def self.record_metrics(opts = {})
53
+ name = opts[:name]
54
+ started = opts[:started]
55
+ raw = opts[:raw].to_s
56
+ ok = raw.include?('"success":true')
57
+ err = raw[/"error":"([^"]{1,300})"/, 1]
58
+ dur = started ? (Time.now - started) : 0.0
59
+ Metrics.record(name: name, success: ok, duration: dur, error: err) if defined?(Metrics)
60
+ rescue StandardError
61
+ nil
62
+ end
63
+
52
64
  private_class_method def self.append_session(opts = {})
53
65
  session_id = opts[:session_id]
54
66
  return unless session_id && defined?(PWN::Sessions)
@@ -163,13 +175,16 @@ module PWN
163
175
  if calls.empty?
164
176
  text = msg[:content].to_s
165
177
  append_session(session_id: session_id, role: 'assistant', content: text)
178
+ Learning.auto_reflect(session_id: session_id, request: request, final: text) if defined?(Learning)
166
179
  return text
167
180
  end
168
181
 
169
182
  calls.each do |tc|
170
- name = tc.dig(:function, :name).to_s
171
- entry = Registry.lookup(name: name)
172
- raw = Dispatch.call(tool_call: tc)
183
+ name = tc.dig(:function, :name).to_s
184
+ entry = Registry.lookup(name: name)
185
+ started = Time.now
186
+ raw = Dispatch.call(tool_call: tc)
187
+ record_metrics(name: name, started: started, raw: raw)
173
188
  result = Result.condition(content: raw, entry: entry)
174
189
 
175
190
  on_tool&.call(name, tc.dig(:function, :arguments), result)
@@ -0,0 +1,155 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'fileutils'
5
+ require 'time'
6
+
7
+ module PWN
8
+ module AI
9
+ module Agent
10
+ # PWN::AI::Agent::Metrics is the telemetry layer of the pwn-ai
11
+ # learning loop. Every tool dispatch performed by
12
+ # PWN::AI::Agent::Loop is recorded here (name, success, duration,
13
+ # last error) and persisted to ~/.pwn/metrics.json.
14
+ #
15
+ # PromptBuilder re-injects a compact effectiveness summary into the
16
+ # system prompt on every turn, so the model gains awareness of which
17
+ # tools historically succeed vs. fail on THIS host and can adapt its
18
+ # tool selection accordingly. This is one half of the closed
19
+ # feedback loop that lets pwn-ai continuously make itself smarter
20
+ # (the other half is PWN::AI::Agent::Learning).
21
+ module Metrics
22
+ METRICS_FILE = File.join(Dir.home, '.pwn', 'metrics.json')
23
+
24
+ # Supported Method Parameters::
25
+ # metrics = PWN::AI::Agent::Metrics.load
26
+
27
+ public_class_method def self.load
28
+ FileUtils.mkdir_p(File.dirname(METRICS_FILE))
29
+ return { tools: {}, updated_at: nil } unless File.exist?(METRICS_FILE)
30
+
31
+ JSON.parse(File.read(METRICS_FILE), symbolize_names: true)
32
+ rescue StandardError
33
+ { tools: {}, updated_at: nil }
34
+ end
35
+
36
+ # Supported Method Parameters::
37
+ # PWN::AI::Agent::Metrics.save(
38
+ # metrics: 'required - Hash returned by .load / mutated in place'
39
+ # )
40
+
41
+ public_class_method def self.save(opts = {})
42
+ metrics = opts[:metrics] ||= { tools: {} }
43
+ metrics[:updated_at] = Time.now.utc.iso8601
44
+ FileUtils.mkdir_p(File.dirname(METRICS_FILE))
45
+ File.write(METRICS_FILE, JSON.pretty_generate(metrics))
46
+ metrics
47
+ end
48
+
49
+ # Supported Method Parameters::
50
+ # PWN::AI::Agent::Metrics.record(
51
+ # name: 'required - tool name that was dispatched',
52
+ # success: 'required - Boolean, did the handler complete without error',
53
+ # duration: 'optional - Float seconds the dispatch took',
54
+ # error: 'optional - String error message when success is false'
55
+ # )
56
+
57
+ public_class_method def self.record(opts = {})
58
+ name = opts[:name].to_s
59
+ success = opts[:success] ? true : false
60
+ duration = opts[:duration].to_f
61
+ error = opts[:error]
62
+ return if name.empty?
63
+
64
+ metrics = load
65
+ metrics[:tools] ||= {}
66
+ key = name.to_sym
67
+ t = metrics[:tools][key] ||= {
68
+ calls: 0, ok: 0, fail: 0, total_duration: 0.0,
69
+ last_error: nil, last_at: nil
70
+ }
71
+ t[:calls] += 1
72
+ t[:ok] += 1 if success
73
+ t[:fail] += 1 unless success
74
+ t[:total_duration] += duration
75
+ t[:last_error] = error.to_s[0, 300] if error && !success
76
+ t[:last_at] = Time.now.utc.iso8601
77
+ save(metrics: metrics)
78
+ t
79
+ end
80
+
81
+ # Supported Method Parameters::
82
+ # rows = PWN::AI::Agent::Metrics.summary(
83
+ # limit: 'optional - cap number of tools returned (default 25)'
84
+ # )
85
+
86
+ public_class_method def self.summary(opts = {})
87
+ limit = opts[:limit] || 25
88
+ tools = load[:tools] || {}
89
+ rows = tools.map do |name, t|
90
+ calls = t[:calls].to_i
91
+ ok = t[:ok].to_i
92
+ rate = calls.positive? ? (ok.to_f / calls).round(3) : 0.0
93
+ avg = calls.positive? ? (t[:total_duration].to_f / calls).round(3) : 0.0
94
+ {
95
+ name: name.to_s,
96
+ calls: calls,
97
+ success_rate: rate,
98
+ avg_duration: avg,
99
+ last_error: t[:last_error],
100
+ last_at: t[:last_at]
101
+ }
102
+ end
103
+ rows.sort_by { |r| [-r[:calls], -r[:success_rate]] }.first(limit)
104
+ end
105
+
106
+ # Supported Method Parameters::
107
+ # ctx = PWN::AI::Agent::Metrics.to_context(
108
+ # limit: 'optional - cap number of tools included (default 8)'
109
+ # )
110
+
111
+ public_class_method def self.to_context(opts = {})
112
+ limit = opts[:limit] || 8
113
+ rows = summary(limit: limit)
114
+ return '' if rows.empty?
115
+
116
+ lines = rows.map do |r|
117
+ err = r[:last_error] ? " last_err=#{r[:last_error][0, 60]}" : ''
118
+ " - #{r[:name]}: calls=#{r[:calls]} success=#{(r[:success_rate] * 100).round(1)}% avg=#{r[:avg_duration]}s#{err}"
119
+ end
120
+ "TOOL EFFECTIVENESS (historical, adapt tool choice accordingly)\n#{lines.join("\n")}\n\n"
121
+ end
122
+
123
+ # Supported Method Parameters::
124
+ # PWN::AI::Agent::Metrics.reset
125
+
126
+ public_class_method def self.reset
127
+ FileUtils.rm_f(METRICS_FILE)
128
+ { tools: {}, updated_at: nil }
129
+ end
130
+
131
+ # Author(s):: 0day Inc. <support@0dayinc.com>
132
+
133
+ public_class_method def self.authors
134
+ "AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
135
+ end
136
+
137
+ # Display Usage for this Module
138
+
139
+ public_class_method def self.help
140
+ puts <<~USAGE
141
+ USAGE:
142
+ PWN::AI::Agent::Metrics.record(name: 'shell', success: true, duration: 0.42)
143
+ PWN::AI::Agent::Metrics.summary(limit: 10)
144
+ PWN::AI::Agent::Metrics.to_context(limit: 8) # injected by PromptBuilder
145
+ PWN::AI::Agent::Metrics.reset
146
+ PWN::AI::Agent::Metrics.load
147
+ PWN::AI::Agent::Metrics.save(metrics: hash)
148
+
149
+ #{self}.authors
150
+ USAGE
151
+ end
152
+ end
153
+ end
154
+ end
155
+ end