pwn 0.5.618 → 0.5.620
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -2
- data/README.md +14 -6
- data/bin/pwn_gqrx_scanner +25 -5
- data/documentation/Agent-Tool-Registry.md +3 -3
- data/documentation/Diagrams.md +6 -2
- data/documentation/Extrospection.md +30 -11
- data/documentation/Home.md +5 -4
- data/documentation/How-PWN-Works.md +7 -4
- data/documentation/Mistakes.md +96 -0
- data/documentation/Persistence.md +3 -1
- data/documentation/SDR.md +23 -5
- data/documentation/Skills-Memory-Learning.md +15 -5
- data/documentation/diagrams/agent-tool-registry.svg +94 -92
- data/documentation/diagrams/dot/agent-tool-registry.dot +2 -2
- data/documentation/diagrams/dot/extrospection-world-awareness.dot +9 -7
- data/documentation/diagrams/dot/memory-skills-detailed.dot +12 -5
- data/documentation/diagrams/dot/mistakes-negative-feedback.dot +93 -0
- data/documentation/diagrams/dot/overall-pwn-architecture.dot +7 -5
- data/documentation/diagrams/dot/persistence-filesystem.dot +3 -2
- data/documentation/diagrams/dot/pwn-ai-feedback-learning-loop.dot +19 -7
- data/documentation/diagrams/dot/sdr-radio-flow.dot +13 -11
- data/documentation/diagrams/extrospection-world-awareness.svg +111 -93
- data/documentation/diagrams/memory-skills-detailed.svg +146 -97
- data/documentation/diagrams/mistakes-negative-feedback.svg +266 -0
- data/documentation/diagrams/overall-pwn-architecture.svg +184 -173
- data/documentation/diagrams/persistence-filesystem.svg +77 -62
- data/documentation/diagrams/pwn-ai-feedback-learning-loop.svg +217 -149
- data/documentation/diagrams/sdr-radio-flow.svg +76 -56
- data/documentation/pwn-ai-Agent.md +28 -15
- data/lib/pwn/ai/agent/extrospection.rb +65 -6
- data/lib/pwn/ai/agent/learning.rb +67 -7
- data/lib/pwn/ai/agent/loop.rb +72 -6
- data/lib/pwn/ai/agent/mistakes.rb +369 -0
- data/lib/pwn/ai/agent/prompt_builder.rb +10 -1
- data/lib/pwn/ai/agent/tools/extrospection.rb +2 -2
- data/lib/pwn/ai/agent/tools/mistakes.rb +132 -0
- data/lib/pwn/ai/agent.rb +1 -0
- data/lib/pwn/plugins/jira_data_center.rb +2 -2
- data/lib/pwn/sdr/decoder/adsb.rb +101 -0
- data/lib/pwn/sdr/decoder/apt.rb +75 -0
- data/lib/pwn/sdr/decoder/bluetooth.rb +78 -0
- data/lib/pwn/sdr/decoder/dect.rb +84 -0
- data/lib/pwn/sdr/decoder/gps.rb +83 -0
- data/lib/pwn/sdr/decoder/gsm.rb +50 -29
- data/lib/pwn/sdr/decoder/iridium.rb +81 -0
- data/lib/pwn/sdr/decoder/lora.rb +87 -0
- data/lib/pwn/sdr/decoder/lte.rb +88 -0
- data/lib/pwn/sdr/decoder/morse.rb +64 -0
- data/lib/pwn/sdr/decoder/p25.rb +71 -0
- data/lib/pwn/sdr/decoder/pager.rb +77 -0
- data/lib/pwn/sdr/decoder/pocsag.rb +26 -11
- data/lib/pwn/sdr/decoder/rfid.rb +88 -0
- data/lib/pwn/sdr/decoder/rtl433.rb +95 -0
- data/lib/pwn/sdr/decoder/rtty.rb +66 -0
- data/lib/pwn/sdr/decoder/wifi.rb +98 -0
- data/lib/pwn/sdr/decoder/zigbee.rb +96 -0
- data/lib/pwn/sdr/decoder.rb +67 -7
- data/lib/pwn/sdr/frequency_allocation.rb +82 -41
- data/lib/pwn/sdr/gqrx.rb +186 -95
- data/lib/pwn/version.rb +1 -1
- data/spec/lib/pwn/ai/agent/mistakes_spec.rb +36 -0
- data/spec/lib/pwn/ai/agent/tools/mistakes_spec.rb +12 -0
- data/spec/lib/pwn/sdr/decoder/adsb_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/apt_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/bluetooth_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/dect_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/gps_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/iridium_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/lora_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/lte_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/morse_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/p25_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/pager_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/rfid_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/rtl433_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/rtty_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/wifi_spec.rb +15 -0
- data/spec/lib/pwn/sdr/decoder/zigbee_spec.rb +15 -0
- data/third_party/pwn_rdoc.jsonl +92 -3
- metadata +44 -6
- data/README.md.bak +0 -200
data/lib/pwn/ai/agent/loop.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'json'
|
|
4
|
+
require 'digest'
|
|
5
|
+
require 'pwn/ai/agent/mistakes'
|
|
4
6
|
|
|
5
7
|
module PWN
|
|
6
8
|
module AI
|
|
@@ -14,6 +16,22 @@ module PWN
|
|
|
14
16
|
# native function-calling. State (memory, skills, sessions) is all
|
|
15
17
|
# externalised — Loop.run is stateless aside from the messages array it
|
|
16
18
|
# builds.
|
|
19
|
+
#
|
|
20
|
+
# NEGATIVE-FEEDBACK CLOSURE
|
|
21
|
+
# -------------------------
|
|
22
|
+
# Loop.run is where "learn from mistakes, don't repeat them" is
|
|
23
|
+
# actually enforced. On EVERY failed dispatch it:
|
|
24
|
+
# 1. Records the (tool, normalised_error) fingerprint into
|
|
25
|
+
# PWN::AI::Agent::Mistakes with a PERSISTENT cross-session count.
|
|
26
|
+
# 2. Reads that count back and, if it OR the in-turn count reaches
|
|
27
|
+
# REPEAT_THRESHOLD, prepends a hard "REPEATED FAILURE — change
|
|
28
|
+
# approach" guard to the tool result the model sees next.
|
|
29
|
+
# 3. Appends Mistakes.correction_hint (seen N×, sig, KNOWN FIX: …)
|
|
30
|
+
# so a previously-discovered fix is handed straight back to the
|
|
31
|
+
# model on the FIRST recurrence in a new session — it does not
|
|
32
|
+
# have to fail 3× again to re-learn what it already knew.
|
|
33
|
+
# PromptBuilder.mistakes_block re-injects the top open mistakes and
|
|
34
|
+
# top known fixes into the system prompt of every future turn.
|
|
17
35
|
module Loop
|
|
18
36
|
DEFAULT_MAX_ITERS = 777
|
|
19
37
|
|
|
@@ -49,16 +67,26 @@ module PWN
|
|
|
49
67
|
DEFAULT_MAX_ITERS
|
|
50
68
|
end
|
|
51
69
|
|
|
70
|
+
# Record per-tool telemetry AND, when the dispatch failed, fingerprint
|
|
71
|
+
# the failure into PWN::AI::Agent::Mistakes so recurring errors are
|
|
72
|
+
# counted, surfaced in the system prompt, and can be resolved with an
|
|
73
|
+
# explicit fix. Returns { ok:, err:, mistake: } — :mistake carries the
|
|
74
|
+
# PERSISTED entry (with cumulative :count and any prior :fix) so the
|
|
75
|
+
# caller drives cross-session repeat detection, not just per-turn.
|
|
52
76
|
private_class_method def self.record_metrics(opts = {})
|
|
53
77
|
name = opts[:name]
|
|
54
78
|
started = opts[:started]
|
|
55
79
|
raw = opts[:raw].to_s
|
|
56
|
-
ok = raw.include?('"success":true')
|
|
80
|
+
ok = raw.include?('"success":true') && !raw.match?(/"exit":[1-9]/)
|
|
57
81
|
err = raw[/"error":"([^"]{1,300})"/, 1]
|
|
82
|
+
err ||= raw[/"stderr":"([^"]{4,300})"/, 1] unless ok
|
|
58
83
|
dur = started ? (Time.now - started) : 0.0
|
|
59
84
|
Metrics.record(name: name, success: ok, duration: dur, error: err) if defined?(Metrics)
|
|
85
|
+
m = nil
|
|
86
|
+
m = Mistakes.record(tool: name, error: err || raw[0, 300], args: opts[:args], session_id: opts[:session_id], source: :tool) if !ok && defined?(Mistakes)
|
|
87
|
+
{ ok: ok, err: err, mistake: m }
|
|
60
88
|
rescue StandardError
|
|
61
|
-
nil
|
|
89
|
+
{ ok: true, err: nil, mistake: nil }
|
|
62
90
|
end
|
|
63
91
|
|
|
64
92
|
# Stash the active session_id under PWN::Env[:ai][:session_id] so
|
|
@@ -88,6 +116,26 @@ module PWN
|
|
|
88
116
|
nil
|
|
89
117
|
end
|
|
90
118
|
|
|
119
|
+
# Repeat circuit-breaker. `count` is max(per-turn, persistent) so a
|
|
120
|
+
# signature that already failed in a PREVIOUS session trips the guard
|
|
121
|
+
# on its FIRST recurrence here — the agent does not get to burn the
|
|
122
|
+
# iteration budget re-learning a lesson it already recorded.
|
|
123
|
+
private_class_method def self.guard_repeated_failure(opts = {})
|
|
124
|
+
count = opts[:count].to_i
|
|
125
|
+
result = opts[:result].to_s
|
|
126
|
+
hint = opts[:hint].to_s
|
|
127
|
+
thresh = defined?(Mistakes) ? Mistakes::REPEAT_THRESHOLD : 3
|
|
128
|
+
result = "#{result}\n#{hint}" unless hint.empty?
|
|
129
|
+
return result if count < thresh
|
|
130
|
+
|
|
131
|
+
guard = "[pwn-ai/mistakes] REPEATED FAILURE — this #{opts[:name]} failure signature has " \
|
|
132
|
+
"occurred #{count}× (across sessions). DO NOT retry it verbatim; change " \
|
|
133
|
+
'arguments, pick a different tool, apply the KNOWN FIX below if present, or ' \
|
|
134
|
+
'explain why it cannot succeed. Once a working alternative is found, call ' \
|
|
135
|
+
'mistakes_resolve(signature:, fix:) so future runs skip straight to it.'
|
|
136
|
+
"#{guard}\n#{result}"
|
|
137
|
+
end
|
|
138
|
+
|
|
91
139
|
# Supported Method Parameters::
|
|
92
140
|
# msg = PWN::AI::Agent::Loop.normalize_llm(
|
|
93
141
|
# response: 'required - chat_with_tools response Hash from any provider'
|
|
@@ -172,6 +220,7 @@ module PWN
|
|
|
172
220
|
|
|
173
221
|
Registry.discover
|
|
174
222
|
expose_current_session(session_id: session_id)
|
|
223
|
+
Mistakes.check_user_correction(request: request, session_id: session_id) if defined?(Mistakes)
|
|
175
224
|
|
|
176
225
|
tools = Registry.definitions(enabled: opts[:enabled_toolsets])
|
|
177
226
|
messages = [
|
|
@@ -180,6 +229,8 @@ module PWN
|
|
|
180
229
|
]
|
|
181
230
|
append_session(session_id: session_id, role: 'user', content: request)
|
|
182
231
|
|
|
232
|
+
turn_fails = Hash.new(0)
|
|
233
|
+
|
|
183
234
|
max_iters.times do |i|
|
|
184
235
|
msg = call_engine(messages: messages, tools: tools)
|
|
185
236
|
return '[pwn-ai] engine returned no message' if msg.nil?
|
|
@@ -196,13 +247,27 @@ module PWN
|
|
|
196
247
|
|
|
197
248
|
calls.each do |tc|
|
|
198
249
|
name = tc.dig(:function, :name).to_s
|
|
250
|
+
args = tc.dig(:function, :arguments)
|
|
199
251
|
entry = Registry.lookup(name: name)
|
|
200
252
|
started = Time.now
|
|
201
253
|
raw = Dispatch.call(tool_call: tc)
|
|
202
|
-
record_metrics(name: name, started: started, raw: raw)
|
|
203
|
-
result
|
|
204
|
-
|
|
205
|
-
|
|
254
|
+
tele = record_metrics(name: name, started: started, raw: raw, args: args, session_id: session_id)
|
|
255
|
+
result = Result.condition(content: raw, entry: entry)
|
|
256
|
+
|
|
257
|
+
unless tele[:ok]
|
|
258
|
+
fkey = Digest::SHA256.hexdigest("#{name}|#{args}")[0, 16]
|
|
259
|
+
turn_fails[fkey] += 1
|
|
260
|
+
persist = tele.dig(:mistake, :count).to_i
|
|
261
|
+
hint = defined?(Mistakes) ? Mistakes.correction_hint(tool: name, error: tele[:err] || raw[0, 300]) : ''
|
|
262
|
+
result = guard_repeated_failure(
|
|
263
|
+
name: name,
|
|
264
|
+
count: [turn_fails[fkey], persist].max,
|
|
265
|
+
hint: hint,
|
|
266
|
+
result: result
|
|
267
|
+
)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
on_tool&.call(name, args, result)
|
|
206
271
|
|
|
207
272
|
messages << {
|
|
208
273
|
role: 'tool',
|
|
@@ -218,6 +283,7 @@ module PWN
|
|
|
218
283
|
end
|
|
219
284
|
end
|
|
220
285
|
|
|
286
|
+
Mistakes.record(tool: 'agent_loop', error: 'iteration budget exhausted without a final answer', session_id: session_id, source: :loop) if defined?(Mistakes)
|
|
221
287
|
'[pwn-ai] iteration budget exhausted'
|
|
222
288
|
end
|
|
223
289
|
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'time'
|
|
6
|
+
require 'digest'
|
|
7
|
+
|
|
8
|
+
module PWN
|
|
9
|
+
module AI
|
|
10
|
+
module Agent
|
|
11
|
+
# PWN::AI::Agent::Mistakes is the negative-feedback half of the pwn-ai
|
|
12
|
+
# learning loop. Where Learning records WHAT WORKED and Metrics records
|
|
13
|
+
# HOW OFTEN a tool worked, Mistakes records SPECIFIC FAILURE PATTERNS
|
|
14
|
+
# with a stable fingerprint so the agent can (a) recognise it is
|
|
15
|
+
# repeating itself, (b) be told exactly what not to do again in every
|
|
16
|
+
# future system prompt, and (c) capture the FIX once one is found so
|
|
17
|
+
# the avoidance lesson becomes an actionable correction.
|
|
18
|
+
#
|
|
19
|
+
# A "mistake" is keyed by sha12(tool + normalised_error). Normalisation
|
|
20
|
+
# strips volatile bits (paths, hex addresses, line numbers, timestamps,
|
|
21
|
+
# UUIDs, PIDs) so "NoMethodError ... at foo.rb:42" and "... at foo.rb:99"
|
|
22
|
+
# collapse to one signature and its :count climbs — that count IS the
|
|
23
|
+
# repeat detector.
|
|
24
|
+
#
|
|
25
|
+
# Closed loop (why it does NOT repeat mistakes):
|
|
26
|
+
# Loop.run --(tool failure)---------> Mistakes.record (persist + count++)
|
|
27
|
+
# Loop.run --(same sig fails ≥N)----> guard_repeated_failure (uses PERSISTENT count,
|
|
28
|
+
# so triggers on the 1st
|
|
29
|
+
# recurrence in a new
|
|
30
|
+
# session, not the 3rd)
|
|
31
|
+
# Loop.run --(failure w/ known fix)-> inline "KNOWN FIX: …" (self-corrects next iter)
|
|
32
|
+
# Loop.run --(user says "wrong")----> check_user_correction (flip last outcome + record)
|
|
33
|
+
# PromptBuilder <-------------------- Mistakes.to_context (DO-NOT-REPEAT + KNOWN-FIXES)
|
|
34
|
+
# model --(tool call)---------------> mistakes_record / mistakes_resolve
|
|
35
|
+
module Mistakes
|
|
36
|
+
MISTAKES_FILE = File.join(Dir.home, '.pwn', 'mistakes.json')
|
|
37
|
+
REPEAT_THRESHOLD = 3
|
|
38
|
+
|
|
39
|
+
CORRECTION_RX = /
|
|
40
|
+
\b(
|
|
41
|
+
no[,.]?\s*(that|this|it)?'?s?\s*(wrong|not\s+right|incorrect)|
|
|
42
|
+
still\s+(broken|failing|wrong|not\s+working|doesn'?t\s+work)|
|
|
43
|
+
(that|it|this)\s+(did(n'?t| not)\s+work|failed|is\s+wrong)|
|
|
44
|
+
not\s+what\s+i\s+(asked|meant|wanted)|
|
|
45
|
+
you\s+(made\s+a|got\s+it)\s+(mistake|wrong)|
|
|
46
|
+
same\s+(mistake|error|problem)\s+again|
|
|
47
|
+
try\s+again|redo\s+that|wrong\s+answer|incorrect
|
|
48
|
+
)\b
|
|
49
|
+
/ix
|
|
50
|
+
|
|
51
|
+
# Supported Method Parameters::
|
|
52
|
+
# store = PWN::AI::Agent::Mistakes.load
|
|
53
|
+
|
|
54
|
+
public_class_method def self.load
|
|
55
|
+
FileUtils.mkdir_p(File.dirname(MISTAKES_FILE))
|
|
56
|
+
return {} unless File.exist?(MISTAKES_FILE)
|
|
57
|
+
|
|
58
|
+
JSON.parse(File.read(MISTAKES_FILE), symbolize_names: true)
|
|
59
|
+
rescue StandardError
|
|
60
|
+
{}
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Supported Method Parameters::
|
|
64
|
+
# PWN::AI::Agent::Mistakes.save(store: hash)
|
|
65
|
+
|
|
66
|
+
public_class_method def self.save(opts = {})
|
|
67
|
+
store = opts[:store] ||= {}
|
|
68
|
+
FileUtils.mkdir_p(File.dirname(MISTAKES_FILE))
|
|
69
|
+
File.write(MISTAKES_FILE, JSON.pretty_generate(store))
|
|
70
|
+
store
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Supported Method Parameters::
|
|
74
|
+
# sig = PWN::AI::Agent::Mistakes.signature(
|
|
75
|
+
# tool: 'required - tool/component name that failed',
|
|
76
|
+
# error: 'required - raw error text (will be normalised)'
|
|
77
|
+
# )
|
|
78
|
+
|
|
79
|
+
public_class_method def self.signature(opts = {})
|
|
80
|
+
tool = opts[:tool].to_s
|
|
81
|
+
norm = normalize_error(error: opts[:error])
|
|
82
|
+
Digest::SHA256.hexdigest("#{tool}|#{norm}")[0, 12]
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Supported Method Parameters::
|
|
86
|
+
# entry = PWN::AI::Agent::Mistakes.find(
|
|
87
|
+
# signature: 'optional - exact signature to fetch',
|
|
88
|
+
# tool: 'optional - with error:, compute signature and fetch',
|
|
89
|
+
# error: 'optional - raw error text (used with tool:)'
|
|
90
|
+
# )
|
|
91
|
+
|
|
92
|
+
public_class_method def self.find(opts = {})
|
|
93
|
+
sig = opts[:signature] || (opts[:tool] && opts[:error] ? signature(tool: opts[:tool], error: opts[:error]) : nil)
|
|
94
|
+
return nil unless sig
|
|
95
|
+
|
|
96
|
+
load[sig.to_sym]
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Supported Method Parameters::
|
|
100
|
+
# rows = PWN::AI::Agent::Mistakes.for_tool(
|
|
101
|
+
# tool: 'required - tool name',
|
|
102
|
+
# unresolved_only: 'optional - default false'
|
|
103
|
+
# )
|
|
104
|
+
|
|
105
|
+
public_class_method def self.for_tool(opts = {})
|
|
106
|
+
tool = opts[:tool].to_s
|
|
107
|
+
only = opts[:unresolved_only] ? true : false
|
|
108
|
+
rows = load.values.select { |m| m[:tool].to_s == tool }
|
|
109
|
+
rows = rows.reject { |m| m[:resolved] } if only
|
|
110
|
+
rows.sort_by { |m| -m[:count].to_i }
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Supported Method Parameters::
|
|
114
|
+
# entry = PWN::AI::Agent::Mistakes.record(
|
|
115
|
+
# tool: 'required - tool/component that produced the failure',
|
|
116
|
+
# error: 'required - error text / message',
|
|
117
|
+
# args: 'optional - args that triggered it (stored truncated as sample)',
|
|
118
|
+
# session_id: 'optional - PWN::Sessions id',
|
|
119
|
+
# source: 'optional - :tool | :user_correction | :loop | :model | :heuristic (default :tool)'
|
|
120
|
+
# )
|
|
121
|
+
#
|
|
122
|
+
# Returns the FULL persisted entry including its cumulative :count so
|
|
123
|
+
# the caller (Loop.run) can drive cross-session repeat detection.
|
|
124
|
+
|
|
125
|
+
public_class_method def self.record(opts = {})
|
|
126
|
+
tool = opts[:tool].to_s
|
|
127
|
+
error = opts[:error].to_s
|
|
128
|
+
return nil if tool.empty? || error.strip.empty?
|
|
129
|
+
|
|
130
|
+
sig = signature(tool: tool, error: error)
|
|
131
|
+
store = load
|
|
132
|
+
key = sig.to_sym
|
|
133
|
+
now = Time.now.utc.iso8601
|
|
134
|
+
norm = normalize_error(error: error)
|
|
135
|
+
|
|
136
|
+
m = store[key] ||= {
|
|
137
|
+
signature: sig, tool: tool, error: norm,
|
|
138
|
+
snippet: error.to_s.strip[0, 300],
|
|
139
|
+
count: 0, first_seen: now, sessions: [],
|
|
140
|
+
resolved: false, fix: nil, source: (opts[:source] || :tool).to_s
|
|
141
|
+
}
|
|
142
|
+
was_resolved = m[:resolved]
|
|
143
|
+
m[:count] += 1
|
|
144
|
+
m[:last_seen] = now
|
|
145
|
+
m[:snippet] = error.to_s.strip[0, 300]
|
|
146
|
+
m[:sample_args] = opts[:args].to_s[0, 200] if opts[:args]
|
|
147
|
+
m[:sessions] = (Array(m[:sessions]) + [opts[:session_id]]).compact.uniq.last(10)
|
|
148
|
+
# A recurrence of a "resolved" mistake means the fix was wrong /
|
|
149
|
+
# incomplete — reopen it so it re-enters the DO-NOT-REPEAT block.
|
|
150
|
+
m[:resolved] = false
|
|
151
|
+
m[:regressed] = true if was_resolved
|
|
152
|
+
save(store: store)
|
|
153
|
+
m
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Supported Method Parameters::
|
|
157
|
+
# entry = PWN::AI::Agent::Mistakes.resolve(
|
|
158
|
+
# signature: 'required - mistake signature (from mistakes_list / .top)',
|
|
159
|
+
# fix: 'required - what to do INSTEAD next time'
|
|
160
|
+
# )
|
|
161
|
+
|
|
162
|
+
public_class_method def self.resolve(opts = {})
|
|
163
|
+
sig = opts[:signature].to_s
|
|
164
|
+
fix = opts[:fix].to_s
|
|
165
|
+
raise 'ERROR: signature is required' if sig.empty?
|
|
166
|
+
raise 'ERROR: fix is required' if fix.strip.empty?
|
|
167
|
+
|
|
168
|
+
store = load
|
|
169
|
+
key = sig.to_sym
|
|
170
|
+
raise "ERROR: unknown mistake signature #{sig}" unless store[key]
|
|
171
|
+
|
|
172
|
+
store[key][:resolved] = true
|
|
173
|
+
store[key][:regressed] = false
|
|
174
|
+
store[key][:fix] = fix.strip[0, 500]
|
|
175
|
+
store[key][:resolved_at] = Time.now.utc.iso8601
|
|
176
|
+
save(store: store)
|
|
177
|
+
|
|
178
|
+
if defined?(PWN::Memory)
|
|
179
|
+
PWN::Memory.remember(
|
|
180
|
+
key: :"mistake_fix_#{sig}",
|
|
181
|
+
value: "AVOID: #{store[key][:tool]} → #{store[key][:error]} — FIX: #{fix.strip[0, 300]}",
|
|
182
|
+
category: :lesson
|
|
183
|
+
)
|
|
184
|
+
end
|
|
185
|
+
store[key]
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Supported Method Parameters::
|
|
189
|
+
# rows = PWN::AI::Agent::Mistakes.top(
|
|
190
|
+
# limit: 'optional - max rows (default 10)',
|
|
191
|
+
# unresolved_only: 'optional - default true'
|
|
192
|
+
# )
|
|
193
|
+
|
|
194
|
+
public_class_method def self.top(opts = {})
|
|
195
|
+
limit = opts[:limit] || 10
|
|
196
|
+
only = opts.key?(:unresolved_only) ? opts[:unresolved_only] : true
|
|
197
|
+
rows = load.values
|
|
198
|
+
rows = rows.reject { |m| m[:resolved] } if only
|
|
199
|
+
rows.sort_by { |m| [-m[:count].to_i, m[:last_seen].to_s] }.first(limit)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Supported Method Parameters::
|
|
203
|
+
# ctx = PWN::AI::Agent::Mistakes.to_context(limit: 6)
|
|
204
|
+
#
|
|
205
|
+
# Injected by PromptBuilder into every system prompt. Emits TWO
|
|
206
|
+
# blocks so the model sees both what NOT to do AND what to do
|
|
207
|
+
# INSTEAD:
|
|
208
|
+
# KNOWN MISTAKES — unresolved, count-sorted, [REPEATING]/[REGRESSED]
|
|
209
|
+
# KNOWN FIXES — resolved entries with their fix, so the correction
|
|
210
|
+
# survives even after dropping out of the first list.
|
|
211
|
+
|
|
212
|
+
public_class_method def self.to_context(opts = {})
|
|
213
|
+
limit = opts[:limit] || 6
|
|
214
|
+
open = top(limit: limit, unresolved_only: true)
|
|
215
|
+
closed = load.values.select { |m| m[:resolved] && m[:fix] }
|
|
216
|
+
.sort_by { |m| m[:resolved_at].to_s }.reverse.first(limit)
|
|
217
|
+
return '' if open.empty? && closed.empty?
|
|
218
|
+
|
|
219
|
+
out = +''
|
|
220
|
+
unless open.empty?
|
|
221
|
+
lines = open.map do |m|
|
|
222
|
+
tags = []
|
|
223
|
+
tags << 'REPEATING' if m[:count].to_i >= REPEAT_THRESHOLD
|
|
224
|
+
tags << 'REGRESSED' if m[:regressed]
|
|
225
|
+
tag = tags.empty? ? '' : " [#{tags.join(',')}]"
|
|
226
|
+
fix = m[:fix] ? " — last fix (insufficient): #{m[:fix][0, 100]}" : ''
|
|
227
|
+
" ✗ [#{m[:signature]}] #{m[:tool]} ×#{m[:count]}#{tag}: #{m[:error][0, 140]}#{fix}"
|
|
228
|
+
end
|
|
229
|
+
out << "KNOWN MISTAKES (do NOT repeat — call mistakes_resolve once fixed)\n#{lines.join("\n")}\n"
|
|
230
|
+
end
|
|
231
|
+
unless closed.empty?
|
|
232
|
+
lines = closed.map { |m| " ✓ [#{m[:signature]}] #{m[:tool]}: #{m[:error][0, 80]} — FIX: #{m[:fix][0, 140]}" }
|
|
233
|
+
out << "KNOWN FIXES (apply these instead of repeating the mistake)\n#{lines.join("\n")}\n"
|
|
234
|
+
end
|
|
235
|
+
"#{out}\n"
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Supported Method Parameters::
|
|
239
|
+
# str = PWN::AI::Agent::Mistakes.correction_hint(
|
|
240
|
+
# tool: 'required - tool that just failed',
|
|
241
|
+
# error: 'required - raw error it failed with'
|
|
242
|
+
# )
|
|
243
|
+
#
|
|
244
|
+
# Called by Loop.run immediately after a failed dispatch. Returns a
|
|
245
|
+
# string to append to the tool result telling the model (a) how many
|
|
246
|
+
# times this exact failure has occurred across ALL sessions, and (b)
|
|
247
|
+
# the recorded fix if one exists — so it can self-correct on the very
|
|
248
|
+
# next iteration instead of re-discovering the fix from scratch.
|
|
249
|
+
|
|
250
|
+
public_class_method def self.correction_hint(opts = {})
|
|
251
|
+
m = find(tool: opts[:tool], error: opts[:error])
|
|
252
|
+
return '' unless m
|
|
253
|
+
|
|
254
|
+
parts = ["seen #{m[:count]}× across #{Array(m[:sessions]).length} session(s), sig=#{m[:signature]}"]
|
|
255
|
+
parts << 'REGRESSED (previous fix did not hold)' if m[:regressed]
|
|
256
|
+
parts << "KNOWN FIX: #{m[:fix]}" if m[:fix].to_s.strip.length.positive?
|
|
257
|
+
"[pwn-ai/mistakes] #{parts.join(' | ')}"
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# Supported Method Parameters::
|
|
261
|
+
# bool = PWN::AI::Agent::Mistakes.correction?(request: user_text)
|
|
262
|
+
|
|
263
|
+
public_class_method def self.correction?(opts = {})
|
|
264
|
+
req = opts[:request].to_s
|
|
265
|
+
return false if req.strip.empty?
|
|
266
|
+
|
|
267
|
+
req.match?(CORRECTION_RX) && req.length < 600
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Supported Method Parameters::
|
|
271
|
+
# entry = PWN::AI::Agent::Mistakes.check_user_correction(
|
|
272
|
+
# request: 'required - the incoming user message',
|
|
273
|
+
# session_id: 'optional - session to inspect for the previous answer'
|
|
274
|
+
# )
|
|
275
|
+
#
|
|
276
|
+
# When the user's new message reads like a correction of the previous
|
|
277
|
+
# answer, this (a) flips the most recent Learning outcome for that
|
|
278
|
+
# session to success:false, and (b) records a mistake with source
|
|
279
|
+
# :user_correction whose "error" is the user's own words. This is the
|
|
280
|
+
# strongest available signal that the agent was WRONG.
|
|
281
|
+
|
|
282
|
+
public_class_method def self.check_user_correction(opts = {})
|
|
283
|
+
request = opts[:request].to_s
|
|
284
|
+
session_id = opts[:session_id]
|
|
285
|
+
return nil unless correction?(request: request)
|
|
286
|
+
|
|
287
|
+
prev = previous_assistant(session_id: session_id)
|
|
288
|
+
Learning.flip_last_outcome(session_id: session_id, reason: request[0, 200]) if defined?(Learning)
|
|
289
|
+
record(
|
|
290
|
+
tool: 'assistant_answer',
|
|
291
|
+
error: "user rejected previous answer: #{request.strip[0, 200]}",
|
|
292
|
+
args: prev.to_s[0, 200],
|
|
293
|
+
session_id: session_id,
|
|
294
|
+
source: :user_correction
|
|
295
|
+
)
|
|
296
|
+
rescue StandardError => e
|
|
297
|
+
warn "[pwn-ai/mistakes] check_user_correction swallowed: #{e.class}: #{e.message}"
|
|
298
|
+
nil
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Supported Method Parameters::
|
|
302
|
+
# PWN::AI::Agent::Mistakes.reset
|
|
303
|
+
|
|
304
|
+
public_class_method def self.reset
|
|
305
|
+
FileUtils.rm_f(MISTAKES_FILE)
|
|
306
|
+
{}
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# -------------------------------------------------------------
|
|
310
|
+
# privates
|
|
311
|
+
# -------------------------------------------------------------
|
|
312
|
+
|
|
313
|
+
# Strip volatile substrings so semantically-identical failures
|
|
314
|
+
# collapse to one signature and their :count actually climbs.
|
|
315
|
+
private_class_method def self.normalize_error(opts = {})
|
|
316
|
+
e = opts[:error].to_s.strip.downcase
|
|
317
|
+
e = e.gsub(/0x[0-9a-f]{4,}/, '0xADDR')
|
|
318
|
+
e = e.gsub(%r{(/[\w.@+-]+)+/?}, '/PATH')
|
|
319
|
+
e = e.gsub(/:\d+:in\b/, ':LINE:in')
|
|
320
|
+
e = e.gsub(/:\d+\b/, ':N')
|
|
321
|
+
e = e.gsub(/\bline\s+\d+\b/, 'line N')
|
|
322
|
+
e = e.gsub(/\bport\s+\d+\b/, 'port N')
|
|
323
|
+
e = e.gsub(/\b\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}[z\d:+.-]*/, 'TIMESTAMP')
|
|
324
|
+
e = e.gsub(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/, 'UUID')
|
|
325
|
+
e = e.gsub(/\bpid\s*\d+\b/, 'pid N')
|
|
326
|
+
e = e.gsub(/\b\d{4,}\b/, 'N')
|
|
327
|
+
e.gsub(/\s+/, ' ')[0, 300]
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
private_class_method def self.previous_assistant(opts = {})
|
|
331
|
+
sid = opts[:session_id]
|
|
332
|
+
return nil unless sid && defined?(PWN::Sessions)
|
|
333
|
+
|
|
334
|
+
t = PWN::Sessions.load(session_id: sid)
|
|
335
|
+
t.rfind { |e| e[:role].to_s == 'assistant' }&.[](:content)
|
|
336
|
+
rescue StandardError
|
|
337
|
+
nil
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# Author(s):: 0day Inc. <support@0dayinc.com>
|
|
341
|
+
|
|
342
|
+
public_class_method def self.authors
|
|
343
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# Display Usage for this Module
|
|
347
|
+
|
|
348
|
+
public_class_method def self.help
|
|
349
|
+
puts <<~USAGE
|
|
350
|
+
USAGE:
|
|
351
|
+
PWN::AI::Agent::Mistakes.record(tool: 'shell', error: 'nmpa: command not found', args: '{"command":"nmpa -sV"}')
|
|
352
|
+
PWN::AI::Agent::Mistakes.top(limit: 10, unresolved_only: true)
|
|
353
|
+
PWN::AI::Agent::Mistakes.find(signature: 'abc123def456')
|
|
354
|
+
PWN::AI::Agent::Mistakes.for_tool(tool: 'shell')
|
|
355
|
+
PWN::AI::Agent::Mistakes.resolve(signature: 'abc123def456', fix: 'binary is spelled `nmap`, not `nmpa`')
|
|
356
|
+
PWN::AI::Agent::Mistakes.correction_hint(tool: 'shell', error: err) # inline self-correct
|
|
357
|
+
PWN::AI::Agent::Mistakes.to_context(limit: 6) # injected by PromptBuilder
|
|
358
|
+
PWN::AI::Agent::Mistakes.correction?(request: "no that's wrong")
|
|
359
|
+
PWN::AI::Agent::Mistakes.check_user_correction(request: req, session_id: sid)
|
|
360
|
+
PWN::AI::Agent::Mistakes.signature(tool: 'shell', error: err)
|
|
361
|
+
PWN::AI::Agent::Mistakes.reset
|
|
362
|
+
|
|
363
|
+
#{self}.authors
|
|
364
|
+
USAGE
|
|
365
|
+
end
|
|
366
|
+
end
|
|
367
|
+
end
|
|
368
|
+
end
|
|
369
|
+
end
|
|
@@ -31,7 +31,7 @@ module PWN
|
|
|
31
31
|
pwn : #{pwn_version}
|
|
32
32
|
session_id : #{session_id || '(none)'}
|
|
33
33
|
|
|
34
|
-
#{memory_block}#{skills_block}#{learning_block}#{metrics_block}#{extrospection_block}TOOL USE
|
|
34
|
+
#{memory_block}#{skills_block}#{learning_block}#{mistakes_block}#{metrics_block}#{extrospection_block}TOOL USE
|
|
35
35
|
Use the provided function tools to act on the host. A reply with
|
|
36
36
|
no tool_calls is treated as your FINAL answer to the user.
|
|
37
37
|
Prefer `pwn_eval` for anything in the PWN:: namespace and `shell`
|
|
@@ -90,6 +90,15 @@ module PWN
|
|
|
90
90
|
''
|
|
91
91
|
end
|
|
92
92
|
|
|
93
|
+
private_class_method def self.mistakes_block
|
|
94
|
+
return '' unless defined?(PWN::AI::Agent::Mistakes)
|
|
95
|
+
|
|
96
|
+
ctx = PWN::AI::Agent::Mistakes.to_context(limit: 6).to_s
|
|
97
|
+
ctx.strip.empty? ? '' : ctx
|
|
98
|
+
rescue StandardError
|
|
99
|
+
''
|
|
100
|
+
end
|
|
101
|
+
|
|
93
102
|
private_class_method def self.metrics_block
|
|
94
103
|
return '' unless defined?(PWN::AI::Agent::Metrics)
|
|
95
104
|
|
|
@@ -29,7 +29,7 @@ PWN::AI::Agent::Registry.register(
|
|
|
29
29
|
type: 'object',
|
|
30
30
|
properties: {
|
|
31
31
|
persist: { type: 'boolean', default: true, description: 'Write to disk & rotate previous baseline.' },
|
|
32
|
-
sections: { type: 'array', items: { type: 'string', enum: %w[host net toolchain repo env] }, description: 'Subset of probes to run (default all).' }
|
|
32
|
+
sections: { type: 'array', items: { type: 'string', enum: %w[host net toolchain repo env rf] }, description: 'Subset of probes to run (default all).' }
|
|
33
33
|
},
|
|
34
34
|
required: []
|
|
35
35
|
}
|
|
@@ -86,7 +86,7 @@ PWN::AI::Agent::Registry.register(
|
|
|
86
86
|
properties: {
|
|
87
87
|
source: { type: 'string', description: 'Where it came from (nmap, shodan, burp, cve, human, ...).' },
|
|
88
88
|
data: { type: 'string', description: 'The observation itself.' },
|
|
89
|
-
category: { type: 'string', enum: %w[recon vuln intel target network env misc], default: 'misc' },
|
|
89
|
+
category: { type: 'string', enum: %w[recon vuln intel target network env rf misc], default: 'misc' },
|
|
90
90
|
target: { type: 'string', description: 'Host / IP / URL / asset the observation is about.' },
|
|
91
91
|
tags: { type: 'array', items: { type: 'string' } },
|
|
92
92
|
ttl: { type: 'integer', description: 'Seconds until stale (omit = forever).' }
|