pwn 0.5.719 → 0.5.721
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/etc/default_skills/pwn/ai/agent/engagement/SKILL.md +53 -0
- data/etc/default_skills/pwn/ai/agent/learning/SKILL.md +4 -0
- data/etc/default_skills/pwn/ai/agent/loop/SKILL.md +5 -2
- data/etc/default_skills/pwn/ai/agent/metrics/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/agent/registry/SKILL.md +3 -0
- data/etc/default_skills/pwn/ai/agent/swarm/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/agent/tool_guard/SKILL.md +3 -0
- data/etc/default_skills/pwn/ai/agent/tools/debug_lane/SKILL.md +45 -0
- data/etc/default_skills/pwn/ai/agent/tools/engagement/SKILL.md +45 -0
- data/etc/default_skills/pwn/ai/agent/turn_finalizer/SKILL.md +4 -0
- data/etc/default_skills/pwn/plugins/artifact_registry/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/bin_workspace/SKILL.md +49 -0
- data/etc/default_skills/pwn/plugins/binary_parser/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/capability/SKILL.md +48 -0
- data/etc/default_skills/pwn/plugins/detonate/SKILL.md +47 -0
- data/etc/default_skills/pwn/plugins/emulator/SKILL.md +47 -0
- data/etc/default_skills/pwn/plugins/exploit_dev/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/findings/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/gdb/SKILL.md +2 -0
- data/etc/default_skills/pwn/plugins/packet/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/preflight_checker/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/process_tube/SKILL.md +2 -0
- data/etc/default_skills/pwn/plugins/repl/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/vault/SKILL.md +4 -0
- data/etc/default_skills/pwn/reports/engagement/SKILL.md +47 -0
- data/lib/pwn/ai/agent/dispatch.rb +69 -1
- data/lib/pwn/ai/agent/engagement.rb +167 -0
- data/lib/pwn/ai/agent/learning.rb +105 -5
- data/lib/pwn/ai/agent/loop.rb +83 -25
- data/lib/pwn/ai/agent/metrics.rb +35 -1
- data/lib/pwn/ai/agent/registry.rb +28 -0
- data/lib/pwn/ai/agent/reward.rb +80 -15
- data/lib/pwn/ai/agent/swarm.rb +36 -0
- data/lib/pwn/ai/agent/tool_guard.rb +93 -9
- data/lib/pwn/ai/agent/tools/debug_lane.rb +160 -0
- data/lib/pwn/ai/agent/tools/engagement.rb +40 -0
- data/lib/pwn/ai/agent/turn_finalizer.rb +84 -0
- data/lib/pwn/ai/agent.rb +1 -0
- data/lib/pwn/config.rb +18 -2
- data/lib/pwn/plugins/artifact_registry.rb +32 -2
- data/lib/pwn/plugins/bin_workspace.rb +81 -0
- data/lib/pwn/plugins/binary_parser.rb +27 -0
- data/lib/pwn/plugins/capability.rb +68 -0
- data/lib/pwn/plugins/detonate.rb +62 -0
- data/lib/pwn/plugins/emulator.rb +64 -0
- data/lib/pwn/plugins/exploit_dev.rb +33 -0
- data/lib/pwn/plugins/findings.rb +56 -1
- data/lib/pwn/plugins/gdb.rb +42 -0
- data/lib/pwn/plugins/ghidra.rb +20 -2
- data/lib/pwn/plugins/packet.rb +26 -0
- data/lib/pwn/plugins/preflight_checker.rb +23 -0
- data/lib/pwn/plugins/process_tube.rb +31 -1
- data/lib/pwn/plugins/repl.rb +25 -2
- data/lib/pwn/plugins/vault.rb +116 -0
- data/lib/pwn/plugins.rb +4 -0
- data/lib/pwn/reports/engagement.rb +63 -0
- data/lib/pwn/reports.rb +1 -0
- data/lib/pwn/sessions.rb +4 -0
- data/lib/pwn/version.rb +1 -1
- data/spec/lib/pwn/ai/agent/dispatch_spec.rb +9 -0
- data/spec/lib/pwn/ai/agent/engagement_spec.rb +27 -0
- data/spec/lib/pwn/ai/agent/learning_spec.rb +23 -0
- data/spec/lib/pwn/ai/agent/loop_spec.rb +44 -0
- data/spec/lib/pwn/ai/agent/reward_spec.rb +17 -1
- data/spec/lib/pwn/ai/agent/swarm_spec.rb +11 -0
- data/spec/lib/pwn/ai/agent/tool_guard_spec.rb +9 -0
- data/spec/lib/pwn/ai/agent/tools/debug_lane_spec.rb +16 -0
- data/spec/lib/pwn/ai/agent/tools/engagement_spec.rb +16 -0
- data/spec/lib/pwn/ai/agent/turn_finalizer_spec.rb +18 -0
- data/spec/lib/pwn/plugins/bin_workspace_spec.rb +26 -0
- data/spec/lib/pwn/plugins/capability_spec.rb +19 -0
- data/spec/lib/pwn/plugins/detonate_spec.rb +24 -0
- data/spec/lib/pwn/plugins/emulator_spec.rb +18 -0
- data/spec/lib/pwn/plugins/findings_spec.rb +16 -3
- data/spec/lib/pwn/plugins/gdb_spec.rb +10 -0
- data/spec/lib/pwn/plugins/packet_spec.rb +6 -0
- data/spec/lib/pwn/plugins/repl_spec.rb +1 -1
- data/spec/lib/pwn/plugins/vault_spec.rb +11 -0
- data/spec/lib/pwn/reports/engagement_spec.rb +30 -0
- data/third_party/pwn_rdoc.jsonl +73 -0
- metadata +25 -1
|
@@ -46,6 +46,21 @@ module PWN
|
|
|
46
46
|
args = parse_args(raw: raw, entry: entry)
|
|
47
47
|
required = Array(entry.schema&.dig(:parameters, :required))
|
|
48
48
|
args = ToolGuard.coerce_args(args: args, required: required) if defined?(ToolGuard)
|
|
49
|
+
blob = args.inspect
|
|
50
|
+
if defined?(PWN::Plugins::Vault)
|
|
51
|
+
blob = PWN::Plugins::Vault.expand(text: blob)
|
|
52
|
+
args = expand_vault_args(args: args)
|
|
53
|
+
end
|
|
54
|
+
if defined?(Engagement)
|
|
55
|
+
denied = ToolGuard.scope_check!(args: args, command: blob) if defined?(ToolGuard) && ToolGuard.respond_to?(:scope_check!)
|
|
56
|
+
denied ||= Engagement.deny_if_out_of_scope(args: args, command: blob)
|
|
57
|
+
return JSON.generate(denied) if denied
|
|
58
|
+
end
|
|
59
|
+
if defined?(ToolGuard) && ToolGuard.respond_to?(:policy_decision)
|
|
60
|
+
pol = ToolGuard.policy_decision(name: entry.name, args: args)
|
|
61
|
+
return JSON.generate(pol) if pol.is_a?(Hash) && pol[:action] == 'deny'
|
|
62
|
+
end
|
|
63
|
+
return JSON.generate(success: false, error: 'taint: tool-output instruction in args', code: 'TAINT_DENY') if taint_blocked?(name: entry.name, args: args)
|
|
49
64
|
if defined?(ToolGuard) && ToolGuard.respond_to?(:canary_leak?) &&
|
|
50
65
|
ToolGuard.canary_leak?(text: args.inspect)
|
|
51
66
|
return JSON.generate(success: false, error: 'refused: session canary in outbound args', code: 'CANARY_DENY', rule_id: 'canary')
|
|
@@ -59,6 +74,12 @@ module PWN
|
|
|
59
74
|
end
|
|
60
75
|
result = entry.handler.call(args)
|
|
61
76
|
result = ToolGuard.quarantine_output(text: result) if defined?(ToolGuard) && result.is_a?(String) && ToolGuard.respond_to?(:quarantine_output)
|
|
77
|
+
note_taint(text: result)
|
|
78
|
+
if defined?(PWN::Plugins::Vault) && result.is_a?(String)
|
|
79
|
+
result = PWN::Plugins::Vault.redact(text: result)
|
|
80
|
+
elsif defined?(PWN::Plugins::Vault) && result.is_a?(Hash)
|
|
81
|
+
result = JSON.parse(PWN::Plugins::Vault.redact(text: JSON.generate(result)))
|
|
82
|
+
end
|
|
62
83
|
JSON.generate(success: true, result: result, effect: effect(name: entry.name, args: args))
|
|
63
84
|
rescue StandardError => e
|
|
64
85
|
JSON.generate(
|
|
@@ -323,8 +344,8 @@ module PWN
|
|
|
323
344
|
return :store if STORE_TOOLS.include?(name)
|
|
324
345
|
|
|
325
346
|
blob = argv_blob(args: opts[:args])
|
|
326
|
-
return :browse if blob.match?(BROWSE_ARGV_RX)
|
|
327
347
|
return :write if blob.match?(WRITE_ARGV_RX)
|
|
348
|
+
return :browse if blob.match?(BROWSE_ARGV_RX)
|
|
328
349
|
return :eval if name == 'pwn_eval'
|
|
329
350
|
|
|
330
351
|
:read
|
|
@@ -348,6 +369,53 @@ module PWN
|
|
|
348
369
|
hash.each_with_object({}) { |(k, v), m| m[k.to_sym] = v }
|
|
349
370
|
end
|
|
350
371
|
|
|
372
|
+
private_class_method def self.expand_vault_args(opts = {})
|
|
373
|
+
args = opts[:args]
|
|
374
|
+
return args unless args.is_a?(Hash)
|
|
375
|
+
|
|
376
|
+
args.transform_values do |v|
|
|
377
|
+
v.is_a?(String) ? PWN::Plugins::Vault.expand(text: v) : v
|
|
378
|
+
end
|
|
379
|
+
rescue StandardError
|
|
380
|
+
opts[:args]
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
private_class_method def self.note_taint(opts = {})
|
|
384
|
+
text = opts[:text].to_s
|
|
385
|
+
grams = text.scan(/.{24,}/).first(20)
|
|
386
|
+
store = Thread.current[:pwn_taint] ||= []
|
|
387
|
+
grams.each { |g| store << g[0, 64] }
|
|
388
|
+
store.shift while store.length > 200
|
|
389
|
+
store
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
private_class_method def self.taint_blocked?(opts = {})
|
|
393
|
+
mode = taint_mode
|
|
394
|
+
return false if mode == 'off'
|
|
395
|
+
|
|
396
|
+
blob = opts[:args].inspect
|
|
397
|
+
return false if blob.length < 24
|
|
398
|
+
return false if opts[:args].is_a?(Hash) && (opts[:args][:taint_ack] == true || opts[:args]['taint_ack'] == true)
|
|
399
|
+
|
|
400
|
+
hit = Array(Thread.current[:pwn_taint]).any? { |g| g.length >= 24 && blob.include?(g) }
|
|
401
|
+
return false unless hit
|
|
402
|
+
return false unless blob.match?(/curl |bash -c|sh -c|\|\s*sh\b/i)
|
|
403
|
+
|
|
404
|
+
mode == 'enforce'
|
|
405
|
+
rescue StandardError
|
|
406
|
+
false
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
private_class_method def self.taint_mode(opts = {})
|
|
410
|
+
override = opts[:mode]
|
|
411
|
+
return override.to_s unless override.to_s.empty?
|
|
412
|
+
return 'enforce' unless defined?(PWN::Env)
|
|
413
|
+
|
|
414
|
+
(PWN::Env.dig(:ai, :taint, :mode) || 'enforce').to_s
|
|
415
|
+
rescue StandardError
|
|
416
|
+
'enforce'
|
|
417
|
+
end
|
|
418
|
+
|
|
351
419
|
# Author(s):: 0day Inc. <support@0dayinc.com>
|
|
352
420
|
|
|
353
421
|
public_class_method def self.authors
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'ipaddr'
|
|
6
|
+
require 'uri'
|
|
7
|
+
|
|
8
|
+
module PWN
|
|
9
|
+
module AI
|
|
10
|
+
module Agent
|
|
11
|
+
# First-class engagement scope, RoE, and findings pointer.
|
|
12
|
+
module Engagement
|
|
13
|
+
ROOT = File.join(Dir.home, '.pwn', 'engagements')
|
|
14
|
+
ACTIVE_FILE = File.join(ROOT, 'active')
|
|
15
|
+
|
|
16
|
+
public_class_method def self.open(opts = {})
|
|
17
|
+
name = (opts[:name] || opts[:engagement] || 'default').to_s
|
|
18
|
+
raise 'ERROR: name is required' if name.empty?
|
|
19
|
+
|
|
20
|
+
FileUtils.mkdir_p(ROOT)
|
|
21
|
+
path = File.join(ROOT, "#{name}.json")
|
|
22
|
+
row = File.file?(path) ? JSON.parse(File.read(path), symbolize_names: true) : {}
|
|
23
|
+
row[:name] = name
|
|
24
|
+
row[:scope_cidrs] = Array(opts[:scope_cidrs] || row[:scope_cidrs])
|
|
25
|
+
row[:scope_domains] = Array(opts[:scope_domains] || row[:scope_domains])
|
|
26
|
+
row[:excluded] = Array(opts[:excluded] || row[:excluded])
|
|
27
|
+
row[:roe] = (opts[:roe] || row[:roe]).to_s
|
|
28
|
+
row[:window] = opts[:window] || row[:window]
|
|
29
|
+
row[:findings] ||= []
|
|
30
|
+
File.write(path, JSON.pretty_generate(row))
|
|
31
|
+
File.write(ACTIVE_FILE, name)
|
|
32
|
+
row.merge(path: path, active: true)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
public_class_method def self.close(opts = {})
|
|
36
|
+
FileUtils.rm_f(ACTIVE_FILE) if opts.is_a?(Hash)
|
|
37
|
+
{ active: nil }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
public_class_method def self.status(opts = {})
|
|
41
|
+
name = (opts[:name] || current_name).to_s
|
|
42
|
+
return { active: nil } if name.empty?
|
|
43
|
+
|
|
44
|
+
path = File.join(ROOT, "#{name}.json")
|
|
45
|
+
return { active: name, missing: true } unless File.file?(path)
|
|
46
|
+
|
|
47
|
+
JSON.parse(File.read(path), symbolize_names: true).merge(active: name)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
public_class_method def self.current_name(opts = {})
|
|
51
|
+
return nil unless opts.is_a?(Hash)
|
|
52
|
+
return nil unless File.file?(ACTIVE_FILE)
|
|
53
|
+
|
|
54
|
+
File.read(ACTIVE_FILE).strip
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
public_class_method def self.in_scope?(opts = {})
|
|
58
|
+
token = (opts[:host] || opts[:ip] || opts[:target] || opts[:url]).to_s
|
|
59
|
+
return true if token.empty?
|
|
60
|
+
|
|
61
|
+
row = status
|
|
62
|
+
return true if row[:active].nil? || row[:missing]
|
|
63
|
+
|
|
64
|
+
enforce = engagement_enforce
|
|
65
|
+
return true if enforce == 'off'
|
|
66
|
+
|
|
67
|
+
host = token
|
|
68
|
+
begin
|
|
69
|
+
host = URI.parse(token).host || token if token.include?('://')
|
|
70
|
+
rescue StandardError
|
|
71
|
+
host = token
|
|
72
|
+
end
|
|
73
|
+
host = host.sub(%r{\Ahttps?://}i, '').split('/').first.to_s.split(':').first
|
|
74
|
+
return false if Array(row[:excluded]).any? { |ex| host.include?(ex.to_s) }
|
|
75
|
+
|
|
76
|
+
cidrs = Array(row[:scope_cidrs]).map(&:to_s).reject(&:empty?)
|
|
77
|
+
domains = Array(row[:scope_domains]).map(&:to_s).reject(&:empty?)
|
|
78
|
+
return true if cidrs.empty? && domains.empty?
|
|
79
|
+
|
|
80
|
+
ip_ok = begin
|
|
81
|
+
addr = IPAddr.new(host)
|
|
82
|
+
cidrs.any? { |c| IPAddr.new(c).include?(addr) }
|
|
83
|
+
rescue StandardError
|
|
84
|
+
false
|
|
85
|
+
end
|
|
86
|
+
dom_ok = domains.any? { |d| host == d || host.end_with?(".#{d}") }
|
|
87
|
+
ip_ok || dom_ok
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
public_class_method def self.deny_if_out_of_scope(opts = {})
|
|
91
|
+
args = opts[:args] || opts[:command] || opts[:text]
|
|
92
|
+
blob = args.is_a?(Hash) ? args.inspect : args.to_s
|
|
93
|
+
tokens = blob.scan(/(?:\d{1,3}\.){3}\d{1,3}|[A-Za-z0-9.-]+\.[A-Za-z]{2,}/).uniq
|
|
94
|
+
bad = tokens.reject { |tok| in_scope?(host: tok) }
|
|
95
|
+
return nil if bad.empty?
|
|
96
|
+
|
|
97
|
+
{
|
|
98
|
+
success: false,
|
|
99
|
+
error: "out_of_scope: #{bad.first}",
|
|
100
|
+
code: 'SCOPE_DENY',
|
|
101
|
+
violating: bad
|
|
102
|
+
}
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
public_class_method def self.authors
|
|
106
|
+
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
public_class_method def self.help
|
|
110
|
+
puts "USAGE:
|
|
111
|
+
# Open or update an engagement and mark it active.
|
|
112
|
+
#{self}.open(
|
|
113
|
+
name: 'required - engagement name',
|
|
114
|
+
engagement: 'optional - alias for name',
|
|
115
|
+
scope_cidrs: 'optional - Array of CIDR strings',
|
|
116
|
+
scope_domains: 'optional - Array of DNS suffixes',
|
|
117
|
+
excluded: 'optional - Array of excluded hosts',
|
|
118
|
+
roe: 'optional - rules of engagement text',
|
|
119
|
+
window: 'optional - time window string'
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# Clear the active engagement pointer.
|
|
123
|
+
#{self}.close(
|
|
124
|
+
name: 'optional - unused; closing always clears the active pointer'
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
# Return the active engagement document.
|
|
128
|
+
#{self}.status(
|
|
129
|
+
name: 'optional - engagement name (defaults to active)'
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# Return the active engagement name or nil.
|
|
133
|
+
#{self}.current_name(
|
|
134
|
+
unused: 'optional - reserved'
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# True when host/ip/url is inside the active engagement scope.
|
|
138
|
+
#{self}.in_scope?(
|
|
139
|
+
host: 'optional - DNS hostname to test against scope_domains',
|
|
140
|
+
ip: 'optional - IPv4 or IPv6 address to test against scope_cidrs',
|
|
141
|
+
target: 'optional - host or IP alias when the caller has one field',
|
|
142
|
+
url: 'optional - URL whose host is extracted and tested'
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
# Structured denial when args mention an out-of-scope host.
|
|
146
|
+
#{self}.deny_if_out_of_scope(
|
|
147
|
+
args: 'optional - Hash of tool args',
|
|
148
|
+
command: 'optional - command string',
|
|
149
|
+
text: 'optional - free-form blob to scan'
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# Print the AUTHOR(S) string for this module.
|
|
153
|
+
#{self}.authors
|
|
154
|
+
"
|
|
155
|
+
constants.sort
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
private_class_method def self.engagement_enforce
|
|
159
|
+
v = (PWN::Env.dig(:ai, :engagement, :enforce) if defined?(PWN::Env))
|
|
160
|
+
(v || 'block').to_s
|
|
161
|
+
rescue StandardError
|
|
162
|
+
'block'
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
@@ -128,6 +128,16 @@ module PWN
|
|
|
128
128
|
entry[:score] = opts[:score].to_f if opts.key?(:score)
|
|
129
129
|
src = opts[:judge_source].to_s
|
|
130
130
|
entry[:judge_source] = src unless src.empty?
|
|
131
|
+
vv = (opts[:verifier_verdict] || opts['verifier_verdict']).to_s
|
|
132
|
+
entry[:verifier_verdict] = vv unless vv.empty?
|
|
133
|
+
vc = (opts[:verdict_class] || opts['verdict_class']).to_s
|
|
134
|
+
entry[:verdict_class] = vc unless vc.empty?
|
|
135
|
+
entry[:remediation_hint] = opts[:remediation_hint].to_s unless opts[:remediation_hint].to_s.empty?
|
|
136
|
+
if vv == 'pass' && opts.key?(:score) && opts[:score].to_f < 0.6
|
|
137
|
+
entry[:status] = 'conflicted'
|
|
138
|
+
entry[:success] = true
|
|
139
|
+
success = true
|
|
140
|
+
end
|
|
131
141
|
FileUtils.mkdir_p(File.dirname(LEARNING_FILE))
|
|
132
142
|
File.open(LEARNING_FILE, 'a') { |f| f.puts(JSON.generate(entry)) }
|
|
133
143
|
maybe_prune_outcomes!
|
|
@@ -137,7 +147,7 @@ module PWN
|
|
|
137
147
|
# are promoted into PWN::Memory[:lesson] so PromptBuilder recall
|
|
138
148
|
# survives across sessions. Without this, the agent re-learns
|
|
139
149
|
# "run rubocop after every patch" every turn (empty memory.json).
|
|
140
|
-
promote_process_lesson(entry: entry) if defined?(PWN::Memory)
|
|
150
|
+
promote_process_lesson(entry: entry) if defined?(PWN::Memory) && entry[:status].to_s != 'conflicted'
|
|
141
151
|
if opts.key?(:score) && defined?(Curriculum) && Curriculum.respond_to?(:calibrate)
|
|
142
152
|
pred = opts[:predicted]
|
|
143
153
|
pred = Thread.current[:pwn_plan_predicted] if pred.nil?
|
|
@@ -186,11 +196,21 @@ module PWN
|
|
|
186
196
|
jmean = total.positive? ? weighted_judge_mean(rows: rows) : nil
|
|
187
197
|
distrust = 0.0
|
|
188
198
|
distrust = Reward.proxy_distrust.to_f.clamp(0.0, 1.0) if defined?(Reward) && Reward.respond_to?(:proxy_distrust)
|
|
199
|
+
orm = rows.select { |r| r[:judge_source].to_s != 'heuristic' && r[:source].to_s != 'heuristic' }
|
|
200
|
+
heur = rows.select { |r| r[:judge_source].to_s == 'heuristic' || r[:source].to_s == 'heuristic' }
|
|
201
|
+
orm_n = orm.length
|
|
202
|
+
heur_n = heur.length
|
|
203
|
+
orm_ok = orm.count { |r| r[:success] == true }
|
|
204
|
+
heur_ok = heur.count { |r| r[:success] == true }
|
|
189
205
|
{
|
|
190
206
|
total_outcomes: total,
|
|
191
207
|
successes: ok,
|
|
192
208
|
failures: total - ok,
|
|
193
209
|
success_rate: raw,
|
|
210
|
+
success_rate_orm: orm_n.positive? ? (orm_ok.to_f / orm_n).round(3) : 0.0,
|
|
211
|
+
success_rate_heur: heur_n.positive? ? (heur_ok.to_f / heur_n).round(3) : 0.0,
|
|
212
|
+
orm_n: orm_n,
|
|
213
|
+
heur_n: heur_n,
|
|
194
214
|
adjusted_success_rate: discount_success_rate(proxy: raw, judge: jmean, distrust: distrust),
|
|
195
215
|
proxy_distrust: distrust,
|
|
196
216
|
skills_known: skills,
|
|
@@ -215,6 +235,14 @@ module PWN
|
|
|
215
235
|
# envelope rows (REQUEST:/GOAL: prefixes) without starving the block.
|
|
216
236
|
rows = prefer_primary_tasks(rows: outcomes(limit: limit * 4)).first(limit)
|
|
217
237
|
fails = prefer_primary_tasks(rows: outcomes(limit: 200, success: false))
|
|
238
|
+
fails = fails.reject { |r| r[:status].to_s == 'conflicted' }
|
|
239
|
+
fails = fails.reject { |r| r[:verifier_verdict].to_s == 'pass' }
|
|
240
|
+
fails = fails.reject { |r| r[:details].to_s.match?(/\bPASS\b/) && r[:success] == true }
|
|
241
|
+
fails = fails.select do |r|
|
|
242
|
+
v = r[:verdict].to_s
|
|
243
|
+
v == 'wrong' || v == 'refused' || (r[:score].to_f < 0.3 && !r[:details].to_s.match?(/\bPASS\b/)) || r[:success] == false
|
|
244
|
+
end
|
|
245
|
+
fails = fails.reject { |r| r[:success] == true }
|
|
218
246
|
# Do not mirror the same ids under both headings — that doubled the
|
|
219
247
|
# failure signal and made RECENT OUTCOMES == RECENT FAILURES when the
|
|
220
248
|
# last N attempts all failed (the injected block looked "stuck").
|
|
@@ -234,8 +262,12 @@ module PWN
|
|
|
234
262
|
# Surface a one-line cause crumb so the agent can actually learn
|
|
235
263
|
# from failures instead of only seeing that they failed.
|
|
236
264
|
if r[:success] != true
|
|
237
|
-
|
|
238
|
-
|
|
265
|
+
if r[:verdict_class].to_s == ''
|
|
266
|
+
crumb = cause_crumb(details: r[:details])
|
|
267
|
+
line += "\n cause: #{crumb}" unless crumb.empty?
|
|
268
|
+
else
|
|
269
|
+
line += "\n cause: #{r[:verdict_class]} #{r[:remediation_hint]}"
|
|
270
|
+
end
|
|
239
271
|
end
|
|
240
272
|
line
|
|
241
273
|
end
|
|
@@ -244,7 +276,7 @@ module PWN
|
|
|
244
276
|
d = s[:proxy_distrust].to_f
|
|
245
277
|
rate = d > 0.05 ? s[:adjusted_success_rate] : s[:success_rate]
|
|
246
278
|
tag = d > 0.05 ? ' adj' : ''
|
|
247
|
-
hdr = "RECENT OUTCOMES (success_rate=#{(rate.to_f * 100).round(1)}%#{tag}#{" judge_mean=#{jm}" if jm} over #{s[:total_outcomes]} attempts)"
|
|
279
|
+
hdr = "RECENT OUTCOMES (success_rate=#{(rate.to_f * 100).round(1)}%#{tag} success=orm:#{(s[:success_rate_orm].to_f * 100).round(1)}%(#{s[:orm_n]}) / heur:#{(s[:success_rate_heur].to_f * 100).round(1)}%(#{s[:heur_n]})#{" judge_mean=#{jm}" if jm} over #{s[:total_outcomes]} attempts)"
|
|
248
280
|
out = "#{hdr}\n#{rows.map(&fmt).join("\n")}\n"
|
|
249
281
|
out += "RECENT FAILURES (learn from these — do not repeat)\n#{fails.map(&fmt).join("\n")}\n" unless fails.empty?
|
|
250
282
|
"#{out}\n"
|
|
@@ -1049,6 +1081,7 @@ module PWN
|
|
|
1049
1081
|
|
|
1050
1082
|
private_class_method def self.cause_crumb(opts = {})
|
|
1051
1083
|
d = opts[:details].to_s.gsub(/\s+/, ' ').strip
|
|
1084
|
+
d = d.gsub(/overlap=\S+/, '').gsub(/ratio=\S+/, '').strip
|
|
1052
1085
|
return '' if d.empty?
|
|
1053
1086
|
|
|
1054
1087
|
# Prefer explicit FLAW / CORRECTED crumbs; else verdict(score) head.
|
|
@@ -1818,6 +1851,55 @@ module PWN
|
|
|
1818
1851
|
File.write(LESSONS_FILE, JSON.pretty_generate(opts[:store]))
|
|
1819
1852
|
end
|
|
1820
1853
|
|
|
1854
|
+
private_class_method def self.learning_max_baks(opts = {})
|
|
1855
|
+
n = opts[:max_baks]
|
|
1856
|
+
return n.to_i if n.to_i.positive?
|
|
1857
|
+
return 5 unless defined?(PWN::Env)
|
|
1858
|
+
|
|
1859
|
+
v = PWN::Env.dig(:ai, :learning, :max_baks)
|
|
1860
|
+
v.to_i.positive? ? v.to_i : 5
|
|
1861
|
+
rescue StandardError
|
|
1862
|
+
5
|
|
1863
|
+
end
|
|
1864
|
+
|
|
1865
|
+
public_class_method def self.list_conflicted(opts = {})
|
|
1866
|
+
limit = (opts[:limit] || 50).to_i
|
|
1867
|
+
outcomes(limit: 500).select { |r| r[:status].to_s == 'conflicted' }.first(limit)
|
|
1868
|
+
end
|
|
1869
|
+
|
|
1870
|
+
public_class_method def self.requeue_conflicted(opts = {})
|
|
1871
|
+
dry = opts[:dry_run] ? true : false
|
|
1872
|
+
rows = list_conflicted(limit: 10_000)
|
|
1873
|
+
return { rescored: 0, dry_run: dry } if rows.empty? || dry
|
|
1874
|
+
|
|
1875
|
+
n = 0
|
|
1876
|
+
rows.each do |r|
|
|
1877
|
+
note_outcome(
|
|
1878
|
+
task: "requeue:#{r[:task]}",
|
|
1879
|
+
success: true,
|
|
1880
|
+
score: [r[:score].to_f, 0.7].max,
|
|
1881
|
+
verifier_verdict: :pass,
|
|
1882
|
+
details: 'rescored after verifier precedence',
|
|
1883
|
+
tags: %w[requeue]
|
|
1884
|
+
)
|
|
1885
|
+
n += 1
|
|
1886
|
+
end
|
|
1887
|
+
{ rescored: n, dry_run: false }
|
|
1888
|
+
end
|
|
1889
|
+
|
|
1890
|
+
public_class_method def self.compact!(opts = {})
|
|
1891
|
+
max_baks = (opts[:max_baks] || learning_max_baks).to_i
|
|
1892
|
+
max_baks = 5 if max_baks <= 0
|
|
1893
|
+
dir = File.dirname(LEARNING_FILE)
|
|
1894
|
+
baks = Dir[File.join(dir, '*.bak*')].sort_by { |p| File.mtime(p) }.reverse
|
|
1895
|
+
pruned = 0
|
|
1896
|
+
baks.drop(max_baks).each do |path|
|
|
1897
|
+
File.delete(path)
|
|
1898
|
+
pruned += 1
|
|
1899
|
+
end
|
|
1900
|
+
{ pruned: pruned, kept: [baks.length, max_baks].min, max_baks: max_baks }
|
|
1901
|
+
end
|
|
1902
|
+
|
|
1821
1903
|
# Author(s):: 0day Inc. <support@0dayinc.com>
|
|
1822
1904
|
|
|
1823
1905
|
public_class_method def self.authors
|
|
@@ -1839,7 +1921,25 @@ module PWN
|
|
|
1839
1921
|
judge_source: 'required - judge source value consumed by #note_outcome',
|
|
1840
1922
|
predicted: 'optional - predicted value consumed by #note_outcome',
|
|
1841
1923
|
confidence: 'optional - confidence value consumed by #note_outcome',
|
|
1842
|
-
engine: 'optional - engine value consumed by #note_outcome'
|
|
1924
|
+
engine: 'optional - engine value consumed by #note_outcome',
|
|
1925
|
+
verifier_verdict: 'optional - pass|fail from a deterministic verifier',
|
|
1926
|
+
verdict_class: 'optional - missing_artifact|wrong_path|unverified_claim|scope_miss|partial_coverage|style_only',
|
|
1927
|
+
remediation_hint: 'optional - one-line fix hint'
|
|
1928
|
+
)
|
|
1929
|
+
|
|
1930
|
+
# List outcomes tagged conflicted (verifier PASS vs low judge).
|
|
1931
|
+
#{self}.list_conflicted(
|
|
1932
|
+
limit: 'optional - max entries (defaults to 50)'
|
|
1933
|
+
)
|
|
1934
|
+
|
|
1935
|
+
# Rescore conflicted outcomes after verifier-precedence lands.
|
|
1936
|
+
#{self}.requeue_conflicted(
|
|
1937
|
+
dry_run: 'optional - true to count without writing'
|
|
1938
|
+
)
|
|
1939
|
+
|
|
1940
|
+
# Prune excess *.bak siblings under the learning directory.
|
|
1941
|
+
#{self}.compact!(
|
|
1942
|
+
max_baks: 'optional - newest bak files to keep (defaults to 5)'
|
|
1843
1943
|
)
|
|
1844
1944
|
|
|
1845
1945
|
# Run outcomes and return its result
|
data/lib/pwn/ai/agent/loop.rb
CHANGED
|
@@ -64,6 +64,18 @@ module PWN
|
|
|
64
64
|
unsatisfied incomplete_final empty_final evidence_final
|
|
65
65
|
].freeze
|
|
66
66
|
|
|
67
|
+
public_class_method def self.evidence_satisfied?(opts = {})
|
|
68
|
+
messages = Array(opts[:messages] || opts[:trace])
|
|
69
|
+
text = opts[:text].to_s
|
|
70
|
+
if defined?(TurnFinalizer) && TurnFinalizer.respond_to?(:arbitrate)
|
|
71
|
+
row = TurnFinalizer.arbitrate(request: opts[:request].to_s, messages: messages, paths: [])
|
|
72
|
+
return true if row[:complete] && row[:unmet].empty? && row[:ledger].any? { |_p, v| v[:write] && v[:read] }
|
|
73
|
+
end
|
|
74
|
+
write_or_read_evidenced?(messages: messages) && !text.strip.empty?
|
|
75
|
+
rescue StandardError
|
|
76
|
+
false
|
|
77
|
+
end
|
|
78
|
+
|
|
67
79
|
public_class_method def self.debug_on?(opts = {})
|
|
68
80
|
return true if opts[:debug]
|
|
69
81
|
return true if defined?(PWN::Plugins::Log) && PWN::Plugins::Log.debug_enabled?
|
|
@@ -394,6 +406,7 @@ module PWN
|
|
|
394
406
|
# the original request → force synthesis. English tasks are an advisory
|
|
395
407
|
# compass only — an open verify tail must not block a finished ask.
|
|
396
408
|
# The original request is the completion signal.
|
|
409
|
+
|
|
397
410
|
private_class_method def self.evidence_enough_to_finalize?(opts = {})
|
|
398
411
|
messages = Array(opts[:messages])
|
|
399
412
|
turn_fails = opts[:turn_fails] || {}
|
|
@@ -595,7 +608,7 @@ module PWN
|
|
|
595
608
|
request: request,
|
|
596
609
|
messages: opts[:messages]
|
|
597
610
|
)
|
|
598
|
-
return true if need == :write && !write_verified?(effects: effects)
|
|
611
|
+
return true if need == :write && !write_verified?(effects: effects, request: request, messages: opts[:messages])
|
|
599
612
|
return true if need == :browse && !effects.include?(:browse)
|
|
600
613
|
return true if need == :any && !effects.intersect?(%i[write browse eval])
|
|
601
614
|
|
|
@@ -607,10 +620,27 @@ module PWN
|
|
|
607
620
|
private_class_method def self.write_verified?(opts = {})
|
|
608
621
|
effects = Array(opts[:effects])
|
|
609
622
|
idx = effects.index(:write)
|
|
610
|
-
|
|
623
|
+
if idx
|
|
624
|
+
tail = effects[(idx + 1)..] || []
|
|
625
|
+
return true if tail.intersect?(%i[read eval])
|
|
626
|
+
end
|
|
611
627
|
|
|
612
|
-
|
|
613
|
-
|
|
628
|
+
saw_write_argv = false
|
|
629
|
+
Array(opts[:messages]).each do |msg|
|
|
630
|
+
next unless msg.is_a?(Hash)
|
|
631
|
+
|
|
632
|
+
if msg[:role].to_s == 'assistant' && defined?(Dispatch)
|
|
633
|
+
Array(msg[:tool_calls]).each do |tc|
|
|
634
|
+
blob = Dispatch.send(:argv_blob, args: tc.dig(:function, :arguments) || tc.dig('function', 'arguments'))
|
|
635
|
+
saw_write_argv = true if blob.to_s.match?(Dispatch::WRITE_ARGV_RX)
|
|
636
|
+
end
|
|
637
|
+
elsif saw_write_argv && msg[:role].to_s == 'tool'
|
|
638
|
+
fx = stamped_effect(content: msg[:content])
|
|
639
|
+
return true if %i[read eval].include?(fx)
|
|
640
|
+
end
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
false
|
|
614
644
|
rescue StandardError
|
|
615
645
|
false
|
|
616
646
|
end
|
|
@@ -875,6 +905,7 @@ module PWN
|
|
|
875
905
|
end
|
|
876
906
|
|
|
877
907
|
private_class_method def self.may_finalize?(opts = {})
|
|
908
|
+
return true if evidence_satisfied?(messages: opts[:messages], text: opts[:text], request: opts[:request])
|
|
878
909
|
return false if incomplete_final?(text: opts[:text], last_iter: false)
|
|
879
910
|
return false if request_unsatisfied?(
|
|
880
911
|
request: opts[:request],
|
|
@@ -1678,6 +1709,7 @@ module PWN
|
|
|
1678
1709
|
}
|
|
1679
1710
|
# Ollama + abliterated / weak chat-templates often ignore tools: and
|
|
1680
1711
|
# answer in prose (or print shell(...) as text). Force native
|
|
1712
|
+
|
|
1681
1713
|
# tool_calls until at least one tool result is already in history;
|
|
1682
1714
|
# after that, auto so the model can emit a real final answer.
|
|
1683
1715
|
# Respect explicit PWN::Env[:ai][:ollama][:tool_choice] override.
|
|
@@ -1705,10 +1737,6 @@ module PWN
|
|
|
1705
1737
|
end
|
|
1706
1738
|
end
|
|
1707
1739
|
|
|
1708
|
-
# 3.1 — sliding-window history compaction for local models.
|
|
1709
|
-
# Keep: system, original user, PLAN assistant (if any), last K tool
|
|
1710
|
-
# pairs (assistant+tool), and the most recent assistant. Stale tool
|
|
1711
|
-
# bodies are truncated to history_tool_max_chars.
|
|
1712
1740
|
private_class_method def self.session_chat_history(opts = {})
|
|
1713
1741
|
return [] unless defined?(PWN::Sessions) && PWN::Sessions.respond_to?(:to_llm_messages)
|
|
1714
1742
|
|
|
@@ -1831,15 +1859,15 @@ module PWN
|
|
|
1831
1859
|
|
|
1832
1860
|
private_class_method def self.spill_tool_body(opts = {})
|
|
1833
1861
|
text = opts[:text].to_s
|
|
1834
|
-
digest = Digest::SHA256.hexdigest(text)
|
|
1835
|
-
dir =
|
|
1862
|
+
digest = Digest::SHA256.hexdigest(text)
|
|
1863
|
+
dir = File.join(Dir.home, '.pwn', 'artifacts', 'transcripts', (Thread.current[:pwn_session_id] || 'default').to_s)
|
|
1836
1864
|
FileUtils.mkdir_p(dir)
|
|
1837
|
-
path = File.join(dir, "#{digest}.txt")
|
|
1865
|
+
path = File.join(dir, "#{digest[0, 12]}.txt")
|
|
1838
1866
|
File.binwrite(path, text) unless File.file?(path)
|
|
1839
1867
|
head = text.byteslice(0, 2_048).to_s
|
|
1840
1868
|
tail = text.bytesize > 3_072 ? text.byteslice(-1_024, 1_024).to_s : ''
|
|
1841
1869
|
mid = tail.empty? ? '' : "\n...\n#{tail}"
|
|
1842
|
-
"#{head}#{mid}\n[compacted path=#{path} sha256=#{digest} bytes=#{text.bytesize}]"
|
|
1870
|
+
"#{head}#{mid}\n[compacted path=#{path} sha256=#{digest} bytes=#{text.bytesize} ref=#{path}]"
|
|
1843
1871
|
rescue StandardError
|
|
1844
1872
|
"[compacted bytes=#{opts[:text].to_s.bytesize}]"
|
|
1845
1873
|
end
|
|
@@ -3002,17 +3030,21 @@ module PWN
|
|
|
3002
3030
|
text: text
|
|
3003
3031
|
)
|
|
3004
3032
|
turn_fails['unsatisfied'] += 1
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3033
|
+
if turn_fails['unsatisfied'] >= 2 && evidence_satisfied?(request: request, messages: messages, text: text)
|
|
3034
|
+
debug_progress(msg: 'nag cap: evidence_satisfied after 2 bounces')
|
|
3035
|
+
else
|
|
3036
|
+
unmet = completion_unmet(request: request, messages: messages)
|
|
3037
|
+
warn "[pwn-ai/loop] original request not evidenced on iter=#{i} unmet=#{unmet.join(',')}; continuing"
|
|
3038
|
+
debug_progress(msg: "bounce unsatisfied unmet=#{unmet.join(',')} snippet=#{debug_snippet(text: text)}")
|
|
3039
|
+
messages << {
|
|
3040
|
+
role: 'user',
|
|
3041
|
+
content: "[pwn-ai] The original request is not evidenced yet. unmet=#{unmet.join(',')} " \
|
|
3042
|
+
'Keep calling CORE_TOOLS (shell, pwn_eval) until that request is ' \
|
|
3043
|
+
'done or a tool returned failure evidence. pwn-ai does not decide ' \
|
|
3044
|
+
'authorization. Do not declare completion from a listing or a refusal.'
|
|
3045
|
+
}
|
|
3046
|
+
next
|
|
3047
|
+
end
|
|
3016
3048
|
end
|
|
3017
3049
|
debug_progress(msg: "final accepted chars=#{text.to_s.length}")
|
|
3018
3050
|
quiet_debug_tui!(reason: 'final')
|
|
@@ -3143,14 +3175,26 @@ module PWN
|
|
|
3143
3175
|
TurnFinalizer.leave_user_path! if defined?(TurnFinalizer)
|
|
3144
3176
|
end
|
|
3145
3177
|
|
|
3178
|
+
public_class_method def self.budget_status(opts = {})
|
|
3179
|
+
t0 = opts[:t0] || Thread.current[:pwn_loop_t0] || Time.now
|
|
3180
|
+
elapsed = Time.now - t0
|
|
3181
|
+
remain = (opts[:remaining_s] || Thread.current[:pwn_loop_budget_s] || 10_800).to_f - elapsed
|
|
3182
|
+
{
|
|
3183
|
+
elapsed_s: elapsed.round,
|
|
3184
|
+
remaining_tool_budget_s: [remain, 0].max.round,
|
|
3185
|
+
mutations_used: Thread.current[:pwn_loop_mutations].to_i,
|
|
3186
|
+
mutations_max: 10,
|
|
3187
|
+
context_tokens_used: Thread.current[:pwn_loop_tokens].to_i,
|
|
3188
|
+
est_max: 128_000
|
|
3189
|
+
}
|
|
3190
|
+
end
|
|
3191
|
+
|
|
3146
3192
|
# Author(s):: 0day Inc. <support@0dayinc.com>
|
|
3147
3193
|
|
|
3148
3194
|
public_class_method def self.authors
|
|
3149
3195
|
"AUTHOR(S):\n 0day Inc. <support@0dayinc.com>\n"
|
|
3150
3196
|
end
|
|
3151
3197
|
|
|
3152
|
-
# Display Usage for this Module
|
|
3153
|
-
|
|
3154
3198
|
public_class_method def self.help
|
|
3155
3199
|
puts "USAGE:
|
|
3156
3200
|
# Run debug on and return its result
|
|
@@ -3158,6 +3202,14 @@ module PWN
|
|
|
3158
3202
|
debug: 'optional - debug value consumed by #debug_on?'
|
|
3159
3203
|
)
|
|
3160
3204
|
|
|
3205
|
+
# True when write-then-readback evidence satisfies the original request.
|
|
3206
|
+
#{self}.evidence_satisfied?(
|
|
3207
|
+
messages: 'optional - Array of role/content hashes',
|
|
3208
|
+
trace: 'optional - alias for messages',
|
|
3209
|
+
text: 'optional - final answer text',
|
|
3210
|
+
request: 'optional - original request'
|
|
3211
|
+
)
|
|
3212
|
+
|
|
3161
3213
|
# True only when the ask needs a live host/file/browser effect. World-knowledge
|
|
3162
3214
|
#{self}.catalog_lookup?(
|
|
3163
3215
|
request: 'required - request value consumed by #catalog_lookup?'
|
|
@@ -3201,6 +3253,12 @@ module PWN
|
|
|
3201
3253
|
force_tools: 'optional - force tools value consumed by #run'
|
|
3202
3254
|
)
|
|
3203
3255
|
|
|
3256
|
+
# Remaining time/token/mutation budget for the current loop.
|
|
3257
|
+
#{self}.budget_status(
|
|
3258
|
+
t0: 'optional - session start Time (defaults to thread t0)',
|
|
3259
|
+
remaining_s: 'optional - override remaining tool budget seconds'
|
|
3260
|
+
)
|
|
3261
|
+
|
|
3204
3262
|
# Print the AUTHOR(S) string for this module.
|
|
3205
3263
|
#{self}.authors
|
|
3206
3264
|
"
|