pwn 0.5.663 → 0.5.665
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/pwn/ai/agent/curriculum.rb +12 -4
- data/lib/pwn/ai/agent/learning.rb +443 -14
- data/lib/pwn/ai/agent/loop.rb +25 -5
- data/lib/pwn/ai/agent/mistakes.rb +124 -6
- data/lib/pwn/ai/agent/tools/learning.rb +22 -0
- data/lib/pwn/ai/agent/tools/memory.rb +29 -4
- data/lib/pwn/ai/agent/tools/mistakes.rb +22 -0
- data/lib/pwn/ai/agent/tools/sessions.rb +37 -2
- data/lib/pwn/memory.rb +105 -6
- data/lib/pwn/plugins/repl.rb +2 -2
- data/lib/pwn/sessions.rb +268 -4
- data/lib/pwn/version.rb +1 -1
- data/spec/integration/persistence_roundtrip_spec.rb +2 -2
- data/spec/lib/pwn/ai/agent/lean_stores_spec.rb +269 -0
- data/spec/lib/pwn/ai/agent/learning_spec.rb +137 -0
- data/spec/lib/pwn/ai/agent/loop_spec.rb +42 -0
- data/spec/lib/pwn/ai/agent/mistakes_spec.rb +75 -0
- data/spec/lib/pwn/memory_spec.rb +46 -1
- data/spec/lib/pwn/sessions_spec.rb +40 -1
- data/third_party/pwn_rdoc.jsonl +21 -3
- metadata +2 -1
data/lib/pwn/sessions.rb
CHANGED
|
@@ -13,6 +13,15 @@ module PWN
|
|
|
13
13
|
# session on each activation.
|
|
14
14
|
module Sessions
|
|
15
15
|
SESSIONS_DIR = File.join(Dir.home, '.pwn', 'sessions')
|
|
16
|
+
# Lean retention — pin gold/mistake refs; drop stubs and unreferenced age.
|
|
17
|
+
RETAIN_DAYS = 30
|
|
18
|
+
HOT_DAYS = 14
|
|
19
|
+
MIN_STUB_BYTES = 200
|
|
20
|
+
TOOL_CONTENT_MAX = 1_500
|
|
21
|
+
ASSISTANT_CONTENT_MAX = 8_000
|
|
22
|
+
COMPACT_SIZE_FLOOR = 100_000
|
|
23
|
+
MAX_FILES = 400
|
|
24
|
+
GOLD_MIN_SCORE = 0.6
|
|
16
25
|
|
|
17
26
|
# Supported Method Parameters::
|
|
18
27
|
# dir = PWN::Sessions.sessions_dir
|
|
@@ -74,9 +83,21 @@ module PWN
|
|
|
74
83
|
path = File.join(sessions_dir, "#{sid}.jsonl")
|
|
75
84
|
raise "Session #{sid} not found" unless File.exist?(path)
|
|
76
85
|
|
|
86
|
+
role = (opts[:role] || 'user').to_s
|
|
87
|
+
content = opts[:content]
|
|
88
|
+
if content.is_a?(String)
|
|
89
|
+
# Write-path policy: cap bulk by role using compact limits so append
|
|
90
|
+
# cannot grow transcripts past TOOL_CONTENT_MAX / ASSISTANT_CONTENT_MAX.
|
|
91
|
+
cap = case role
|
|
92
|
+
when 'tool' then TOOL_CONTENT_MAX
|
|
93
|
+
when 'assistant' then ASSISTANT_CONTENT_MAX
|
|
94
|
+
else ASSISTANT_CONTENT_MAX * 2
|
|
95
|
+
end
|
|
96
|
+
content = "#{content[0, cap]}…[compacted]" if content.bytesize > cap
|
|
97
|
+
end
|
|
77
98
|
entry = {
|
|
78
|
-
role:
|
|
79
|
-
content:
|
|
99
|
+
role: role,
|
|
100
|
+
content: content,
|
|
80
101
|
timestamp: Time.now.utc.iso8601
|
|
81
102
|
}
|
|
82
103
|
File.open(path, 'a') { |f| f.puts(JSON.dump(entry)) }
|
|
@@ -115,10 +136,22 @@ module PWN
|
|
|
115
136
|
end
|
|
116
137
|
|
|
117
138
|
# Supported Method Parameters::
|
|
118
|
-
# PWN::Sessions.delete(
|
|
139
|
+
# PWN::Sessions.delete(
|
|
140
|
+
# session_id: 'required',
|
|
141
|
+
# force: 'optional - Boolean bypass pin policy (default false)'
|
|
142
|
+
# )
|
|
143
|
+
# Refuses deletion of pinned sessions (hot/gold/mistake/current) unless force:true.
|
|
119
144
|
public_class_method def self.delete(opts = {}) # rubocop:disable Naming/PredicateMethod
|
|
120
145
|
sid = opts[:session_id]
|
|
146
|
+
raise 'ERROR: session_id required' if sid.to_s.empty?
|
|
147
|
+
|
|
148
|
+
force = opts[:force] ? true : false
|
|
121
149
|
path = File.join(sessions_dir, "#{sid}.jsonl")
|
|
150
|
+
unless force
|
|
151
|
+
pins = protected_session_ids(current_session_id: opts[:current_session_id])
|
|
152
|
+
raise "ERROR: refusing to delete protected session #{sid.inspect} (pinned by HOT_DAYS/gold/mistakes/current; pass force:true)" if pins.include?(sid.to_s)
|
|
153
|
+
end
|
|
154
|
+
|
|
122
155
|
FileUtils.rm_f(path)
|
|
123
156
|
true
|
|
124
157
|
end
|
|
@@ -135,6 +168,236 @@ module PWN
|
|
|
135
168
|
}
|
|
136
169
|
end
|
|
137
170
|
|
|
171
|
+
# Supported Method Parameters::
|
|
172
|
+
# result = PWN::Sessions.lean!(
|
|
173
|
+
# dry_run: 'optional - Boolean (default false)',
|
|
174
|
+
# retain_days: 'optional - drop unreferenced older than this',
|
|
175
|
+
# hot_days: 'optional - always keep newer than this',
|
|
176
|
+
# max_files: 'optional - soft file cap after protections',
|
|
177
|
+
# current_session_id: 'optional - never delete this id'
|
|
178
|
+
# )
|
|
179
|
+
#
|
|
180
|
+
# Reference-protect sessions cited by gold learning outcomes and open
|
|
181
|
+
# mistakes. Delete stubs and aged unreferenced files. Compact oversized
|
|
182
|
+
# tool/assistant content on kept cold transcripts.
|
|
183
|
+
public_class_method def self.lean!(opts = {})
|
|
184
|
+
dry = opts[:dry_run] ? true : false
|
|
185
|
+
retain_days = (opts[:retain_days] || RETAIN_DAYS).to_f
|
|
186
|
+
hot_days = (opts[:hot_days] || HOT_DAYS).to_f
|
|
187
|
+
max_files = (opts[:max_files] || MAX_FILES).to_i
|
|
188
|
+
tool_max = (opts[:tool_content_max] || TOOL_CONTENT_MAX).to_i
|
|
189
|
+
asst_max = (opts[:assistant_content_max] || ASSISTANT_CONTENT_MAX).to_i
|
|
190
|
+
current = opts[:current_session_id].to_s
|
|
191
|
+
current = PWN::Env.dig(:ai, :session_id).to_s if current.empty? && defined?(PWN::Env)
|
|
192
|
+
|
|
193
|
+
dir = sessions_dir
|
|
194
|
+
files = Dir.glob(File.join(dir, '*.jsonl'))
|
|
195
|
+
before_bytes = files.sum { |f| File.size(f) }
|
|
196
|
+
now = Time.now.utc
|
|
197
|
+
|
|
198
|
+
protect = protected_session_ids(
|
|
199
|
+
current_session_id: current,
|
|
200
|
+
hot_days: hot_days,
|
|
201
|
+
retain_days: retain_days,
|
|
202
|
+
now: now
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
deleted = []
|
|
206
|
+
compacted = []
|
|
207
|
+
protected_n = 0
|
|
208
|
+
|
|
209
|
+
# Classify
|
|
210
|
+
entries = files.map do |path|
|
|
211
|
+
id = File.basename(path, '.jsonl')
|
|
212
|
+
mtime = File.mtime(path).utc
|
|
213
|
+
age = (now - mtime) / 86_400.0
|
|
214
|
+
size = File.size(path)
|
|
215
|
+
pinned = protect.include?(id) || age <= hot_days
|
|
216
|
+
protected_n += 1 if pinned
|
|
217
|
+
{ id: id, path: path, age: age, size: size, pinned: pinned, mtime: mtime }
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Delete stubs and aged unreferenced
|
|
221
|
+
entries.each do |e|
|
|
222
|
+
next if e[:pinned]
|
|
223
|
+
|
|
224
|
+
stub = e[:size] < MIN_STUB_BYTES && stub_session?(path: e[:path])
|
|
225
|
+
aged = e[:age] > retain_days
|
|
226
|
+
next unless stub || aged
|
|
227
|
+
|
|
228
|
+
deleted << e[:id]
|
|
229
|
+
FileUtils.rm_f(e[:path]) unless dry
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# Refresh remaining after deletes
|
|
233
|
+
remaining = entries.reject { |e| deleted.include?(e[:id]) }
|
|
234
|
+
|
|
235
|
+
# Enforce max_files on unpinned oldest-first
|
|
236
|
+
if remaining.size > max_files
|
|
237
|
+
unpinned = remaining.reject { |e| e[:pinned] }.sort_by { |e| e[:mtime] }
|
|
238
|
+
over = remaining.size - max_files
|
|
239
|
+
unpinned.first(over).each do |e|
|
|
240
|
+
deleted << e[:id]
|
|
241
|
+
FileUtils.rm_f(e[:path]) unless dry
|
|
242
|
+
remaining.delete(e)
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Compact large/cold kept files
|
|
247
|
+
remaining.each do |e|
|
|
248
|
+
next if e[:age] <= hot_days && e[:size] < COMPACT_SIZE_FLOOR
|
|
249
|
+
next if e[:size] < COMPACT_SIZE_FLOOR && e[:age] <= retain_days
|
|
250
|
+
|
|
251
|
+
changed = compact_transcript!(
|
|
252
|
+
path: e[:path],
|
|
253
|
+
tool_max: tool_max,
|
|
254
|
+
asst_max: asst_max,
|
|
255
|
+
dry_run: dry
|
|
256
|
+
)
|
|
257
|
+
compacted << e[:id] if changed
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
after_files = Dir.glob(File.join(dir, '*.jsonl'))
|
|
261
|
+
after_bytes = after_files.sum { |f| File.size(f) }
|
|
262
|
+
|
|
263
|
+
{
|
|
264
|
+
deleted: deleted.length,
|
|
265
|
+
deleted_ids: deleted.first(30),
|
|
266
|
+
compacted: compacted.length,
|
|
267
|
+
compacted_ids: compacted.first(30),
|
|
268
|
+
protected: protected_n,
|
|
269
|
+
remaining: after_files.size,
|
|
270
|
+
bytes_before: before_bytes,
|
|
271
|
+
bytes_after: dry ? before_bytes : after_bytes,
|
|
272
|
+
dry_run: dry
|
|
273
|
+
}
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Build the pin set from learning gold + open mistakes + hot files + current.
|
|
277
|
+
public_class_method def self.protected_session_ids(opts = {})
|
|
278
|
+
now = opts[:now] || Time.now.utc
|
|
279
|
+
hot_days = (opts[:hot_days] || HOT_DAYS).to_f
|
|
280
|
+
retain_days = (opts[:retain_days] || RETAIN_DAYS).to_f
|
|
281
|
+
pins = {}
|
|
282
|
+
pins[opts[:current_session_id].to_s] = true unless opts[:current_session_id].to_s.empty?
|
|
283
|
+
|
|
284
|
+
# Hot by mtime
|
|
285
|
+
Dir.glob(File.join(sessions_dir, '*.jsonl')).each do |path|
|
|
286
|
+
age = (now - File.mtime(path).utc) / 86_400.0
|
|
287
|
+
pins[File.basename(path, '.jsonl')] = true if age <= hot_days
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# Learning gold
|
|
291
|
+
learn_file = File.join(Dir.home, '.pwn', 'learning.jsonl')
|
|
292
|
+
if File.exist?(learn_file)
|
|
293
|
+
File.foreach(learn_file) do |line|
|
|
294
|
+
r = JSON.parse(line, symbolize_names: true)
|
|
295
|
+
sid = r[:session_id].to_s
|
|
296
|
+
next if sid.empty?
|
|
297
|
+
|
|
298
|
+
gold = r[:success] == true && (
|
|
299
|
+
!r.key?(:score) || r[:score].to_f >= GOLD_MIN_SCORE
|
|
300
|
+
)
|
|
301
|
+
pins[sid] = true if gold
|
|
302
|
+
rescue StandardError
|
|
303
|
+
next
|
|
304
|
+
end
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Mistakes refs
|
|
308
|
+
mist_file = File.join(Dir.home, '.pwn', 'mistakes.json')
|
|
309
|
+
if File.exist?(mist_file)
|
|
310
|
+
begin
|
|
311
|
+
store = JSON.parse(File.read(mist_file), symbolize_names: true)
|
|
312
|
+
store.each_value do |m|
|
|
313
|
+
next unless m.is_a?(Hash)
|
|
314
|
+
|
|
315
|
+
openish = !m[:resolved] || m[:regressed]
|
|
316
|
+
recent_closed = false
|
|
317
|
+
if m[:resolved] && m[:last_seen]
|
|
318
|
+
begin
|
|
319
|
+
recent_closed = ((now - Time.parse(m[:last_seen].to_s)) / 86_400.0) <= retain_days
|
|
320
|
+
rescue StandardError
|
|
321
|
+
recent_closed = false
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
next unless openish || recent_closed
|
|
325
|
+
|
|
326
|
+
Array(m[:sessions]).each { |s| pins[s.to_s] = true if s.to_s != '' }
|
|
327
|
+
end
|
|
328
|
+
rescue StandardError
|
|
329
|
+
# ignore
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
pins.keys.reject(&:empty?)
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
private_class_method def self.stub_session?(opts = {})
|
|
337
|
+
path = opts[:path]
|
|
338
|
+
return true unless File.exist?(path)
|
|
339
|
+
|
|
340
|
+
rows = File.readlines(path).map do |l|
|
|
341
|
+
JSON.parse(l, symbolize_names: true)
|
|
342
|
+
rescue StandardError
|
|
343
|
+
nil
|
|
344
|
+
end.compact
|
|
345
|
+
substantive = rows.any? do |e|
|
|
346
|
+
role = e[:role].to_s
|
|
347
|
+
%w[user assistant].include?(role) && e[:content].to_s.strip.length > 20
|
|
348
|
+
end
|
|
349
|
+
!substantive
|
|
350
|
+
rescue StandardError
|
|
351
|
+
true
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
private_class_method def self.compact_transcript!(opts = {})
|
|
355
|
+
path = opts[:path]
|
|
356
|
+
tool_max = opts[:tool_max] || TOOL_CONTENT_MAX
|
|
357
|
+
asst_max = opts[:assistant_content_max] || opts[:asst_max] || ASSISTANT_CONTENT_MAX
|
|
358
|
+
dry = opts[:dry_run] ? true : false
|
|
359
|
+
return false unless File.exist?(path)
|
|
360
|
+
|
|
361
|
+
lines = File.readlines(path)
|
|
362
|
+
rows = lines.map do |l|
|
|
363
|
+
JSON.parse(l, symbolize_names: true)
|
|
364
|
+
rescue StandardError
|
|
365
|
+
nil
|
|
366
|
+
end.compact
|
|
367
|
+
return false if rows.empty?
|
|
368
|
+
|
|
369
|
+
# Preserve last assistant fully-ish; compact tools and older assistants
|
|
370
|
+
last_asst_i = rows.rindex { |e| e[:role].to_s == 'assistant' }
|
|
371
|
+
changed = false
|
|
372
|
+
rows.each_with_index do |e, i|
|
|
373
|
+
role = e[:role].to_s
|
|
374
|
+
content = e[:content].to_s
|
|
375
|
+
if role == 'tool' && content.bytesize > tool_max
|
|
376
|
+
e[:content] = "#{content[0, tool_max]}…[compacted]"
|
|
377
|
+
changed = true
|
|
378
|
+
elsif role == 'assistant' && i != last_asst_i && content.bytesize > asst_max
|
|
379
|
+
e[:content] = "#{content[0, asst_max]}…[compacted]"
|
|
380
|
+
changed = true
|
|
381
|
+
elsif role == 'assistant' && i == last_asst_i && content.bytesize > (asst_max * 2)
|
|
382
|
+
e[:content] = "#{content[0, asst_max * 2]}…[compacted]"
|
|
383
|
+
changed = true
|
|
384
|
+
end
|
|
385
|
+
end
|
|
386
|
+
return changed if dry || !changed
|
|
387
|
+
|
|
388
|
+
tmp = "#{path}.#{Process.pid}.tmp"
|
|
389
|
+
File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC, 0o644) do |f|
|
|
390
|
+
f.flock(File::LOCK_EX)
|
|
391
|
+
rows.each { |e| f.puts(JSON.generate(e)) }
|
|
392
|
+
f.flush
|
|
393
|
+
f.fsync
|
|
394
|
+
end
|
|
395
|
+
File.rename(tmp, path)
|
|
396
|
+
true
|
|
397
|
+
ensure
|
|
398
|
+
FileUtils.rm_f(tmp) if defined?(tmp) && tmp && File.exist?(tmp)
|
|
399
|
+
end
|
|
400
|
+
|
|
138
401
|
# Author(s):: 0day Inc. <support@0dayinc.com>
|
|
139
402
|
|
|
140
403
|
public_class_method def self.authors
|
|
@@ -151,7 +414,8 @@ module PWN
|
|
|
151
414
|
hist = PWN::Sessions.to_response_history(session_id: sess[:id])
|
|
152
415
|
PWN::Sessions.list
|
|
153
416
|
PWN::Sessions.stats
|
|
154
|
-
PWN::Sessions.delete(session_id: sess[:id])
|
|
417
|
+
PWN::Sessions.delete(session_id: sess[:id], force: true)
|
|
418
|
+
PWN::Sessions.lean!(dry_run: true) # pin gold/mistake refs; drop stubs/age
|
|
155
419
|
|
|
156
420
|
#{self}.authors
|
|
157
421
|
USAGE
|
data/lib/pwn/version.rb
CHANGED
|
@@ -25,7 +25,7 @@ RSpec.describe 'PWN persistence stores — write → reload → delete round-tri
|
|
|
25
25
|
hist = PWN::Sessions.to_response_history(session_id: s[:id])
|
|
26
26
|
expect(hist).to be_a(Hash).or be_an(Array)
|
|
27
27
|
expect(PWN::Sessions.stats[:total_sessions]).to be >= 1
|
|
28
|
-
expect(PWN::Sessions.delete(session_id: s[:id])).to be_truthy
|
|
28
|
+
expect(PWN::Sessions.delete(session_id: s[:id], force: true)).to be_truthy
|
|
29
29
|
expect(PWN::Sessions.load(session_id: s[:id])).to eq([])
|
|
30
30
|
end
|
|
31
31
|
end
|
|
@@ -38,7 +38,7 @@ RSpec.describe 'PWN persistence stores — write → reload → delete round-tri
|
|
|
38
38
|
expect(PWN::Memory.forget(key: 'rt_key')).to be_truthy
|
|
39
39
|
expect(PWN::Memory.recall(query: 'needle')).to be_empty
|
|
40
40
|
PWN::Memory.remember(key: 'x', value: 'y')
|
|
41
|
-
PWN::Memory.clear
|
|
41
|
+
PWN::Memory.clear(force: true)
|
|
42
42
|
expect(PWN::Memory.load).to eq({})
|
|
43
43
|
end
|
|
44
44
|
end
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'fileutils'
|
|
6
|
+
require 'tmpdir'
|
|
7
|
+
require 'time'
|
|
8
|
+
|
|
9
|
+
RSpec.describe 'lean RL stores' do
|
|
10
|
+
let(:tmpdir) { Dir.mktmpdir('pwn-lean-') }
|
|
11
|
+
after { FileUtils.rm_rf(tmpdir) }
|
|
12
|
+
|
|
13
|
+
def with_store_paths
|
|
14
|
+
mem = File.join(tmpdir, 'memory.json')
|
|
15
|
+
learn = File.join(tmpdir, 'learning.jsonl')
|
|
16
|
+
mist = File.join(tmpdir, 'mistakes.json')
|
|
17
|
+
sess = File.join(tmpdir, 'sessions')
|
|
18
|
+
FileUtils.mkdir_p(sess)
|
|
19
|
+
|
|
20
|
+
stub_const('PWN::Memory::MEMORY_FILE', mem)
|
|
21
|
+
stub_const('PWN::AI::Agent::Learning::LEARNING_FILE', learn)
|
|
22
|
+
stub_const('PWN::AI::Agent::Mistakes::MISTAKES_FILE', mist)
|
|
23
|
+
stub_const('PWN::Sessions::SESSIONS_DIR', sess)
|
|
24
|
+
yield(mem: mem, learn: learn, mist: mist, sess: sess)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
it 'Memory.lean! drops expired session_* but keeps prefs and SOPs' do
|
|
28
|
+
with_store_paths do
|
|
29
|
+
PWN::Memory.remember(key: :operator_pref_x, value: 'keep me', category: :preference, importance: 1.0)
|
|
30
|
+
PWN::Memory.remember(key: :process_sop_y, value: 'always rake', category: :lesson, importance: 1.0)
|
|
31
|
+
PWN::Memory.remember(
|
|
32
|
+
key: :session_old_context,
|
|
33
|
+
value: 'ephemeral',
|
|
34
|
+
category: :fact,
|
|
35
|
+
ttl: 1
|
|
36
|
+
)
|
|
37
|
+
# backdate
|
|
38
|
+
m = PWN::Memory.load
|
|
39
|
+
m[:session_old_context][:timestamp] = (Time.now.utc - 86_400).iso8601
|
|
40
|
+
PWN::Memory.save(mem: m)
|
|
41
|
+
|
|
42
|
+
r = PWN::Memory.lean!
|
|
43
|
+
expect(r[:removed]).to be >= 1
|
|
44
|
+
left = PWN::Memory.load
|
|
45
|
+
expect(left.keys.map(&:to_s)).to include('operator_pref_x', 'process_sop_y')
|
|
46
|
+
expect(left.keys.map(&:to_s)).not_to include('session_old_context')
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
it 'Learning.prune_outcomes! keeps gold success+score and never empties failure window' do
|
|
51
|
+
with_store_paths do |paths|
|
|
52
|
+
now = Time.now.utc
|
|
53
|
+
rows = []
|
|
54
|
+
# gold
|
|
55
|
+
5.times do |i|
|
|
56
|
+
rows << {
|
|
57
|
+
id: "g#{i}", task: "gold task #{i}", success: true, score: 0.9,
|
|
58
|
+
session_id: "sid_gold_#{i}", tags: %w[auto loop], details: 'x' * 50,
|
|
59
|
+
timestamp: (now - i).iso8601
|
|
60
|
+
}
|
|
61
|
+
end
|
|
62
|
+
# bulk old low-score auto noise
|
|
63
|
+
40.times do |i|
|
|
64
|
+
rows << {
|
|
65
|
+
id: "n#{i}", task: "noise #{i % 3}", success: false, score: 0.1,
|
|
66
|
+
session_id: "sid_n_#{i}", tags: %w[auto loop wrong], details: 'y' * 1200,
|
|
67
|
+
timestamp: (now - ((50 + i) * 86_400)).iso8601
|
|
68
|
+
}
|
|
69
|
+
end
|
|
70
|
+
File.write(paths[:learn], "#{rows.map { |r| JSON.generate(r) }.join("\n")}\n")
|
|
71
|
+
|
|
72
|
+
r = PWN::AI::Agent::Learning.prune_outcomes!(max_rows: 30, retain_days: 45, recent_days: 14)
|
|
73
|
+
kept = File.readlines(paths[:learn]).map { |l| JSON.parse(l, symbolize_names: true) }
|
|
74
|
+
gold_left = kept.select { |r| r[:id].to_s.start_with?('g') }
|
|
75
|
+
expect(gold_left.size).to eq(5)
|
|
76
|
+
expect(kept.size).to be <= 30
|
|
77
|
+
expect(r[:kept]).to eq(kept.size)
|
|
78
|
+
# details truncated
|
|
79
|
+
expect(kept.any? { |r| r[:details].to_s.include?('[compacted]') || r[:details].to_s.bytesize <= 800 }).to be true
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
it 'Mistakes.lean! never drops unresolved signatures' do
|
|
84
|
+
with_store_paths do |paths|
|
|
85
|
+
store = {
|
|
86
|
+
open1: {
|
|
87
|
+
signature: 'open1', tool: 'shell', error: 'boom', snippet: 'x' * 300,
|
|
88
|
+
sample_args: 'a' * 400, count: 2, first_seen: Time.now.utc.iso8601,
|
|
89
|
+
last_seen: Time.now.utc.iso8601, sessions: %w[s1 s2 s3 s4],
|
|
90
|
+
resolved: false, fix: nil
|
|
91
|
+
},
|
|
92
|
+
oldres: {
|
|
93
|
+
signature: 'oldres', tool: 'shell', error: 'old', snippet: 's',
|
|
94
|
+
sample_args: 'a', count: 1, first_seen: (Time.now.utc - (40 * 86_400)).iso8601,
|
|
95
|
+
last_seen: (Time.now.utc - (40 * 86_400)).iso8601,
|
|
96
|
+
resolved_at: (Time.now.utc - (40 * 86_400)).iso8601,
|
|
97
|
+
sessions: %w[s9], resolved: true, fix: 'do the thing'
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
File.write(paths[:mist], JSON.pretty_generate(store))
|
|
101
|
+
# memory already has fix lesson
|
|
102
|
+
PWN::Memory.remember(key: :mistake_fix_oldres, value: 'AVOID: shell → old — FIX: do the thing', category: :lesson)
|
|
103
|
+
|
|
104
|
+
r = PWN::AI::Agent::Mistakes.lean!
|
|
105
|
+
after = PWN::AI::Agent::Mistakes.load
|
|
106
|
+
expect(after.keys.map(&:to_s)).to include('open1')
|
|
107
|
+
expect(after[:open1][:sample_args].to_s.bytesize).to be <= PWN::AI::Agent::Mistakes::SAMPLE_ARGS_MAX
|
|
108
|
+
expect(Array(after[:open1][:sessions]).size).to be <= PWN::AI::Agent::Mistakes::SESSIONS_KEEP
|
|
109
|
+
expect(r[:unresolved]).to eq(1)
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
it 'Sessions.lean! pins gold session ids and deletes unreferenced stubs' do
|
|
114
|
+
with_store_paths do |paths|
|
|
115
|
+
gold_id = '20260101_000000_gold0001'
|
|
116
|
+
stub_id = '20260101_000000_stub0001'
|
|
117
|
+
old_id = '20260101_000000_old00001'
|
|
118
|
+
|
|
119
|
+
File.write(File.join(paths[:sess], "#{gold_id}.jsonl"), "#{JSON.generate(role: 'user', content: 'do gold work here', timestamp: Time.now.utc.iso8601)}\n")
|
|
120
|
+
File.write(File.join(paths[:sess], "#{stub_id}.jsonl"), "#{JSON.generate(role: 'system', content: 'Session started', timestamp: Time.now.utc.iso8601)}\n")
|
|
121
|
+
File.write(File.join(paths[:sess], "#{old_id}.jsonl"), "#{JSON.generate(role: 'user', content: 'old stuff session', timestamp: (Time.now.utc - (60 * 86_400)).iso8601)}\n")
|
|
122
|
+
# age the old/stub mtimes
|
|
123
|
+
old_t = Time.now - (60 * 86_400)
|
|
124
|
+
File.utime(old_t, old_t, File.join(paths[:sess], "#{old_id}.jsonl"))
|
|
125
|
+
File.utime(old_t, old_t, File.join(paths[:sess], "#{stub_id}.jsonl"))
|
|
126
|
+
|
|
127
|
+
gold_row = {
|
|
128
|
+
id: '1', task: 't', success: true, score: 0.95, session_id: gold_id,
|
|
129
|
+
tags: [], details: 'ok', timestamp: Time.now.utc.iso8601
|
|
130
|
+
}
|
|
131
|
+
File.write(paths[:learn], "#{JSON.generate(gold_row)}\n")
|
|
132
|
+
File.write(paths[:mist], '{}')
|
|
133
|
+
|
|
134
|
+
r = PWN::Sessions.lean!(retain_days: 30, hot_days: 14, max_files: 400)
|
|
135
|
+
expect(File.exist?(File.join(paths[:sess], "#{gold_id}.jsonl"))).to be true
|
|
136
|
+
expect(File.exist?(File.join(paths[:sess], "#{stub_id}.jsonl"))).to be false
|
|
137
|
+
expect(File.exist?(File.join(paths[:sess], "#{old_id}.jsonl"))).to be false
|
|
138
|
+
expect(r[:deleted]).to be >= 2
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
it 'Learning.consolidate does not evict protected operator_pref keys' do
|
|
143
|
+
with_store_paths do
|
|
144
|
+
5.times do |i|
|
|
145
|
+
PWN::Memory.remember(key: :"noise_#{i}", value: "noise lesson #{i} " * 5, category: :lesson, importance: 0.1, confidence: 0.2)
|
|
146
|
+
end
|
|
147
|
+
PWN::Memory.remember(key: :operator_pref_keep, value: 'must survive', category: :preference, importance: 1.0, confidence: 1.0)
|
|
148
|
+
r = PWN::AI::Agent::Learning.consolidate(max_entries: 3)
|
|
149
|
+
left = PWN::Memory.load
|
|
150
|
+
expect(left.keys.map(&:to_s)).to include('operator_pref_keep')
|
|
151
|
+
expect(left.size).to be <= 3
|
|
152
|
+
expect(r[:remaining]).to eq(left.size)
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
it 'Learning.should_gc_stores? gates on PRUNE_EVERY_N_APPENDS and MAX_OUTCOME_ROWS' do
|
|
157
|
+
with_store_paths do |paths|
|
|
158
|
+
# empty / missing → true (force initial lean opportunity)
|
|
159
|
+
FileUtils.rm_f(paths[:learn])
|
|
160
|
+
expect(PWN::AI::Agent::Learning.send(:should_gc_stores?)).to be true
|
|
161
|
+
|
|
162
|
+
n = PWN::AI::Agent::Learning::PRUNE_EVERY_N_APPENDS
|
|
163
|
+
rows = n.times.map do |i|
|
|
164
|
+
{
|
|
165
|
+
id: "r#{i}", task: "t#{i}", success: false, score: 0.2,
|
|
166
|
+
session_id: "s#{i}", tags: %w[auto loop], details: 'd',
|
|
167
|
+
timestamp: Time.now.utc.iso8601
|
|
168
|
+
}
|
|
169
|
+
end
|
|
170
|
+
File.write(paths[:learn], "#{rows.map { |r| JSON.generate(r) }.join("\n")}\n")
|
|
171
|
+
expect(PWN::AI::Agent::Learning.send(:should_gc_stores?)).to be true
|
|
172
|
+
|
|
173
|
+
# one under the modulo boundary
|
|
174
|
+
short = rows.first(n - 1)
|
|
175
|
+
File.write(paths[:learn], "#{short.map { |r| JSON.generate(r) }.join("\n")}\n")
|
|
176
|
+
expect(PWN::AI::Agent::Learning.send(:should_gc_stores?)).to be false
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
it 'Learning.gc_stores! leans memory+learning+mistakes+sessions and pins current_session_id' do
|
|
181
|
+
with_store_paths do |paths|
|
|
182
|
+
now = Time.now.utc
|
|
183
|
+
cur = '20260101_000000_current1'
|
|
184
|
+
stub_id = '20260101_000000_stub9999'
|
|
185
|
+
File.write(File.join(paths[:sess], "#{cur}.jsonl"), "#{JSON.generate(role: 'user', content: 'active turn', timestamp: now.iso8601)}\n")
|
|
186
|
+
File.write(File.join(paths[:sess], "#{stub_id}.jsonl"), "#{JSON.generate(role: 'system', content: 'Session started', timestamp: now.iso8601)}\n")
|
|
187
|
+
old_t = Time.now - (60 * 86_400)
|
|
188
|
+
File.utime(old_t, old_t, File.join(paths[:sess], "#{stub_id}.jsonl"))
|
|
189
|
+
|
|
190
|
+
PWN::Memory.remember(key: :session_tmp_gc, value: 'ephemeral', category: :fact, ttl: 1)
|
|
191
|
+
m = PWN::Memory.load
|
|
192
|
+
m[:session_tmp_gc][:timestamp] = (now - 86_400).iso8601
|
|
193
|
+
PWN::Memory.save(mem: m)
|
|
194
|
+
|
|
195
|
+
# modulo-aligned noise outcomes
|
|
196
|
+
n = PWN::AI::Agent::Learning::PRUNE_EVERY_N_APPENDS
|
|
197
|
+
rows = n.times.map do |i|
|
|
198
|
+
{
|
|
199
|
+
id: "n#{i}", task: "noise #{i % 2}", success: false, score: 0.1,
|
|
200
|
+
session_id: "sid_#{i}", tags: %w[auto loop wrong], details: 'z' * 50,
|
|
201
|
+
timestamp: (now - ((50 + i) * 86_400)).iso8601
|
|
202
|
+
}
|
|
203
|
+
end
|
|
204
|
+
File.write(paths[:learn], "#{rows.map { |r| JSON.generate(r) }.join("\n")}\n")
|
|
205
|
+
File.write(paths[:mist], JSON.pretty_generate({}))
|
|
206
|
+
|
|
207
|
+
res = PWN::AI::Agent::Learning.gc_stores!(current_session_id: cur)
|
|
208
|
+
expect(res).to include(:memory, :learning, :mistakes, :sessions)
|
|
209
|
+
expect(File.exist?(File.join(paths[:sess], "#{cur}.jsonl"))).to be true
|
|
210
|
+
expect(File.exist?(File.join(paths[:sess], "#{stub_id}.jsonl"))).to be false
|
|
211
|
+
left_mem = PWN::Memory.load
|
|
212
|
+
expect(left_mem.keys.map(&:to_s)).not_to include('session_tmp_gc')
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
it 'auto_introspect runs :lean_gc via gc_stores! when should_gc_stores?' do
|
|
217
|
+
with_store_paths do |paths|
|
|
218
|
+
# Enable auto_introspect for this process
|
|
219
|
+
PWN::Env[:ai] ||= {}
|
|
220
|
+
PWN::Env[:ai][:agent] ||= {}
|
|
221
|
+
prev = PWN::Env[:ai][:agent][:auto_introspect]
|
|
222
|
+
PWN::Env[:ai][:agent][:auto_introspect] = true
|
|
223
|
+
|
|
224
|
+
n = PWN::AI::Agent::Learning::PRUNE_EVERY_N_APPENDS
|
|
225
|
+
now = Time.now.utc
|
|
226
|
+
# Leave learning at n-1 so note_outcome append hits modulo n and should_gc fires
|
|
227
|
+
rows = (n - 1).times.map do |i|
|
|
228
|
+
{
|
|
229
|
+
id: "pre#{i}", task: "pre #{i}", success: true, score: 0.9,
|
|
230
|
+
session_id: "sid_pre_#{i}", tags: %w[auto loop solved], details: 'ok',
|
|
231
|
+
timestamp: now.iso8601
|
|
232
|
+
}
|
|
233
|
+
end
|
|
234
|
+
File.write(paths[:learn], "#{rows.map { |r| JSON.generate(r) }.join("\n")}\n")
|
|
235
|
+
File.write(paths[:mist], '{}')
|
|
236
|
+
FileUtils.mkdir_p(paths[:sess])
|
|
237
|
+
|
|
238
|
+
called = []
|
|
239
|
+
allow(PWN::AI::Agent::Learning).to receive(:gc_stores!).and_wrap_original do |m, **opts|
|
|
240
|
+
called << opts
|
|
241
|
+
m.call(**opts)
|
|
242
|
+
end
|
|
243
|
+
# Avoid LLM/slow stages
|
|
244
|
+
allow(PWN::AI::Agent::Reward).to receive(:judge).and_return(
|
|
245
|
+
{ score: 0.8, success: true, verdict: :solved, rationale: 'ok', confidence: 0.7 }
|
|
246
|
+
)
|
|
247
|
+
allow(PWN::AI::Agent::Reward).to receive(:prm)
|
|
248
|
+
allow(PWN::AI::Agent::Reward).to receive(:sentinel)
|
|
249
|
+
allow(PWN::AI::Agent::Curriculum).to receive(:critic).and_return(verdict: :pass, source: :test)
|
|
250
|
+
allow(PWN::AI::Agent::Learning).to receive(:reflect)
|
|
251
|
+
allow(PWN::AI::Agent::Extrospection).to receive(:auto_extrospect) if defined?(PWN::AI::Agent::Extrospection)
|
|
252
|
+
|
|
253
|
+
sid = '20260101_000000_introspect'
|
|
254
|
+
File.write(File.join(paths[:sess], "#{sid}.jsonl"), "#{JSON.generate(role: 'user', content: 'hi', timestamp: now.iso8601)}\n")
|
|
255
|
+
|
|
256
|
+
out = PWN::AI::Agent::Learning.auto_introspect(
|
|
257
|
+
session_id: sid,
|
|
258
|
+
request: 'keep stores lean please',
|
|
259
|
+
final: 'done lean'
|
|
260
|
+
)
|
|
261
|
+
expect(out).to be_a(Hash)
|
|
262
|
+
expect(Array(out[:stages_run])).to include(:lean_gc)
|
|
263
|
+
expect(called).not_to be_empty
|
|
264
|
+
expect(called.last[:current_session_id]).to eq(sid)
|
|
265
|
+
ensure
|
|
266
|
+
PWN::Env[:ai][:agent][:auto_introspect] = prev if defined?(prev)
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|