pwn 0.5.783 → 0.5.784
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +3 -3
- data/Rakefile +19 -1
- data/bin/pwn_setup +1 -1
- data/build_gem.sh +1 -1
- data/documentation/Backup-Retention.md +27 -0
- data/documentation/Cron.md +64 -35
- data/documentation/How-PWN-Works.md +1 -1
- data/documentation/Installation.md +247 -6
- data/documentation/Persistence.md +1 -1
- data/documentation/Reinforcement-Learning.md +5 -6
- data/documentation/diagrams/cron-scheduling.svg +130 -128
- data/documentation/diagrams/dot/cron-scheduling.dot +3 -3
- data/etc/default_cron/session_cleanup.rb +1217 -10
- data/lib/pwn/ai/agent/extrospection.rb +6 -7
- data/lib/pwn/ai/agent/learning.rb +48 -7
- data/lib/pwn/ai/agent/policy.rb +5 -0
- data/lib/pwn/ai/agent/swarm.rb +31 -16
- data/lib/pwn/ai/router.rb +45 -2
- data/lib/pwn/cron.rb +120 -49
- data/lib/pwn/migrate.rb +292 -15
- data/lib/pwn/plugins/binary_parser.rb +5 -4
- data/lib/pwn/plugins/ghidra.rb +4 -3
- data/lib/pwn/plugins/log.rb +18 -2
- data/lib/pwn/plugins/repl/ai.rb +12 -1
- data/lib/pwn/plugins/repl/ai_console.rb +6 -2
- data/lib/pwn/plugins/repl/ai_swarm.rb +4 -4
- data/lib/pwn/sessions.rb +58 -16
- data/lib/pwn/setup.rb +9 -17
- data/lib/pwn/version.rb +1 -1
- data/scripts/build_gate.rb +58 -0
- data/spec/integration/build_gate_spec.rb +73 -0
- data/spec/integration/persistence_roundtrip_spec.rb +1 -7
- data/spec/lib/pwn/ai/agent/policy_spec.rb +27 -0
- data/spec/lib/pwn/ai/router_spec.rb +30 -2
- data/spec/lib/pwn/cron_spec.rb +113 -6
- data/spec/lib/pwn/migrate_spec.rb +160 -5
- data/spec/lib/pwn/plugins/log_spec.rb +7 -0
- data/spec/lib/pwn/plugins/repl/ai_console_spec.rb +16 -4
- data/spec/lib/pwn/plugins/repl/ai_swarm_spec.rb +11 -0
- data/spec/lib/pwn/redaction_spec.rb +13 -10
- data/spec/lib/pwn/session_cleanup_concurrency_spec.rb +201 -0
- data/spec/lib/pwn/session_cleanup_default_spec.rb +165 -2
- data/spec/lib/pwn/session_cleanup_legacy_spec.rb +101 -0
- data/spec/lib/pwn/session_cleanup_payload_spec.rb +335 -2
- data/spec/lib/pwn/session_cleanup_recovery_spec.rb +389 -0
- data/spec/lib/pwn/sessions_spec.rb +112 -0
- data/spec/lib/pwn/setup_spec.rb +3 -1
- data/spec/support/gate_failure_formatter.rb +14 -0
- data/third_party/pwn_rdoc.jsonl +15 -1
- metadata +14 -7
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# PWN managed default: session_cleanup
|
|
3
|
+
# PWN managed default: session_cleanup v10
|
|
4
4
|
# No provider calls or broad store GC. Unsummarized candidates are backed up first.
|
|
5
5
|
require 'pwn'
|
|
6
6
|
require 'json'
|
|
@@ -12,7 +12,1034 @@ require 'securerandom'
|
|
|
12
12
|
module PWN
|
|
13
13
|
# Provenance-based session cleanup, shared by seeded cron payloads.
|
|
14
14
|
module SessionCleanup
|
|
15
|
-
VERSION =
|
|
15
|
+
VERSION = 10
|
|
16
|
+
|
|
17
|
+
# Logical categories deliberately preserve existing literal evidence paths.
|
|
18
|
+
STORAGE = {
|
|
19
|
+
'sessions' => %w[sessions],
|
|
20
|
+
'learning' => %w[learning.jsonl learning lessons.json memory.json memory.archive.json memory.idx mistakes.json policy policy.json policy_traj.jsonl curriculum curriculum_kpi.jsonl extrospection.json finetune preferences.jsonl reward_sentinel.json skill_review.jsonl operator-learning-review.jsonl],
|
|
21
|
+
'artifacts' => %w[artifacts exports findings.jsonl engagements corpora binws redb ghidra sandbox_snapshots],
|
|
22
|
+
'state' => %w[swarm cron agents.yml open_goal.json jobs missions runs metrics metrics.json capabilities.json],
|
|
23
|
+
'logs' => %w[logs], 'cache' => %w[cache], 'tmp' => %w[tmp],
|
|
24
|
+
'maintenance' => %w[maintenance session_cleanup_archive],
|
|
25
|
+
'backups' => %w[backup]
|
|
26
|
+
}.freeze
|
|
27
|
+
POLICY = { 'version' => 1, 'batch_size' => 100, 'scan_limit' => 100_000, 'work_items' => 100_000, 'work_seconds' => 30, 'work_bytes' => 67_108_864,
|
|
28
|
+
'retention_days' => { 'sessions' => 0, 'learning' => nil, 'artifacts' => nil, 'state' => nil,
|
|
29
|
+
'logs' => 30, 'cache' => 7, 'tmp' => 1, 'maintenance' => 90 },
|
|
30
|
+
'budget_bytes' => { 'logs' => 268_435_456, 'cache' => 536_870_912, 'tmp' => 268_435_456, 'maintenance' => 1_073_741_824 } }.freeze
|
|
31
|
+
|
|
32
|
+
# Versioned additive setup: no old path moves or permission rewrites.
|
|
33
|
+
def self.install(root: File.join(Dir.home, '.pwn'))
|
|
34
|
+
directory = private_directory(File.join(root, 'maintenance'))
|
|
35
|
+
File.open(File.join(directory, '.setup.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
36
|
+
lock.flock(File::LOCK_EX)
|
|
37
|
+
path = File.join(directory, 'policy.json')
|
|
38
|
+
raise IOError, 'Symlinked maintenance policy' if File.symlink?(path)
|
|
39
|
+
|
|
40
|
+
publish(path, JSON.pretty_generate(POLICY)) unless File.exist?(path)
|
|
41
|
+
%w[cache tmp].each { |name| private_directory(File.join(root, name)) unless File.exist?(File.join(root, name)) }
|
|
42
|
+
layout = File.join(directory, 'layout.json')
|
|
43
|
+
raise IOError, 'Unsafe maintenance inventory' if File.symlink?(layout)
|
|
44
|
+
|
|
45
|
+
previous = File.file?(layout) ? JSON.parse(File.binread(layout)) : nil
|
|
46
|
+
raise IOError, 'Unknown maintenance inventory version' if previous && ![1, 2].include?(previous['version'])
|
|
47
|
+
|
|
48
|
+
publish(layout, JSON.generate(version: 2, categories: STORAGE)) unless previous && previous['version'] == 2
|
|
49
|
+
end
|
|
50
|
+
{ version: 2, root: directory }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Startup performs only fixed-path checks and one coalesced request write.
|
|
54
|
+
# The existing cron worker owns all scanning, migration and deletion.
|
|
55
|
+
def self.request_background(root: File.join(Dir.home, '.pwn'))
|
|
56
|
+
directory = File.join(root, 'maintenance')
|
|
57
|
+
return { state: 'not_installed' } unless File.directory?(directory)
|
|
58
|
+
raise IOError, 'Unsafe maintenance directory' unless File.realpath(directory) == directory
|
|
59
|
+
|
|
60
|
+
File.open(File.join(directory, '.schedule.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
61
|
+
return { state: 'coalesced' } unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
62
|
+
|
|
63
|
+
path = File.join(directory, 'background.json')
|
|
64
|
+
raise IOError, 'Unsafe background request' if File.symlink?(path)
|
|
65
|
+
|
|
66
|
+
if File.file?(path)
|
|
67
|
+
prior = JSON.parse(File.binread(path))
|
|
68
|
+
return { state: 'coalesced' } if prior['state'] == 'pending' || prior.fetch('requested_at', 0) > Time.now.to_i - 86_400
|
|
69
|
+
end
|
|
70
|
+
publish(path, JSON.generate(version: 1, state: 'pending', requested_at: Time.now.to_i))
|
|
71
|
+
{ state: 'queued', path: path }
|
|
72
|
+
end
|
|
73
|
+
rescue StandardError => e
|
|
74
|
+
{ state: 'unhealthy', error: e.class.name }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def self.background_request(root: File.join(Dir.home, '.pwn'))
|
|
78
|
+
directory = File.join(root, 'maintenance')
|
|
79
|
+
path = File.join(directory, 'background.json')
|
|
80
|
+
return unless File.file?(path)
|
|
81
|
+
raise IOError, 'Unsafe background request' unless File.realpath(path) == path
|
|
82
|
+
|
|
83
|
+
File.open(File.join(directory, '.schedule.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
84
|
+
next unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
85
|
+
|
|
86
|
+
row = JSON.parse(File.binread(path))
|
|
87
|
+
next unless row['version'] == 1 && row['state'] == 'pending'
|
|
88
|
+
|
|
89
|
+
result = yield
|
|
90
|
+
next unless result
|
|
91
|
+
|
|
92
|
+
row['state'] = result[:status] == 'success' ? 'completed' : 'interrupted'
|
|
93
|
+
row['completed_at'] = Time.now.to_i
|
|
94
|
+
publish(path, JSON.generate(row))
|
|
95
|
+
result
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def self.policy
|
|
100
|
+
path = File.join(Dir.home, '.pwn', 'maintenance', 'policy.json')
|
|
101
|
+
return Marshal.load(Marshal.dump(POLICY)) unless File.exist?(path) || File.symlink?(path)
|
|
102
|
+
|
|
103
|
+
raise IOError, 'Unsafe maintenance policy' if File.symlink?(path)
|
|
104
|
+
|
|
105
|
+
value = JSON.parse(File.binread(path))
|
|
106
|
+
raise ArgumentError, 'Unknown maintenance policy' unless value.is_a?(Hash) && value['version'] == 1 && (value.keys - POLICY.keys).empty?
|
|
107
|
+
|
|
108
|
+
result = Marshal.load(Marshal.dump(POLICY)).merge(value)
|
|
109
|
+
%w[retention_days budget_bytes].each do |key|
|
|
110
|
+
raise ArgumentError, "Invalid #{key}" unless result[key].is_a?(Hash) && (result[key].keys - STORAGE.keys).empty? && result[key].values.all? { |n| n.nil? || (n.is_a?(Integer) && n >= 0) }
|
|
111
|
+
|
|
112
|
+
result[key] = POLICY[key].merge(result[key])
|
|
113
|
+
end
|
|
114
|
+
raise ArgumentError, 'Invalid read budget' unless result['work_bytes'].is_a?(Integer) && result['work_bytes'].positive?
|
|
115
|
+
raise ArgumentError, 'Invalid maintenance bounds' unless %w[batch_size scan_limit work_items work_seconds].all? { |key| result[key].is_a?(Integer) && result[key].between?(1, 100_000) }
|
|
116
|
+
|
|
117
|
+
result
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# No mkdir, locks, provider access, metadata rewrites or repair in status.
|
|
121
|
+
def self.status
|
|
122
|
+
config = policy
|
|
123
|
+
root = File.join(Dir.home, '.pwn')
|
|
124
|
+
errors = []
|
|
125
|
+
remaining = config['scan_limit']
|
|
126
|
+
categories = STORAGE.to_h do |category, names|
|
|
127
|
+
paths = names.map { |name| name == 'sessions' ? PWN::Sessions::SESSIONS_DIR : File.join(root, name) }
|
|
128
|
+
row = { paths: paths, files: 0, bytes: 0, unique_bytes: 0, symlinks: 0, retention_days: config['retention_days'][category], budget_bytes: config['budget_bytes'][category] }
|
|
129
|
+
inodes = {}
|
|
130
|
+
pending = paths.dup
|
|
131
|
+
until pending.empty? || remaining.zero? || stopping?
|
|
132
|
+
begin
|
|
133
|
+
entry = pending.pop
|
|
134
|
+
if entry.is_a?(Array)
|
|
135
|
+
begin
|
|
136
|
+
path = File.join(entry[0], entry[1].next)
|
|
137
|
+
pending << entry
|
|
138
|
+
rescue StopIteration
|
|
139
|
+
next
|
|
140
|
+
end
|
|
141
|
+
else
|
|
142
|
+
path = entry
|
|
143
|
+
end
|
|
144
|
+
next unless File.exist?(path) || File.symlink?(path)
|
|
145
|
+
|
|
146
|
+
remaining -= 1
|
|
147
|
+
stat = File.lstat(path)
|
|
148
|
+
if stat.symlink?
|
|
149
|
+
row[:symlinks] += 1
|
|
150
|
+
elsif stat.directory?
|
|
151
|
+
pending << [path, Dir.each_child(path)]
|
|
152
|
+
elsif stat.file?
|
|
153
|
+
row[:files] += 1
|
|
154
|
+
row[:bytes] += stat.size
|
|
155
|
+
key = [stat.dev, stat.ino]
|
|
156
|
+
row[:unique_bytes] += stat.size unless inodes[key]
|
|
157
|
+
inodes[key] = true
|
|
158
|
+
end
|
|
159
|
+
rescue SystemCallError => e
|
|
160
|
+
errors << { path: path, error: e.class.name }
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
row[:incomplete] = !pending.empty?
|
|
164
|
+
row[:over_budget] = row[:budget_bytes] && row[:bytes] > row[:budget_bytes]
|
|
165
|
+
row[:policy_mode] = %w[learning artifacts state backups].include?(category) ? 'protected_evidence_no_automatic_deletion' : 'verified_disposable_only'
|
|
166
|
+
[category, row]
|
|
167
|
+
end
|
|
168
|
+
transactions = Dir.glob(File.join(root, 'maintenance', 'runs', '*.json')).last(config['batch_size']).map do |path|
|
|
169
|
+
raise IOError, 'Unsafe transaction ledger' unless File.realpath(path) == path
|
|
170
|
+
|
|
171
|
+
row = JSON.parse(admitted_bytes(path))
|
|
172
|
+
{ ledger: path, state: row['state'], recovery: %w[prepared interrupted].include?(row['state']) ? 'inspect_child_ledgers_before_retry' : nil }
|
|
173
|
+
rescue StandardError => e
|
|
174
|
+
{ ledger: path, state: 'unknown', error: e.class.name }
|
|
175
|
+
end
|
|
176
|
+
{ version: VERSION, operation: 'status', categories: categories, errors: errors, transactions: transactions,
|
|
177
|
+
policy: config, reasons: ['Unknown files and immutable evidence are retained; budgets do not override evidence or leases.',
|
|
178
|
+
'Historical backup expiry requires explicit retention policy and verified current recovery coverage; pinned evidence remains.'] }
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Explicit legacy expiry, not broad GC. No payload quarantine or new per-file
|
|
182
|
+
# leases: old producers do not honor those. Permanent maintenance locking,
|
|
183
|
+
# closed-stream markers, process descriptors and exact byte identities fence
|
|
184
|
+
# adoption. References are conservatively matched even in retained transcripts.
|
|
185
|
+
def self.expire_legacy(dry_run: true, manifest: nil)
|
|
186
|
+
require 'zlib'
|
|
187
|
+
root = File.join(Dir.home, '.pwn')
|
|
188
|
+
raise IOError, 'Unsafe root' unless File.realpath(root) == root
|
|
189
|
+
raise ArgumentError, 'New external metadata manifest required' if !dry_run && (manifest.nil? || File.expand_path(manifest).start_with?("#{root}/") || File.exist?(manifest))
|
|
190
|
+
|
|
191
|
+
result = { removed: [], retained: [], candidates: [] }
|
|
192
|
+
File.open(File.join(root, 'sessions', '.maintenance.lock'), File::RDWR | File::NOFOLLOW) do |lock|
|
|
193
|
+
raise IOError, 'Maintenance active' unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
194
|
+
|
|
195
|
+
candidates = legacy_candidates(root) + legacy_recovery_candidates(root) + legacy_cache_candidates(root)
|
|
196
|
+
pins = legacy_references(root, candidates)
|
|
197
|
+
opened = legacy_open_files
|
|
198
|
+
candidates.each do |row|
|
|
199
|
+
reason = pins[row[:path]] ? 'evidence_reference' : nil
|
|
200
|
+
reason ||= 'open_descriptor' if opened.include?([row[:dev], row[:ino]])
|
|
201
|
+
(reason ? result[:retained] : result[:candidates]) << row.merge(reason: reason, references: pins[row[:path]])
|
|
202
|
+
end
|
|
203
|
+
return result if dry_run
|
|
204
|
+
|
|
205
|
+
# Publish metadata before irreversible expiry. An interrupted manifest is
|
|
206
|
+
# an intent ledger, never a claim that every planned removal completed.
|
|
207
|
+
result[:state] = 'prepared'
|
|
208
|
+
legacy_manifest(manifest, result)
|
|
209
|
+
raise IOError, 'Evidence references changed' unless pins == legacy_references(root, candidates)
|
|
210
|
+
|
|
211
|
+
result[:candidates].each do |row|
|
|
212
|
+
path = row[:path]
|
|
213
|
+
File.open(path, File::RDONLY | File::NOFOLLOW) do |file|
|
|
214
|
+
raise IOError, 'Candidate locked' unless file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
215
|
+
|
|
216
|
+
lease = File.open("#{path}.lease", File::RDWR | File::NOFOLLOW) if File.exist?("#{path}.lease") || File.symlink?("#{path}.lease") # rubocop:disable Style/FileOpen -- closed in ensure while optional producer lease is held
|
|
217
|
+
raise IOError, 'Producer lease active' if lease && !lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
218
|
+
raise IOError, 'Candidate changed' unless legacy_identity(path) == row.except(:category, :reason, :references, :proofs)
|
|
219
|
+
|
|
220
|
+
Array(row[:proofs]).each do |proof|
|
|
221
|
+
raise IOError, 'Recovery proof changed' unless legacy_identity(proof[:path]) == proof
|
|
222
|
+
end
|
|
223
|
+
raise IOError, 'Candidate became active' if legacy_open_files(except_fd: file.fileno).include?([row[:dev], row[:ino]])
|
|
224
|
+
|
|
225
|
+
File.unlink(path)
|
|
226
|
+
result[:removed] << row
|
|
227
|
+
ensure
|
|
228
|
+
lease&.close
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
result[:state] = 'completed'
|
|
232
|
+
legacy_manifest(manifest, result)
|
|
233
|
+
end
|
|
234
|
+
result
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def self.legacy_manifest(path, result)
|
|
238
|
+
File.open(path, File::WRONLY | File::CREAT | File::TRUNC | File::NOFOLLOW, 0o600) do |file|
|
|
239
|
+
gzip = Zlib::GzipWriter.new(file)
|
|
240
|
+
gzip.write(JSON.generate(result))
|
|
241
|
+
gzip.finish
|
|
242
|
+
file.flush
|
|
243
|
+
file.fsync
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def self.legacy_identity(path)
|
|
248
|
+
stat = File.lstat(path)
|
|
249
|
+
raise IOError, 'Unsafe legacy file' unless stat.file? && stat.uid == File.stat(File.join(Dir.home, '.pwn')).uid && stat.nlink == 1 && File.realpath(path) == path
|
|
250
|
+
|
|
251
|
+
{ path: path, dev: stat.dev, ino: stat.ino, size: stat.size, mtime: stat.mtime.to_r.to_s,
|
|
252
|
+
ctime: stat.ctime.to_r.to_s, sha256: Digest::SHA256.file(path).hexdigest }
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def self.legacy_candidates(root)
|
|
256
|
+
days = policy['retention_days']['logs']
|
|
257
|
+
return [] unless days.is_a?(Numeric) && days.positive?
|
|
258
|
+
|
|
259
|
+
Dir.glob(File.join(root, 'logs', '*.log')).filter_map do |path|
|
|
260
|
+
next unless File.basename(path).match?(/\Apwn-ai-DEBUG-[a-zA-Z0-9_-]+-R[1-9]\d*\.log\z/)
|
|
261
|
+
|
|
262
|
+
row = legacy_identity(path)
|
|
263
|
+
next unless File.lstat(path).mtime < Time.now - (days * 86_400)
|
|
264
|
+
|
|
265
|
+
bytes = File.binread(path)
|
|
266
|
+
next unless bytes.include?('debug session stop') || bytes.include?('footer iter=')
|
|
267
|
+
|
|
268
|
+
row.merge(category: 'logs')
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Historical learning/reward recovery is explicitly bounded to at least
|
|
273
|
+
# three parseable copies per exact producer family, plus all reference pins.
|
|
274
|
+
# A thirty-day floor excludes files an older daily producer might still use.
|
|
275
|
+
def self.legacy_recovery_candidates(root)
|
|
276
|
+
families = { 'learning.jsonl' => /\Alearning\.jsonl\.bak-lean-\d{8}\z/,
|
|
277
|
+
'preferences.jsonl' => /\Apreferences\.jsonl\.bak-p15-\d{14}\z/ }
|
|
278
|
+
families.flat_map do |source, pattern|
|
|
279
|
+
current = File.join(root, source)
|
|
280
|
+
next [] unless File.file?(current)
|
|
281
|
+
|
|
282
|
+
legacy_identity(current)
|
|
283
|
+
File.foreach(current) { |line| JSON.parse(line) unless line.strip.empty? }
|
|
284
|
+
copies = Dir.children(root).grep(pattern).sort.map { |name| File.join(root, name) }
|
|
285
|
+
valid = copies.select do |path|
|
|
286
|
+
legacy_identity(path)
|
|
287
|
+
File.foreach(path) { |line| JSON.parse(line) unless line.strip.empty? }
|
|
288
|
+
File.size(path).positive?
|
|
289
|
+
rescue JSON::ParserError
|
|
290
|
+
false
|
|
291
|
+
end
|
|
292
|
+
(valid - valid.last(3)).filter_map do |path|
|
|
293
|
+
next unless File.mtime(path) < Time.now - (30 * 86_400)
|
|
294
|
+
|
|
295
|
+
legacy_identity(path).merge(category: 'root_recovery', proofs: valid.last(3).map { |copy| legacy_identity(copy) })
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# BinaryParser persists exactly these cache bytes to ArtifactRegistry. Missing
|
|
301
|
+
# source binaries prevent recomputation; require a surviving byte-exact copy
|
|
302
|
+
# instead, never classify arbitrary decompiler stdout as reproducible.
|
|
303
|
+
def self.legacy_cache_candidates(root)
|
|
304
|
+
days = policy['retention_days']['cache']
|
|
305
|
+
return [] unless days.is_a?(Numeric) && days.positive?
|
|
306
|
+
|
|
307
|
+
copies = Dir.glob(File.join(root, 'artifacts', '**', '*')).select { |path| File.file?(path) && !File.symlink?(path) }
|
|
308
|
+
Dir.glob(File.join(root, 'cache', 'triage', '*.json')).filter_map do |path|
|
|
309
|
+
next unless File.basename(path).match?(/\A[a-f0-9]{64}\.json\z/) && File.mtime(path) < Time.now - (days * 86_400)
|
|
310
|
+
|
|
311
|
+
row = legacy_identity(path)
|
|
312
|
+
body = JSON.parse(File.binread(path))
|
|
313
|
+
next unless body['sha256'] == File.basename(path, '.json') && body['path'].is_a?(String)
|
|
314
|
+
|
|
315
|
+
copy = copies.find { |candidate| File.size(candidate) == row[:size] && Digest::SHA256.file(candidate).hexdigest == row[:sha256] }
|
|
316
|
+
next unless copy
|
|
317
|
+
|
|
318
|
+
row.merge(category: 'cache', proofs: [legacy_identity(copy)])
|
|
319
|
+
end
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
def self.legacy_references(root, candidates)
|
|
323
|
+
names = candidates.to_h { |row| [File.basename(row[:path]), row[:path]] }
|
|
324
|
+
return {} if names.empty?
|
|
325
|
+
|
|
326
|
+
pattern = Regexp.union(names.keys)
|
|
327
|
+
references = {}
|
|
328
|
+
# Recovery generations are historical copies, not live evidence pins.
|
|
329
|
+
# Candidate payloads cannot pin themselves. Everything else is scanned
|
|
330
|
+
# without interpreting or displaying credential/transcript contents.
|
|
331
|
+
Dir.glob(File.join(root, '**', '*'), File::FNM_DOTMATCH).sort.each do |path|
|
|
332
|
+
relative = path.delete_prefix("#{root}/")
|
|
333
|
+
next if %w[backup logs cache].include?(relative.split('/').first)
|
|
334
|
+
next if names.value?(path) || !File.file?(path)
|
|
335
|
+
raise IOError, 'Symlink in reference scope' if File.symlink?(path)
|
|
336
|
+
|
|
337
|
+
File.binread(path).scan(pattern).uniq.each do |name|
|
|
338
|
+
(references[names.fetch(name)] ||= []) << path
|
|
339
|
+
end
|
|
340
|
+
end
|
|
341
|
+
references
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def self.legacy_open_files(except_fd: nil)
|
|
345
|
+
raise IOError, 'Process inventory unavailable' unless File.directory?('/proc/self/fd')
|
|
346
|
+
|
|
347
|
+
opened = []
|
|
348
|
+
Dir.glob('/proc/[0-9]*').each do |process|
|
|
349
|
+
next unless Process.uid.zero? || File.stat(process).uid == Process.uid
|
|
350
|
+
|
|
351
|
+
Dir.children("#{process}/fd").each do |fd|
|
|
352
|
+
next if File.basename(process).to_i == Process.pid && fd.to_i == except_fd
|
|
353
|
+
|
|
354
|
+
stat = File.stat("#{process}/fd/#{fd}")
|
|
355
|
+
opened << [stat.dev, stat.ino] if stat.file?
|
|
356
|
+
rescue Errno::ENOENT, Errno::ESRCH
|
|
357
|
+
next
|
|
358
|
+
end
|
|
359
|
+
rescue Errno::ENOENT, Errno::ESRCH
|
|
360
|
+
next
|
|
361
|
+
end
|
|
362
|
+
opened.uniq
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# A producer registers only reproducible/disposable bytes, never evidence.
|
|
366
|
+
# Writers hold the permanent .lease exclusively while changing a file;
|
|
367
|
+
# consumers hold it shared for the whole read/use lifetime.
|
|
368
|
+
def self.register_disposable(path:, category:, expires_at: nil)
|
|
369
|
+
raise ArgumentError, 'Not a disposable category' unless %w[cache tmp logs].include?(category)
|
|
370
|
+
|
|
371
|
+
root = File.join(Dir.home, '.pwn', category)
|
|
372
|
+
path = File.expand_path(path)
|
|
373
|
+
raise IOError, 'Disposable path escapes its category' unless path.start_with?("#{root}/") && File.realpath(path) == path
|
|
374
|
+
raise IOError, 'Disposable must be an owned regular file' unless File.file?(path) && File.stat(path).uid == Process.uid
|
|
375
|
+
|
|
376
|
+
days = policy['retention_days'][category]
|
|
377
|
+
raise ArgumentError, 'Retention disabled for category' if days.nil?
|
|
378
|
+
|
|
379
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
380
|
+
raise IOError, 'Disposable is active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
381
|
+
|
|
382
|
+
record_disposable(path: path, category: category, expires_at: expires_at || (Time.now.to_i + (days * 86_400)))
|
|
383
|
+
end
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# Caller holds the permanent exclusive producer lease through publication.
|
|
387
|
+
def self.record_disposable(path:, category:, expires_at:)
|
|
388
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'disposable'))
|
|
389
|
+
record = { version: 1, path: path, category: category, expires_at: expires_at,
|
|
390
|
+
sha256: Digest::SHA256.file(path).hexdigest, bytes: File.size(path), inode: File.stat(path).ino }
|
|
391
|
+
publish(File.join(directory, "#{Digest::SHA256.hexdigest(path)}.json"), JSON.generate(record))
|
|
392
|
+
record
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
# Cache consumers use the same permanent inode as publication and eviction.
|
|
396
|
+
def self.cache(path:, bytes: nil)
|
|
397
|
+
root = File.join(Dir.home, '.pwn', 'cache')
|
|
398
|
+
raise IOError, 'Unsafe cache path' unless File.expand_path(path) == path && path.start_with?("#{root}/")
|
|
399
|
+
|
|
400
|
+
private_directory(File.dirname(path)) unless File.directory?(File.dirname(path))
|
|
401
|
+
raise IOError, 'Unsafe cache directory' unless File.realpath(File.dirname(path)) == File.dirname(path)
|
|
402
|
+
|
|
403
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
404
|
+
lease.flock(bytes ? File::LOCK_EX : File::LOCK_SH)
|
|
405
|
+
if bytes
|
|
406
|
+
publish(path, bytes)
|
|
407
|
+
days = policy['retention_days']['cache']
|
|
408
|
+
if days
|
|
409
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'disposable'))
|
|
410
|
+
row = { version: 1, path: path, category: 'cache', expires_at: Time.now.to_i + (days * 86_400),
|
|
411
|
+
sha256: Digest::SHA256.file(path).hexdigest, bytes: File.size(path), inode: File.stat(path).ino }
|
|
412
|
+
publish(File.join(directory, "#{Digest::SHA256.hexdigest(path)}.json"), JSON.generate(row))
|
|
413
|
+
end
|
|
414
|
+
bytes
|
|
415
|
+
elsif File.file?(path) && !File.symlink?(path)
|
|
416
|
+
File.binread(path)
|
|
417
|
+
end
|
|
418
|
+
end
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
# Known disposable input producer. The permanent lease covers publication,
|
|
422
|
+
# consumption and unlink; a killed consumer leaves a registered exact file.
|
|
423
|
+
def self.temporary(bytes:)
|
|
424
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'tmp'))
|
|
425
|
+
path = File.join(directory, "input-#{SecureRandom.hex(12)}.txt")
|
|
426
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
427
|
+
lease.flock(File::LOCK_EX)
|
|
428
|
+
begin
|
|
429
|
+
publish(path, bytes)
|
|
430
|
+
days = policy['retention_days']['tmp']
|
|
431
|
+
record_disposable(path: path, category: 'tmp', expires_at: Time.now.to_i + (days * 86_400)) if days
|
|
432
|
+
yield path
|
|
433
|
+
ensure
|
|
434
|
+
File.unlink(path) if File.file?(path) && !File.symlink?(path)
|
|
435
|
+
end
|
|
436
|
+
end
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
# Rotate ordinary diagnostics under the same lease as append and eviction.
|
|
440
|
+
# Request debug streams rotate at request boundaries and hold a lifetime lease.
|
|
441
|
+
def self.append_log(path:, bytes:, rotate_bytes: 1_024_000)
|
|
442
|
+
root = File.join(Dir.home, '.pwn', 'logs')
|
|
443
|
+
raise IOError, 'Unsafe log path' unless File.expand_path(path) == path && path.start_with?("#{root}/")
|
|
444
|
+
|
|
445
|
+
private_directory(File.dirname(path)) unless File.directory?(File.dirname(path))
|
|
446
|
+
raise IOError, 'Unsafe log directory' unless File.realpath(File.dirname(path)) == File.dirname(path)
|
|
447
|
+
|
|
448
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
449
|
+
lease.flock(File::LOCK_EX)
|
|
450
|
+
raise IOError, 'Symlinked log' if File.symlink?(path)
|
|
451
|
+
|
|
452
|
+
if File.file?(path) && File.size(path) >= rotate_bytes
|
|
453
|
+
rotated = "#{path}.#{SecureRandom.hex(12)}"
|
|
454
|
+
File.rename(path, rotated)
|
|
455
|
+
register_disposable(path: rotated, category: 'logs') unless policy['retention_days']['logs'].nil?
|
|
456
|
+
end
|
|
457
|
+
File.open(path, File::WRONLY | File::CREAT | File::APPEND | File::NOFOLLOW, 0o600) do |file|
|
|
458
|
+
file.write(bytes)
|
|
459
|
+
file.flush
|
|
460
|
+
file.fsync
|
|
461
|
+
end
|
|
462
|
+
days = policy['retention_days']['logs']
|
|
463
|
+
record_disposable(path: path, category: 'logs', expires_at: Time.now.to_i + (days * 86_400)) if days
|
|
464
|
+
end
|
|
465
|
+
path
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
def self.private_directory(path)
|
|
469
|
+
ancestor = File.expand_path(path)
|
|
470
|
+
loop do
|
|
471
|
+
raise IOError, 'Symlinked maintenance directory' if File.symlink?(ancestor)
|
|
472
|
+
break if ancestor == File.dirname(ancestor)
|
|
473
|
+
|
|
474
|
+
ancestor = File.dirname(ancestor)
|
|
475
|
+
end
|
|
476
|
+
FileUtils.mkdir_p(path, mode: 0o700)
|
|
477
|
+
stat = File.stat(path)
|
|
478
|
+
raise IOError, 'Unsafe maintenance directory permissions' unless stat.uid == Process.uid && stat.mode.nobits?(0o077)
|
|
479
|
+
|
|
480
|
+
path
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
# Exact-size reads cannot grow without bound if an uncooperative writer appends.
|
|
484
|
+
def self.exact_bytes(path, size)
|
|
485
|
+
bytes = File.binread(path, size + 1)
|
|
486
|
+
raise IOError, 'File size changed' unless bytes.bytesize == size
|
|
487
|
+
|
|
488
|
+
bytes
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
def self.admitted_bytes(path)
|
|
492
|
+
size = File.size(path)
|
|
493
|
+
raise IOError, 'Maintenance read budget exhausted' unless admit?(size + 1)
|
|
494
|
+
|
|
495
|
+
exact_bytes(path, size)
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
def self.disposable_reason(row, pressure: false, verify: true)
|
|
499
|
+
return 'unknown_schema' unless row.is_a?(Hash) && row['version'] == 1 && %w[cache tmp logs].include?(row['category']) && row['expires_at'].is_a?(Integer)
|
|
500
|
+
|
|
501
|
+
path = row['path']
|
|
502
|
+
root = File.join(Dir.home, '.pwn', row['category'])
|
|
503
|
+
return 'unsafe_path' unless path.is_a?(String) && path.start_with?("#{root}/") && File.expand_path(path) == path
|
|
504
|
+
return 'missing' unless File.exist?(path)
|
|
505
|
+
return 'unsafe_path' unless File.realpath(path) == path && File.file?(path) && File.stat(path).uid == Process.uid
|
|
506
|
+
return 'changed' unless File.stat(path).ino == row['inode'] && File.size(path) == row['bytes']
|
|
507
|
+
return 'changed' if verify && Digest::SHA256.hexdigest(exact_bytes(path, row['bytes'])) != row['sha256']
|
|
508
|
+
return 'retention_disabled' if policy['retention_days'][row['category']].nil?
|
|
509
|
+
return 'not_expired' if !pressure && row['expires_at'] > Time.now.to_i
|
|
510
|
+
return 'missing_lease' unless File.file?("#{path}.lease") && !File.symlink?("#{path}.lease")
|
|
511
|
+
|
|
512
|
+
nil
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
def self.clean_disposables(dry_run: true, stop: nil)
|
|
516
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
517
|
+
result = { operation: 'routine', state: 'planned', dry_run: dry_run, eligible: [], deleted: [], retained: [], records: [] }
|
|
518
|
+
root = File.join(Dir.home, '.pwn', 'maintenance')
|
|
519
|
+
usage = status[:categories]
|
|
520
|
+
# A lexical round-robin cursor is only scheduling state, never deletion
|
|
521
|
+
# authority. Manifests stay put, so concurrent re-registration cannot
|
|
522
|
+
# be removed or overwritten by cleanup completion.
|
|
523
|
+
cursor_path = File.join(root, 'disposable.cursor')
|
|
524
|
+
cursor = File.file?(cursor_path) && !File.symlink?(cursor_path) ? File.binread(cursor_path, 70) : ''
|
|
525
|
+
cursor = '' unless cursor.match?(/\A[a-f0-9]{64}\.json\z/)
|
|
526
|
+
files = Dir.glob(File.join(root, 'disposable', '*.json'))
|
|
527
|
+
after, before = files.partition { |file| File.basename(file) > cursor }
|
|
528
|
+
files = after + before
|
|
529
|
+
result[:incomplete] = true if files.length > policy['scan_limit']
|
|
530
|
+
files = files.first(policy['scan_limit'])
|
|
531
|
+
files.each do |manifest|
|
|
532
|
+
if !admit? || stop&.call || result[:deleted].length >= policy['batch_size']
|
|
533
|
+
result[:incomplete] = true
|
|
534
|
+
break
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
publish(cursor_path, File.basename(manifest)) unless dry_run
|
|
538
|
+
raise IOError, 'Unsafe disposable manifest' unless File.realpath(manifest) == manifest
|
|
539
|
+
|
|
540
|
+
manifest_size = File.size(manifest)
|
|
541
|
+
break unless admit?((manifest_size + 1) * 2)
|
|
542
|
+
|
|
543
|
+
manifest_bytes = exact_bytes(manifest, manifest_size)
|
|
544
|
+
row = JSON.parse(manifest_bytes)
|
|
545
|
+
category_usage = usage[row['category']] || {}
|
|
546
|
+
budget = category_usage[:budget_bytes]
|
|
547
|
+
pressure = budget && category_usage[:bytes] > budget
|
|
548
|
+
reason = disposable_reason(row, pressure: pressure, verify: false)
|
|
549
|
+
if reason
|
|
550
|
+
result[:retained] << { manifest: manifest, reason: reason }
|
|
551
|
+
next
|
|
552
|
+
end
|
|
553
|
+
path = row['path']
|
|
554
|
+
File.open("#{path}.lease", File::RDWR | File::NOFOLLOW) do |lease|
|
|
555
|
+
unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
556
|
+
result[:retained] << { path: path, reason: 'active' }
|
|
557
|
+
next
|
|
558
|
+
end
|
|
559
|
+
if exact_bytes(manifest, manifest_size) != manifest_bytes
|
|
560
|
+
result[:retained] << { path: path, reason: 'registration_changed' }
|
|
561
|
+
next
|
|
562
|
+
end
|
|
563
|
+
# Reserve all payload reads, backup write and verification before commit.
|
|
564
|
+
unless admit?((row['bytes'] + 1) * (dry_run ? 1 : 5))
|
|
565
|
+
result[:retained] << { path: path, reason: 'work_budget' }
|
|
566
|
+
next
|
|
567
|
+
end
|
|
568
|
+
if (reason = disposable_reason(row, pressure: pressure))
|
|
569
|
+
result[:retained] << { path: path, reason: reason }
|
|
570
|
+
next
|
|
571
|
+
end
|
|
572
|
+
result[:eligible] << path
|
|
573
|
+
category_usage[:bytes] -= row['bytes']
|
|
574
|
+
next if dry_run
|
|
575
|
+
|
|
576
|
+
unless result[:ledger]
|
|
577
|
+
directory = private_directory(File.join(root, 'quarantine', SecureRandom.hex(12)))
|
|
578
|
+
result[:ledger] = File.join(directory, 'ledger.json')
|
|
579
|
+
end
|
|
580
|
+
backup_path = File.join(File.dirname(result[:ledger]), "#{Digest::SHA256.hexdigest(path)}.bin")
|
|
581
|
+
publish(backup_path, exact_bytes(path, row['bytes']))
|
|
582
|
+
raise IOError, 'Disposable backup mismatch' unless Digest::SHA256.hexdigest(exact_bytes(backup_path, row['bytes'])) == row['sha256']
|
|
583
|
+
|
|
584
|
+
record = row.merge('backup' => backup_path, 'state' => 'prepared')
|
|
585
|
+
result[:records] << record
|
|
586
|
+
result[:state] = 'prepared'
|
|
587
|
+
publish(result[:ledger], JSON.pretty_generate(result))
|
|
588
|
+
raise IOError, 'Disposable changed after backup' if disposable_reason(row, pressure: pressure)
|
|
589
|
+
|
|
590
|
+
File.unlink(path)
|
|
591
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
592
|
+
result[:deleted] << path
|
|
593
|
+
record['state'] = 'committed'
|
|
594
|
+
publish(result[:ledger], JSON.pretty_generate(result))
|
|
595
|
+
# Keep permanent lease inode and manifest for audit/recovery.
|
|
596
|
+
end
|
|
597
|
+
rescue StandardError => e
|
|
598
|
+
result[:retained] << { manifest: manifest, reason: e.class.name }
|
|
599
|
+
end
|
|
600
|
+
unless dry_run
|
|
601
|
+
result[:state] = stopping? || stop&.call ? 'stopped' : 'committed'
|
|
602
|
+
result[:state] = 'interrupted' if result[:records].any? { |row| row['state'] == 'prepared' }
|
|
603
|
+
end
|
|
604
|
+
publish(result[:ledger], JSON.pretty_generate(result)) if result[:ledger]
|
|
605
|
+
result
|
|
606
|
+
end
|
|
607
|
+
end
|
|
608
|
+
|
|
609
|
+
# Only disposable quarantine expires. Learning/session/retirement backups
|
|
610
|
+
# are evidence, and are deliberately outside this pruning protocol.
|
|
611
|
+
def self.prune_quarantine(dry_run: true)
|
|
612
|
+
config = policy
|
|
613
|
+
result = { eligible: [], deleted: [], retained: [] }
|
|
614
|
+
days = config['retention_days']['maintenance']
|
|
615
|
+
return result if days.nil?
|
|
616
|
+
|
|
617
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
618
|
+
remaining_bytes = status[:categories]['maintenance'][:unique_bytes]
|
|
619
|
+
budget = config['budget_bytes']['maintenance']
|
|
620
|
+
Dir.glob(File.join(Dir.home, '.pwn', 'maintenance', 'quarantine', '*', 'ledger.json')).each do |ledger|
|
|
621
|
+
break if stopping? || result[:deleted].length >= config['batch_size']
|
|
622
|
+
|
|
623
|
+
raise IOError, 'Unsafe quarantine ledger' unless File.realpath(ledger) == ledger
|
|
624
|
+
|
|
625
|
+
data = JSON.parse(admitted_bytes(ledger))
|
|
626
|
+
next if data['state'] == 'expired'
|
|
627
|
+
next if data['state'] != 'expiry_prepared' && File.mtime(ledger) > Time.now - (days * 86_400) && !(budget && remaining_bytes > budget)
|
|
628
|
+
raise IOError, 'Uncommitted quarantine retained' unless data['operation'] == 'routine' && %w[committed expiry_prepared].include?(data['state']) && data['records'].is_a?(Array) && data['records'].all? { |row| row['state'] == 'committed' }
|
|
629
|
+
|
|
630
|
+
backups = data['records'].filter_map do |row|
|
|
631
|
+
path = row.fetch('backup')
|
|
632
|
+
raise IOError, 'Unsafe quarantine backup' unless File.dirname(path) == File.dirname(ledger) && File.expand_path(path) == path
|
|
633
|
+
next if data['state'] == 'expiry_prepared' && !File.exist?(path) && !File.symlink?(path)
|
|
634
|
+
|
|
635
|
+
raise IOError, 'Unknown quarantine bytes' unless File.dirname(path) == File.dirname(ledger) && File.realpath(path) == path && Digest::SHA256.hexdigest(admitted_bytes(path)) == row['sha256']
|
|
636
|
+
|
|
637
|
+
path
|
|
638
|
+
end
|
|
639
|
+
raise IOError, 'Unknown quarantine contents' unless (Dir.children(File.dirname(ledger)) - ['ledger.json', *backups.map { |path| File.basename(path) }]).empty?
|
|
640
|
+
|
|
641
|
+
result[:eligible].concat(backups)
|
|
642
|
+
remaining_bytes -= backups.sum { |path| File.size(path) }
|
|
643
|
+
next if dry_run
|
|
644
|
+
|
|
645
|
+
data['state'] = 'expiry_prepared'
|
|
646
|
+
publish(ledger, JSON.pretty_generate(data))
|
|
647
|
+
backups.each do |path|
|
|
648
|
+
break if !admit? || result[:deleted].length >= config['batch_size']
|
|
649
|
+
|
|
650
|
+
File.unlink(path)
|
|
651
|
+
result[:deleted] << path
|
|
652
|
+
end
|
|
653
|
+
File.open(File.dirname(ledger), File::RDONLY, &:fsync)
|
|
654
|
+
data['state'] = backups.any? { |path| File.exist?(path) } ? 'expiry_prepared' : 'expired'
|
|
655
|
+
publish(ledger, JSON.pretty_generate(data))
|
|
656
|
+
rescue StandardError => e
|
|
657
|
+
result[:retained] << { ledger: ledger, reason: e.class.name }
|
|
658
|
+
end
|
|
659
|
+
end
|
|
660
|
+
result
|
|
661
|
+
end
|
|
662
|
+
|
|
663
|
+
# Prune duplicate physical bytes, not recovery names: hardlink replacement
|
|
664
|
+
# preserves every store-specific rollback API and never loses the last copy.
|
|
665
|
+
# Only committed writer-defined backup rows enter this inventory.
|
|
666
|
+
def self.legacy_backups
|
|
667
|
+
root = File.join(Dir.home, '.pwn')
|
|
668
|
+
patterns = ['session_cleanup_archive/*/ledger.json', 'learning/migrations/*.json', 'maintenance/retirements/*/ledger.json']
|
|
669
|
+
rows = []
|
|
670
|
+
patterns.each do |pattern|
|
|
671
|
+
Dir.glob(File.join(root, pattern)).first(policy['scan_limit']).each do |ledger|
|
|
672
|
+
break unless admit?
|
|
673
|
+
next unless File.realpath(ledger) == ledger
|
|
674
|
+
|
|
675
|
+
data = JSON.parse(admitted_bytes(ledger))
|
|
676
|
+
candidates = []
|
|
677
|
+
if ledger.start_with?("#{File.join(root, 'learning', 'migrations')}/")
|
|
678
|
+
hash = File.basename(ledger, '.json')
|
|
679
|
+
next unless data['version'] == 1 && data['state'] == 'committed' && data['before_sha256'] == hash && hash.match?(/\A[a-f0-9]{64}\z/)
|
|
680
|
+
|
|
681
|
+
candidates << [File.join(File.dirname(ledger), "#{hash}.backup.jsonl"), hash]
|
|
682
|
+
elsif ledger.start_with?("#{File.join(root, 'maintenance', 'retirements')}/")
|
|
683
|
+
next unless data['state'] == 'committed' && data['session_id'].to_s.match?(PWN::Sessions::SESSION_ID)
|
|
684
|
+
|
|
685
|
+
candidates << [data['backup'], data['sha256']]
|
|
686
|
+
elsif data['dry_run'] == false && data['records'].is_a?(Array) && data['deleted_ids'].is_a?(Array)
|
|
687
|
+
data['records'].each do |row|
|
|
688
|
+
next unless row['deleted'] == true && data['deleted_ids'].include?(row['id']) && row['id'].to_s.match?(PWN::Sessions::SESSION_ID)
|
|
689
|
+
|
|
690
|
+
row.fetch('files', {}).each do |name, info|
|
|
691
|
+
next unless info && ["#{row['id']}.jsonl", "#{row['id']}.meta.json"].include?(name)
|
|
692
|
+
|
|
693
|
+
candidates << [File.join(File.dirname(ledger), name), info['sha256']]
|
|
694
|
+
end
|
|
695
|
+
end
|
|
696
|
+
Array(data['dependency_removals']).each do |row|
|
|
697
|
+
candidates << [row['backup'], row['sha256']] if row['state'] == 'committed'
|
|
698
|
+
end
|
|
699
|
+
end
|
|
700
|
+
candidates.each do |path, hash|
|
|
701
|
+
next unless path.is_a?(String) && File.dirname(path) == File.dirname(ledger)
|
|
702
|
+
next unless verified_backup?(path, hash)
|
|
703
|
+
|
|
704
|
+
rows << { path: path, sha256: hash, ledger: ledger }
|
|
705
|
+
end
|
|
706
|
+
rescue StandardError
|
|
707
|
+
# Unknown, partial and unreadable ledgers remain untouched.
|
|
708
|
+
next
|
|
709
|
+
end
|
|
710
|
+
end
|
|
711
|
+
rows.uniq { |row| row[:path] }
|
|
712
|
+
end
|
|
713
|
+
|
|
714
|
+
def self.verified_backup?(path, hash, admitted: false, size: nil)
|
|
715
|
+
hash.is_a?(String) && hash.match?(/\A[a-f0-9]{64}\z/) && File.file?(path) &&
|
|
716
|
+
File.realpath(path) == path && File.stat(path).uid == Process.uid &&
|
|
717
|
+
Digest::SHA256.hexdigest(admitted ? exact_bytes(path, size || File.size(path)) : admitted_bytes(path)) == hash
|
|
718
|
+
end
|
|
719
|
+
|
|
720
|
+
def self.prune_legacy_backups(dry_run: true)
|
|
721
|
+
result = { operation: 'backup_compaction', state: 'planned', compacted: [], retained: [], records: [] }
|
|
722
|
+
days = policy['retention_days']['maintenance']
|
|
723
|
+
return result if days.nil?
|
|
724
|
+
|
|
725
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
726
|
+
# Learning and continuation producers also serialize with these locks.
|
|
727
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
728
|
+
groups = legacy_backups.group_by { |row| row[:sha256] }
|
|
729
|
+
usage = status[:categories]['maintenance'][:unique_bytes]
|
|
730
|
+
budget = policy['budget_bytes']['maintenance']
|
|
731
|
+
groups.each_value do |rows|
|
|
732
|
+
canonical = rows.first
|
|
733
|
+
rows.drop(1).each do |row|
|
|
734
|
+
break unless admit?
|
|
735
|
+
break if result[:compacted].length >= policy['batch_size']
|
|
736
|
+
next if File.identical?(canonical[:path], row[:path])
|
|
737
|
+
next if File.mtime(row[:ledger]) > Time.now - (days * 86_400) && !(budget && usage > budget)
|
|
738
|
+
|
|
739
|
+
size = File.size(row[:path])
|
|
740
|
+
break unless admit?((size + 1) * 5)
|
|
741
|
+
|
|
742
|
+
usage -= size
|
|
743
|
+
record = row.merge(canonical: canonical[:path], bytes: size, state: 'prepared')
|
|
744
|
+
result[:records] << record
|
|
745
|
+
next if dry_run
|
|
746
|
+
|
|
747
|
+
unless result[:ledger]
|
|
748
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'compactions'))
|
|
749
|
+
result[:ledger] = File.join(directory, "#{SecureRandom.hex(12)}.json")
|
|
750
|
+
end
|
|
751
|
+
result[:state] = 'prepared'
|
|
752
|
+
publish(result[:ledger], JSON.generate(result))
|
|
753
|
+
compact_backup(record)
|
|
754
|
+
record[:state] = 'committed'
|
|
755
|
+
result[:compacted] << row[:path]
|
|
756
|
+
publish(result[:ledger], JSON.generate(result))
|
|
757
|
+
rescue StandardError => e
|
|
758
|
+
result[:retained] << { path: row[:path], reason: e.class.name }
|
|
759
|
+
end
|
|
760
|
+
end
|
|
761
|
+
result[:state] = stopping? || result[:records].any? { |row| row[:state] == 'prepared' } ? 'interrupted' : 'committed' unless dry_run
|
|
762
|
+
publish(result[:ledger], JSON.generate(result)) if result[:ledger]
|
|
763
|
+
end
|
|
764
|
+
end
|
|
765
|
+
result
|
|
766
|
+
end
|
|
767
|
+
|
|
768
|
+
def self.compact_backup(row)
|
|
769
|
+
path, canonical, hash = row.values_at(:path, :canonical, :sha256)
|
|
770
|
+
size = row[:bytes] || File.size(path)
|
|
771
|
+
raise IOError, 'Changed backup' unless verified_backup?(path, hash, admitted: true, size: size) && verified_backup?(canonical, hash, admitted: true, size: size)
|
|
772
|
+
return if File.identical?(path, canonical)
|
|
773
|
+
|
|
774
|
+
temporary = "#{path}.compact-#{SecureRandom.hex(12)}"
|
|
775
|
+
begin
|
|
776
|
+
File.link(canonical, temporary)
|
|
777
|
+
raise IOError, 'Backup readback mismatch' unless exact_bytes(temporary, size) == exact_bytes(path, size)
|
|
778
|
+
|
|
779
|
+
File.rename(temporary, path)
|
|
780
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
781
|
+
raise IOError, 'Compacted backup mismatch' unless verified_backup?(path, hash, admitted: true, size: size)
|
|
782
|
+
ensure
|
|
783
|
+
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
784
|
+
end
|
|
785
|
+
end
|
|
786
|
+
|
|
787
|
+
# Status is not rollback: broad maintenance has independently committed
|
|
788
|
+
# children. Restoring a ledger alone never undoes deletion or migration.
|
|
789
|
+
def self.recovery_status(ledger:)
|
|
790
|
+
root = File.join(Dir.home, '.pwn')
|
|
791
|
+
prefixes = %w[maintenance session_cleanup_archive learning/migrations].map { |name| "#{File.join(root, name)}/" }
|
|
792
|
+
raise IOError, 'Unsafe recovery ledger' unless prefixes.any? { |prefix| ledger.start_with?(prefix) } && File.realpath(ledger) == ledger
|
|
793
|
+
|
|
794
|
+
data = JSON.parse(File.binread(ledger))
|
|
795
|
+
operation = data['operation']
|
|
796
|
+
operation = 'session_layout' if File.basename(ledger) == 'layout.json' && ledger.start_with?("#{File.join(root, 'session_cleanup_archive')}/") && data['records'].is_a?(Array)
|
|
797
|
+
operation = 'learning_migration' if ledger.start_with?("#{File.join(root, 'learning', 'migrations')}/") && data['version'] == 1 && data['before_sha256'] == File.basename(ledger, '.json')
|
|
798
|
+
operation = 'session_cleanup' if ledger.start_with?("#{File.join(root, 'session_cleanup_archive')}/") && data['dry_run'] == false && data['records'].is_a?(Array) && data['deleted_ids'].is_a?(Array)
|
|
799
|
+
operation = 'continuation_retirement' if ledger.start_with?("#{File.join(root, 'maintenance', 'retirements')}/") && %w[prepared committed rolled_back].include?(data['state']) && data['backup'] == File.join(File.dirname(ledger), 'personas.json')
|
|
800
|
+
actions = { 'session_layout' => ['resume'], 'session_cleanup' => ['restore'], 'continuation_retirement' => ['rollback'], 'backup_compaction' => ['resume'], 'maintenance' => ['resume'], 'learning_migration' => ['rollback'], 'routine' => ['restore'] }.fetch(operation, ['inspect_store_specific_recovery'])
|
|
801
|
+
actions << 'rollback_dependencies' if operation == 'session_cleanup' && data['dependency_removals'].is_a?(Array)
|
|
802
|
+
{ ledger: ledger, state: data['state'] || 'legacy_per_record', operation: operation,
|
|
803
|
+
actions: actions,
|
|
804
|
+
data: data, warning: 'Ledger restoration does not undo data changes; restore source sessions before learning rollback.' }
|
|
805
|
+
end
|
|
806
|
+
|
|
807
|
+
# Recovery is explicit repair, never startup/cron behavior. Both prepared
|
|
808
|
+
# and committed rows retain exact backups; an existing destination wins.
|
|
809
|
+
def self.recover(ledger:, dry_run: true, action: 'restore', writers_stopped: false)
|
|
810
|
+
report = recovery_status(ledger: ledger)
|
|
811
|
+
raise ArgumentError, 'Unsupported recovery action; inspect recovery_status' unless report[:actions].include?(action)
|
|
812
|
+
|
|
813
|
+
return reorganize(dry_run: dry_run, writers_stopped: writers_stopped) if report[:operation] == 'session_layout'
|
|
814
|
+
|
|
815
|
+
return recover_dependencies(ledger: ledger, data: report[:data], dry_run: dry_run) if action == 'rollback_dependencies'
|
|
816
|
+
return recover_sessions(ledger: ledger, data: report[:data], dry_run: dry_run) if report[:operation] == 'session_cleanup'
|
|
817
|
+
return recover_continuation(ledger: ledger, data: report[:data], dry_run: dry_run) if report[:operation] == 'continuation_retirement'
|
|
818
|
+
if report[:operation] == 'learning_migration'
|
|
819
|
+
return with_maintenance_lock(dry_run: dry_run) do
|
|
820
|
+
PWN::AI::Agent::Learning.rollback_evidence(migration: File.basename(ledger, '.json'), dry_run: dry_run)
|
|
821
|
+
end
|
|
822
|
+
end
|
|
823
|
+
return dry_run ? report.merge(dry_run: true) : maintain(dry_run: false) if report[:operation] == 'maintenance'
|
|
824
|
+
if report[:operation] == 'backup_compaction'
|
|
825
|
+
return with_maintenance_lock(dry_run: dry_run) do
|
|
826
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
827
|
+
known = legacy_backups.to_h { |row| [row[:path], row[:sha256]] }
|
|
828
|
+
data = report[:data]
|
|
829
|
+
data.fetch('records').each do |row|
|
|
830
|
+
raise IOError, 'Unknown compaction source' unless [row['path'], row['canonical']].all? { |path| known[path] == row['sha256'] }
|
|
831
|
+
next if dry_run
|
|
832
|
+
|
|
833
|
+
compact_backup(row.transform_keys(&:to_sym))
|
|
834
|
+
row['state'] = 'committed'
|
|
835
|
+
publish(ledger, JSON.generate(data))
|
|
836
|
+
end
|
|
837
|
+
data['state'] = 'committed' unless dry_run
|
|
838
|
+
publish(ledger, JSON.generate(data)) unless dry_run
|
|
839
|
+
{ dry_run: dry_run, state: data['state'], operation: 'backup_compaction' }
|
|
840
|
+
end
|
|
841
|
+
end
|
|
842
|
+
end
|
|
843
|
+
root = File.join(Dir.home, '.pwn', 'maintenance', 'quarantine')
|
|
844
|
+
raise IOError, 'Unsafe recovery ledger' unless ledger.start_with?("#{root}/") && File.realpath(ledger) == ledger
|
|
845
|
+
|
|
846
|
+
data = JSON.parse(File.binread(ledger))
|
|
847
|
+
raise IOError, 'Unknown recovery ledger' unless data['operation'] == 'routine' && data['records'].is_a?(Array)
|
|
848
|
+
|
|
849
|
+
result = { operation: 'repair', dry_run: dry_run, restored: [] }
|
|
850
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
851
|
+
data['records'].each do |row|
|
|
852
|
+
path = row.fetch('path')
|
|
853
|
+
category = row.fetch('category')
|
|
854
|
+
target_root = File.join(Dir.home, '.pwn', category)
|
|
855
|
+
backup_path = row.fetch('backup')
|
|
856
|
+
raise IOError, 'Unsafe recovery path' unless %w[cache tmp logs].include?(category) && path.start_with?("#{target_root}/") && File.expand_path(path) == path && File.realpath(File.dirname(path)) == File.dirname(path)
|
|
857
|
+
raise IOError, 'Unsafe recovery backup' unless File.dirname(backup_path) == File.dirname(ledger) && File.realpath(backup_path) == backup_path && Digest::SHA256.file(backup_path).hexdigest == row['sha256']
|
|
858
|
+
raise IOError, 'Refusing existing recovery destination' if File.exist?(path) || File.symlink?(path)
|
|
859
|
+
next if dry_run
|
|
860
|
+
|
|
861
|
+
File.open("#{path}.lease", File::RDWR | File::NOFOLLOW) do |lease|
|
|
862
|
+
raise IOError, 'Recovery destination active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
863
|
+
|
|
864
|
+
temporary = "#{path}.restore-#{SecureRandom.hex(12)}"
|
|
865
|
+
begin
|
|
866
|
+
publish(temporary, File.binread(backup_path))
|
|
867
|
+
File.link(temporary, path) # atomic no-clobber; never replace a reader's new file
|
|
868
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
869
|
+
result[:restored] << path
|
|
870
|
+
row['recovered_at'] = Time.now.utc.iso8601
|
|
871
|
+
publish(ledger, JSON.pretty_generate(data))
|
|
872
|
+
ensure
|
|
873
|
+
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
874
|
+
end
|
|
875
|
+
end
|
|
876
|
+
end
|
|
877
|
+
end
|
|
878
|
+
result
|
|
879
|
+
end
|
|
880
|
+
|
|
881
|
+
def self.recover_dependencies(ledger:, data:, dry_run:)
|
|
882
|
+
result = { dry_run: dry_run, restored: [] }
|
|
883
|
+
root = File.join(Dir.home, '.pwn')
|
|
884
|
+
allowed = %w[learning.jsonl memory.json memory.archive.json mistakes.json].map { |name| File.join(root, name) }
|
|
885
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
886
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
887
|
+
data.fetch('dependency_removals').reverse_each do |row|
|
|
888
|
+
path, backup_path, id = row.values_at('path', 'backup', 'session_id')
|
|
889
|
+
raise IOError, 'Unsafe dependency recovery target' unless allowed.include?(path) && File.realpath(path) == path
|
|
890
|
+
raise IOError, 'Invalid dependency recovery backup' unless File.dirname(backup_path) == File.dirname(ledger) && verified_backup?(backup_path, row['sha256'])
|
|
891
|
+
raise IOError, 'Restore source session before dependency rollback' unless id.is_a?(String) && id.match?(PWN::Sessions::SESSION_ID) && File.file?(File.join(PWN::Sessions.sessions_dir, "#{id}.jsonl"))
|
|
892
|
+
|
|
893
|
+
current = Digest::SHA256.file(path).hexdigest
|
|
894
|
+
next if current == row['sha256']
|
|
895
|
+
raise IOError, 'Dependency store changed since removal' unless current == row['updated_sha256']
|
|
896
|
+
next if dry_run
|
|
897
|
+
|
|
898
|
+
publish(path, File.binread(backup_path))
|
|
899
|
+
raise IOError, 'Dependency restore mismatch' unless Digest::SHA256.file(path).hexdigest == row['sha256']
|
|
900
|
+
|
|
901
|
+
row['state'] = 'rolled_back'
|
|
902
|
+
publish(ledger, JSON.generate(data))
|
|
903
|
+
result[:restored] << path
|
|
904
|
+
end
|
|
905
|
+
end
|
|
906
|
+
end
|
|
907
|
+
result
|
|
908
|
+
end
|
|
909
|
+
|
|
910
|
+
def self.restore_copy(backup_path, path, hash)
|
|
911
|
+
raise IOError, 'Invalid recovery backup' unless verified_backup?(backup_path, hash)
|
|
912
|
+
raise IOError, 'Refusing existing recovery destination' if File.exist?(path) || File.symlink?(path)
|
|
913
|
+
raise IOError, 'Unsafe recovery parent' unless File.realpath(File.dirname(path)) == File.dirname(path)
|
|
914
|
+
|
|
915
|
+
temporary = "#{path}.restore-#{SecureRandom.hex(12)}"
|
|
916
|
+
begin
|
|
917
|
+
publish(temporary, File.binread(backup_path))
|
|
918
|
+
raise IOError, 'Restore readback mismatch' unless Digest::SHA256.file(temporary).hexdigest == hash
|
|
919
|
+
|
|
920
|
+
File.link(temporary, path)
|
|
921
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
922
|
+
ensure
|
|
923
|
+
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
924
|
+
end
|
|
925
|
+
end
|
|
926
|
+
|
|
927
|
+
def self.recover_sessions(ledger:, data:, dry_run:)
|
|
928
|
+
result = { dry_run: dry_run, restored: [], warning: 'Session bytes only; dependency remediations and learning migration are separate transactions.' }
|
|
929
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
930
|
+
data.fetch('records').each do |row|
|
|
931
|
+
id = row['id']
|
|
932
|
+
next unless id.is_a?(String) && id.match?(PWN::Sessions::SESSION_ID)
|
|
933
|
+
|
|
934
|
+
# A crash can follow unlink before deleted_ids was journaled. The
|
|
935
|
+
# exact prepared snapshot is also eligible, but never replaces data.
|
|
936
|
+
next unless row['files'].is_a?(Hash)
|
|
937
|
+
|
|
938
|
+
row['files'].each do |name, info|
|
|
939
|
+
next unless info && ["#{id}.jsonl", "#{id}.meta.json"].include?(name)
|
|
940
|
+
|
|
941
|
+
backup_path = File.join(File.dirname(ledger), name)
|
|
942
|
+
next unless File.file?(backup_path)
|
|
943
|
+
raise IOError, 'Invalid session recovery backup' unless verified_backup?(backup_path, info['sha256'])
|
|
944
|
+
|
|
945
|
+
path = name.end_with?('.meta.json') ? PWN::Sessions.support_path(kind: :metadata, session_id: id) : File.join(PWN::Sessions.sessions_dir, name)
|
|
946
|
+
next if verified_backup?(path, info['sha256'])
|
|
947
|
+
raise IOError, 'Refusing existing recovery destination' if File.exist?(path) || File.symlink?(path)
|
|
948
|
+
next if dry_run
|
|
949
|
+
|
|
950
|
+
File.open(PWN::Sessions.support_path(kind: :lease, session_id: id), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
951
|
+
raise IOError, 'Session recovery active' unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
952
|
+
|
|
953
|
+
restore_copy(backup_path, path, info['sha256'])
|
|
954
|
+
result[:restored] << path
|
|
955
|
+
(data['restored_paths'] ||= []) << path
|
|
956
|
+
publish(ledger, JSON.generate(data))
|
|
957
|
+
end
|
|
958
|
+
end
|
|
959
|
+
end
|
|
960
|
+
end
|
|
961
|
+
result
|
|
962
|
+
end
|
|
963
|
+
|
|
964
|
+
def self.recover_continuation(ledger:, data:, dry_run:)
|
|
965
|
+
root = File.join(Dir.home, '.pwn')
|
|
966
|
+
path, id = data.values_at('path', 'session_id')
|
|
967
|
+
raise IOError, 'Unsafe continuation recovery path' unless path.is_a?(String) && path.match?(%r{\A#{Regexp.escape(root)}/swarm/\d{8}_\d{6}_[a-f0-9]{6}/personas\.json\z}) && File.realpath(path) == path
|
|
968
|
+
raise IOError, 'Invalid continuation recovery identity' unless id.is_a?(String) && id.match?(PWN::Sessions::SESSION_ID)
|
|
969
|
+
raise IOError, 'Invalid continuation backup' unless verified_backup?(data['backup'], data['sha256'])
|
|
970
|
+
|
|
971
|
+
before = File.binread(data['backup'])
|
|
972
|
+
map = JSON.parse(before)
|
|
973
|
+
raise IOError, 'Invalid continuation backup schema' unless map.is_a?(Hash) && map[data['name']] == id
|
|
974
|
+
|
|
975
|
+
expected = map.reject { |name, _| name == data['name'] }
|
|
976
|
+
result = { dry_run: dry_run, restored: [] }
|
|
977
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
978
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
979
|
+
current = JSON.parse(File.binread(path))
|
|
980
|
+
raise IOError, 'Continuation map changed after retirement' unless current == expected || current == map
|
|
981
|
+
next if dry_run || current == map
|
|
982
|
+
|
|
983
|
+
File.open(PWN::Sessions.support_path(kind: :lease, session_id: id), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
984
|
+
raise IOError, 'Continuation recovery active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
985
|
+
raise IOError, 'Restore source session before continuation' unless File.file?(File.join(PWN::Sessions.sessions_dir, "#{id}.jsonl"))
|
|
986
|
+
|
|
987
|
+
publish(path, before)
|
|
988
|
+
raise IOError, 'Continuation restore readback mismatch' unless File.binread(path) == before
|
|
989
|
+
|
|
990
|
+
data['state'] = 'rolled_back'
|
|
991
|
+
publish(ledger, JSON.generate(data))
|
|
992
|
+
result[:restored] << path
|
|
993
|
+
end
|
|
994
|
+
end
|
|
995
|
+
end
|
|
996
|
+
result
|
|
997
|
+
end
|
|
998
|
+
|
|
999
|
+
def self.with_continuation_lock(root: File.join(Dir.home, '.pwn'))
|
|
1000
|
+
File.open(File.join(root, '.swarm-continuations.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
1001
|
+
lock.flock(File::LOCK_EX)
|
|
1002
|
+
yield
|
|
1003
|
+
end
|
|
1004
|
+
end
|
|
1005
|
+
|
|
1006
|
+
def self.retire_continuation(root:, swarm_id:, name:)
|
|
1007
|
+
raise ArgumentError, 'Invalid swarm identifier' unless swarm_id.match?(/\A\d{8}_\d{6}_[a-f0-9]{6}\z/)
|
|
1008
|
+
|
|
1009
|
+
path = File.join(root, swarm_id, 'personas.json')
|
|
1010
|
+
with_maintenance_lock(dry_run: false) do
|
|
1011
|
+
with_continuation_lock(root: File.dirname(root)) do
|
|
1012
|
+
raise IOError, 'Unsafe continuation map' unless File.realpath(path) == path
|
|
1013
|
+
|
|
1014
|
+
bytes = File.binread(path)
|
|
1015
|
+
map = JSON.parse(bytes)
|
|
1016
|
+
raise IOError, 'Unknown continuation map' unless map.is_a?(Hash) && map.values.all? { |id| id.is_a?(String) && id.match?(/\A\d{8}_\d{6}_[a-f0-9]{8}\z/) }
|
|
1017
|
+
|
|
1018
|
+
id = map[name]
|
|
1019
|
+
next { state: 'absent' } unless id
|
|
1020
|
+
|
|
1021
|
+
File.open(PWN::Sessions.support_path(kind: :lease, session_id: id), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
1022
|
+
raise IOError, 'Swarm continuation active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
1023
|
+
|
|
1024
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'retirements', SecureRandom.hex(12)))
|
|
1025
|
+
backup_path = File.join(directory, 'personas.json')
|
|
1026
|
+
publish(backup_path, bytes)
|
|
1027
|
+
raise IOError, 'Continuation backup mismatch' unless File.binread(backup_path) == bytes
|
|
1028
|
+
|
|
1029
|
+
result = { state: 'prepared', session_id: id, path: path, name: name, backup: backup_path, sha256: Digest::SHA256.hexdigest(bytes) }
|
|
1030
|
+
ledger = File.join(directory, 'ledger.json')
|
|
1031
|
+
publish(ledger, JSON.pretty_generate(result))
|
|
1032
|
+
map.delete(name)
|
|
1033
|
+
publish(path, JSON.pretty_generate(map))
|
|
1034
|
+
raise IOError, 'Continuation retirement mismatch' unless JSON.parse(File.binread(path)) == map
|
|
1035
|
+
|
|
1036
|
+
result[:state] = 'committed'
|
|
1037
|
+
publish(ledger, JSON.pretty_generate(result))
|
|
1038
|
+
result
|
|
1039
|
+
end
|
|
1040
|
+
end
|
|
1041
|
+
end
|
|
1042
|
+
end
|
|
16
1043
|
|
|
17
1044
|
def self.classify(path, id)
|
|
18
1045
|
return :unknown unless path == File.join(PWN::Sessions.sessions_dir, "#{id}.jsonl")
|
|
@@ -113,15 +1140,73 @@ module PWN
|
|
|
113
1140
|
|
|
114
1141
|
def self.publish(path, bytes)
|
|
115
1142
|
temporary = "#{path}.#{SecureRandom.hex(8)}.tmp"
|
|
1143
|
+
root = File.join(Dir.home, '.pwn')
|
|
1144
|
+
category = %w[cache tmp].find { |name| path.start_with?("#{File.join(root, name)}/") }
|
|
1145
|
+
if category
|
|
1146
|
+
directory = private_directory(File.join(root, 'maintenance', 'atomic'))
|
|
1147
|
+
journal = File.join(directory, "#{SecureRandom.hex(12)}.json")
|
|
1148
|
+
end
|
|
116
1149
|
File.open(temporary, File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW, 0o600) do |file|
|
|
1150
|
+
file.flock(File::LOCK_EX)
|
|
1151
|
+
if journal
|
|
1152
|
+
row = { version: 1, operation: 'atomic_disposable', category: category, path: temporary, target: path,
|
|
1153
|
+
inode: file.stat.ino, device: file.stat.dev, sha256: Digest::SHA256.hexdigest(bytes), created_at: Time.now.to_i }
|
|
1154
|
+
publish(journal, JSON.generate(row))
|
|
1155
|
+
end
|
|
117
1156
|
file.write(bytes)
|
|
118
1157
|
file.flush
|
|
119
1158
|
file.fsync
|
|
1159
|
+
File.rename(temporary, path)
|
|
1160
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
120
1161
|
end
|
|
121
|
-
File.rename(temporary, path)
|
|
122
|
-
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
123
1162
|
ensure
|
|
124
1163
|
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
1164
|
+
File.unlink(journal) if journal && File.exist?(journal)
|
|
1165
|
+
end
|
|
1166
|
+
|
|
1167
|
+
# Only registered disposable atomic outputs are recoverable automatically.
|
|
1168
|
+
# Missing journals, partial bytes, wrong inode, active writers and evidence
|
|
1169
|
+
# atomic outputs are preserved, regardless of filename or apparent age.
|
|
1170
|
+
def self.recover_atomic_temps(dry_run: true, expired_only: false)
|
|
1171
|
+
result = { deleted: [], retained: [] }
|
|
1172
|
+
root = File.join(Dir.home, '.pwn')
|
|
1173
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
1174
|
+
Dir.glob(File.join(root, 'maintenance', 'atomic', '*.json')).first(policy['scan_limit']).each do |ledger|
|
|
1175
|
+
break unless admit?
|
|
1176
|
+
break if result[:deleted].length >= policy['batch_size']
|
|
1177
|
+
raise IOError, 'Unsafe atomic ledger' unless File.realpath(ledger) == ledger
|
|
1178
|
+
|
|
1179
|
+
row = JSON.parse(admitted_bytes(ledger))
|
|
1180
|
+
raise IOError, 'Unknown atomic owner' unless row['version'] == 1 && row['operation'] == 'atomic_disposable' && %w[cache tmp].include?(row['category'])
|
|
1181
|
+
|
|
1182
|
+
days = policy['retention_days'][row['category']]
|
|
1183
|
+
next if days.nil? || (expired_only && row.fetch('created_at') > Time.now.to_i - (days * 86_400))
|
|
1184
|
+
|
|
1185
|
+
path, target = row.values_at('path', 'target')
|
|
1186
|
+
base = "#{File.join(root, row['category'])}/"
|
|
1187
|
+
raise IOError, 'Unsafe atomic path' unless [path, target].all? { |name| name.is_a?(String) && name.start_with?(base) && File.expand_path(name) == name } && File.dirname(path) == File.dirname(target) && path.match?(/\A#{Regexp.escape(target)}\.[a-f0-9]{16}\.tmp\z/)
|
|
1188
|
+
next unless File.exist?(path) || File.symlink?(path)
|
|
1189
|
+
raise IOError, 'Changed atomic bytes' unless verified_backup?(path, row['sha256']) && File.stat(path).ino == row['inode'] && File.stat(path).dev == row['device']
|
|
1190
|
+
|
|
1191
|
+
File.open("#{target}.lease", File::RDWR | File::NOFOLLOW) do |lease|
|
|
1192
|
+
raise IOError, 'Atomic producer active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
1193
|
+
|
|
1194
|
+
File.open(path, File::RDONLY | File::NOFOLLOW) do |file|
|
|
1195
|
+
raise IOError, 'Atomic file active or changed' unless file.flock(File::LOCK_EX | File::LOCK_NB) && File.identical?(file, path) && file.stat.ino == row['inode'] && Digest::SHA256.hexdigest(admitted_bytes(path)) == row['sha256']
|
|
1196
|
+
next if dry_run
|
|
1197
|
+
|
|
1198
|
+
File.unlink(path)
|
|
1199
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
1200
|
+
row['state'] = 'recovered'
|
|
1201
|
+
publish(ledger, JSON.generate(row))
|
|
1202
|
+
result[:deleted] << path
|
|
1203
|
+
end
|
|
1204
|
+
end
|
|
1205
|
+
rescue StandardError => e
|
|
1206
|
+
result[:retained] << { ledger: ledger, reason: e.class.name }
|
|
1207
|
+
end
|
|
1208
|
+
end
|
|
1209
|
+
result
|
|
125
1210
|
end
|
|
126
1211
|
|
|
127
1212
|
def self.backup(path, expected, result)
|
|
@@ -175,14 +1260,16 @@ module PWN
|
|
|
175
1260
|
state
|
|
176
1261
|
end
|
|
177
1262
|
|
|
178
|
-
def self.with_dependency_lock(id)
|
|
1263
|
+
def self.with_dependency_lock(id, dry_run: false)
|
|
1264
|
+
return yield if dry_run
|
|
1265
|
+
|
|
179
1266
|
rows = invalid_dependencies(id)
|
|
180
1267
|
raise 'Unresolved dependency remediation' unless rows.empty? || remediable?(id)
|
|
181
1268
|
|
|
182
1269
|
# Lock even absent stores: a concurrent writer can introduce the first
|
|
183
1270
|
# reference between the final index and unlink. These inodes are permanent.
|
|
184
1271
|
root = File.join(Dir.home, '.pwn')
|
|
185
|
-
paths = %w[.learning-write.lock memory.json.write.lock mistakes.json.write.lock].map { |name| File.join(root, name) }.sort
|
|
1272
|
+
paths = %w[.learning-write.lock .swarm-continuations.lock memory.json.write.lock mistakes.json.write.lock].map { |name| File.join(root, name) }.sort
|
|
186
1273
|
locks = []
|
|
187
1274
|
begin
|
|
188
1275
|
paths.each do |path|
|
|
@@ -262,15 +1349,110 @@ module PWN
|
|
|
262
1349
|
end
|
|
263
1350
|
end
|
|
264
1351
|
|
|
1352
|
+
def self.stopping?
|
|
1353
|
+
control = Thread.current[:pwn_steering_input]
|
|
1354
|
+
budget = Thread.current[:pwn_maintenance_budget]
|
|
1355
|
+
(control.respond_to?(:stopping?) && control.stopping?) || (budget && (budget[:remaining] <= 0 || budget[:bytes] <= 0 || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= budget[:deadline]))
|
|
1356
|
+
end
|
|
1357
|
+
|
|
1358
|
+
# Fresh schedules use one coordinator. Existing split jobs keep maintain
|
|
1359
|
+
# until an operator explicitly replaces their schedules after deployment.
|
|
1360
|
+
# Preserve evidence before retention can remove its last source edge.
|
|
1361
|
+
def self.coordinate(dry_run: false, current_session_id: nil, proc_root: '/proc')
|
|
1362
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
1363
|
+
result = { state: 'prepared', dry_run: dry_run, phases: {} }
|
|
1364
|
+
unless dry_run
|
|
1365
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'runs'))
|
|
1366
|
+
ledger = File.join(directory, "coordinator-#{SecureRandom.hex(12)}.json")
|
|
1367
|
+
publish(ledger, JSON.generate(result))
|
|
1368
|
+
end
|
|
1369
|
+
begin
|
|
1370
|
+
maintenance = maintain(dry_run: dry_run, current_session_id: current_session_id, proc_root: proc_root)
|
|
1371
|
+
result[:phases][:maintenance] = maintenance
|
|
1372
|
+
if maintenance[:apply_blocked] || maintenance[:incomplete] || stopping?
|
|
1373
|
+
result[:state] = maintenance[:apply_blocked] ? 'blocked' : 'interrupted'
|
|
1374
|
+
else
|
|
1375
|
+
result[:phases][:memory_consolidation] = dry_run ? { planned: true } : PWN::AI::Agent::Learning.consolidate
|
|
1376
|
+
result[:phases][:learning_retention] = PWN::AI::Agent::Learning.prune_outcomes!(dry_run: dry_run)
|
|
1377
|
+
result[:phases][:mistakes_retention] = PWN::AI::Agent::Mistakes.lean!(dry_run: dry_run)
|
|
1378
|
+
result[:phases][:policy_retention] = PWN::AI::Agent::Policy.lean!(dry_run: dry_run)
|
|
1379
|
+
result[:state] = dry_run ? 'planned' : 'committed'
|
|
1380
|
+
end
|
|
1381
|
+
result[:transaction_ledger] = ledger if ledger
|
|
1382
|
+
publish(ledger, JSON.generate(result)) if ledger
|
|
1383
|
+
result
|
|
1384
|
+
rescue StandardError, Interrupt, SystemExit => e
|
|
1385
|
+
result[:state] = 'interrupted'
|
|
1386
|
+
publish(ledger, JSON.generate(result.merge(error: e.class.name))) if ledger
|
|
1387
|
+
raise
|
|
1388
|
+
end
|
|
1389
|
+
end
|
|
1390
|
+
end
|
|
1391
|
+
|
|
265
1392
|
def self.maintain(dry_run: false, current_session_id: nil, proc_root: '/proc')
|
|
1393
|
+
config = policy # validate policy before any mutation or migration
|
|
1394
|
+
prior_budget = Thread.current[:pwn_maintenance_budget]
|
|
1395
|
+
prior_admission = Thread.current[:pwn_maintenance_admission]
|
|
1396
|
+
Thread.current[:pwn_maintenance_budget] = { remaining: config['work_items'], bytes: config['work_bytes'], deadline: Process.clock_gettime(Process::CLOCK_MONOTONIC) + config['work_seconds'] }
|
|
1397
|
+
Thread.current[:pwn_maintenance_admission] = method(:admit?)
|
|
266
1398
|
with_maintenance_lock(dry_run: dry_run) do
|
|
267
|
-
|
|
1399
|
+
unless dry_run
|
|
1400
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'runs'))
|
|
1401
|
+
ledger = File.join(directory, "#{Time.now.utc.strftime('%Y%m%dT%H%M%S')}-#{SecureRandom.hex(8)}.json")
|
|
1402
|
+
publish(ledger, JSON.generate(version: 1, state: 'prepared', operation: 'maintenance'))
|
|
1403
|
+
end
|
|
1404
|
+
begin
|
|
1405
|
+
result = maintenance_transaction(dry_run: dry_run, current_session_id: current_session_id, proc_root: proc_root)
|
|
1406
|
+
result[:storage] = clean_disposables(dry_run: dry_run || result[:apply_blocked] == true)
|
|
1407
|
+
result[:quarantine] = prune_quarantine(dry_run: dry_run || result[:apply_blocked] == true)
|
|
1408
|
+
unless stopping?
|
|
1409
|
+
result[:atomic_temps] = recover_atomic_temps(dry_run: dry_run || result[:apply_blocked] == true, expired_only: true)
|
|
1410
|
+
result[:legacy_backups] = prune_legacy_backups(dry_run: dry_run || result[:apply_blocked] == true)
|
|
1411
|
+
end
|
|
1412
|
+
result[:migration_backups] = PWN::Migrate.prune_backups(dry_run: dry_run || result[:apply_blocked] == true, admission: method(:admit?)) unless stopping?
|
|
1413
|
+
result[:usage] = status unless stopping?
|
|
1414
|
+
result[:incomplete] = true if stopping? || result[:storage][:incomplete]
|
|
1415
|
+
result[:state] = if dry_run
|
|
1416
|
+
'planned'
|
|
1417
|
+
elsif result[:incomplete] || result[:storage][:state] == 'interrupted'
|
|
1418
|
+
'interrupted'
|
|
1419
|
+
elsif result[:apply_blocked]
|
|
1420
|
+
'blocked'
|
|
1421
|
+
else
|
|
1422
|
+
'committed'
|
|
1423
|
+
end
|
|
1424
|
+
result[:transaction_ledger] = ledger if ledger
|
|
1425
|
+
publish(ledger, JSON.generate(version: 1, state: result[:state], operation: 'maintenance', result: result)) if ledger
|
|
1426
|
+
result
|
|
1427
|
+
rescue StandardError, Interrupt, SystemExit => e
|
|
1428
|
+
publish(ledger, JSON.generate(version: 1, state: 'interrupted', operation: 'maintenance', error: e.class.name, result: result)) if ledger
|
|
1429
|
+
raise
|
|
1430
|
+
end
|
|
1431
|
+
end
|
|
1432
|
+
ensure
|
|
1433
|
+
Thread.current[:pwn_maintenance_budget] = prior_budget
|
|
1434
|
+
Thread.current[:pwn_maintenance_admission] = prior_admission
|
|
1435
|
+
end
|
|
1436
|
+
|
|
1437
|
+
# Admission is cooperative: an admitted atomic write is never interrupted.
|
|
1438
|
+
def self.admit?(bytes = 0)
|
|
1439
|
+
return false if stopping?
|
|
1440
|
+
|
|
1441
|
+
budget = Thread.current[:pwn_maintenance_budget]
|
|
1442
|
+
if budget
|
|
1443
|
+
budget[:remaining] -= 1
|
|
1444
|
+
if bytes > budget[:bytes]
|
|
1445
|
+
budget[:bytes] = 0
|
|
1446
|
+
return false
|
|
1447
|
+
end
|
|
1448
|
+
budget[:bytes] -= bytes
|
|
268
1449
|
end
|
|
1450
|
+
true
|
|
269
1451
|
end
|
|
270
1452
|
|
|
271
1453
|
def self.maintenance_transaction(dry_run:, current_session_id:, proc_root:)
|
|
272
1454
|
preflight = sweep(dry_run: true, current_session_id: current_session_id, proc_root: proc_root)
|
|
273
|
-
blocked = !preflight[:legacy_owners].empty? || Array(preflight[:unresolved_store_remediation]).any?
|
|
1455
|
+
blocked = stopping? || !preflight[:legacy_owners].empty? || Array(preflight[:unresolved_store_remediation]).any?
|
|
274
1456
|
unless dry_run || blocked
|
|
275
1457
|
# Private metadata-only preflight survives an interrupted migration.
|
|
276
1458
|
backup(File.join(PWN::Sessions.sessions_dir, 'inventory.jsonl'), {}, preflight)
|
|
@@ -278,7 +1460,10 @@ module PWN
|
|
|
278
1460
|
publish(ledger, JSON.pretty_generate(state: 'prepared', inventory: preflight))
|
|
279
1461
|
blocked = !legacy_owners(proc_root).empty?
|
|
280
1462
|
end
|
|
281
|
-
|
|
1463
|
+
return preflight.merge(incomplete: true, apply_blocked: true) if stopping?
|
|
1464
|
+
|
|
1465
|
+
migration = PWN::AI::Agent::Learning.migrate_evidence(dry_run: dry_run || blocked, max_sessions: policy['batch_size'], stop: method(:stopping?), nonblocking: true)
|
|
1466
|
+
blocked = true if migration[:deferred]
|
|
282
1467
|
result = if dry_run || blocked
|
|
283
1468
|
preflight
|
|
284
1469
|
else
|
|
@@ -404,7 +1589,7 @@ module PWN
|
|
|
404
1589
|
{ version: VERSION, dry_run: result[:dry_run], apply_blocked: result[:apply_blocked] == true,
|
|
405
1590
|
scanned: result[:scanned], candidates: result[:candidate_ids].length, deleted: result[:deleted_ids].length,
|
|
406
1591
|
remaining: result[:remaining_sessions], visible: result[:visible_sessions], skipped: result[:skipped],
|
|
407
|
-
evidence_migration: result[:evidence_migration], archive_dir: result[:archive_dir],
|
|
1592
|
+
evidence_migration: result[:evidence_migration], archive_dir: result[:archive_dir], storage: result[:storage], usage: result[:usage], quarantine: result[:quarantine],
|
|
408
1593
|
unresolved_dependency_stores: result[:dependency_errors] || [], legacy_owners: result[:legacy_owners] }
|
|
409
1594
|
end
|
|
410
1595
|
|
|
@@ -430,6 +1615,7 @@ module PWN
|
|
|
430
1615
|
legacy = legacy_owners(proc_root)
|
|
431
1616
|
result = { version: VERSION, dry_run: dry_run, scanned: 0, unnamed: 0, candidate_ids: [], deleted_ids: [], records: [], skipped: Hash.new(0), legacy_owners: legacy }
|
|
432
1617
|
references = PWN::Sessions.dependency_index
|
|
1618
|
+
result[:incomplete] = true if references[:errors].include?('incomplete_dependency_index')
|
|
433
1619
|
unless references[:errors].empty?
|
|
434
1620
|
result[:skipped][:unknown_pins] = 1
|
|
435
1621
|
result[:dependency_errors] = references[:errors]
|
|
@@ -439,6 +1625,10 @@ module PWN
|
|
|
439
1625
|
dependencies = references[:dependencies]
|
|
440
1626
|
pins = dependencies.keys
|
|
441
1627
|
Dir.glob(File.join(root, '*.jsonl')).each do |path|
|
|
1628
|
+
unless admit?
|
|
1629
|
+
result[:incomplete] = true
|
|
1630
|
+
break
|
|
1631
|
+
end
|
|
442
1632
|
id = File.basename(path, '.jsonl')
|
|
443
1633
|
result[:scanned] += 1
|
|
444
1634
|
state = cleanup_state(path, id, references)
|
|
@@ -457,6 +1647,12 @@ module PWN
|
|
|
457
1647
|
result[:skipped][:unresolved_dependency_remediation] += 1
|
|
458
1648
|
next
|
|
459
1649
|
end
|
|
1650
|
+
days = policy['retention_days']['sessions']
|
|
1651
|
+
if days.nil? || File.mtime(path) > Time.now - (days * 86_400)
|
|
1652
|
+
record[:skip] = :retention
|
|
1653
|
+
result[:skipped][:retention] += 1
|
|
1654
|
+
next
|
|
1655
|
+
end
|
|
460
1656
|
expected = snapshot(path)
|
|
461
1657
|
expected_dependencies = invalid_dependencies(id, references)
|
|
462
1658
|
record[:dependency_records] = expected_dependencies
|
|
@@ -484,6 +1680,17 @@ module PWN
|
|
|
484
1680
|
record[:apply_blocked_by] = :unknown_legacy_ownership unless legacy.empty?
|
|
485
1681
|
next if dry_run
|
|
486
1682
|
|
|
1683
|
+
if stopping?
|
|
1684
|
+
record[:skip] = :stopped
|
|
1685
|
+
result[:skipped][:stopped] += 1
|
|
1686
|
+
next
|
|
1687
|
+
end
|
|
1688
|
+
if result[:deleted_ids].length >= policy['batch_size']
|
|
1689
|
+
record[:skip] = :batch_limit
|
|
1690
|
+
result[:skipped][:batch_limit] += 1
|
|
1691
|
+
next
|
|
1692
|
+
end
|
|
1693
|
+
|
|
487
1694
|
unless legacy.empty? && legacy_owners(proc_root).empty?
|
|
488
1695
|
record[:skip] = :unknown_legacy_ownership
|
|
489
1696
|
result[:skipped][:unknown_legacy_ownership] += 1
|