pwn 0.5.783 → 0.5.785
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/Gemfile +8 -8
- data/README.md +37 -0
- data/Rakefile +19 -1
- data/bin/pwn +2 -1
- data/bin/pwn_setup +1 -1
- data/bin/pwn_web_cache_deception +35 -7
- data/build_gem.sh +1 -1
- data/documentation/AI-Integration.md +7 -5
- data/documentation/Agent-Tool-Registry.md +2 -2
- data/documentation/Backup-Retention.md +82 -0
- data/documentation/Configuration.md +10 -6
- data/documentation/Contributing.md +1 -1
- data/documentation/Cron.md +64 -35
- data/documentation/Diagrams.md +2 -2
- data/documentation/General-PWN-Usage.md +4 -3
- data/documentation/Home.md +6 -5
- data/documentation/How-PWN-Works.md +7 -6
- data/documentation/Installation.md +254 -10
- data/documentation/Mission-Cancellation.md +134 -0
- data/documentation/Operability.md +17 -0
- data/documentation/Persistence.md +8 -6
- data/documentation/Plugin-Degradation.md +12 -0
- data/documentation/Plugins.md +3 -2
- data/documentation/Reinforcement-Learning.md +5 -6
- data/documentation/Runtime-Source-Protection.md +162 -0
- data/documentation/Transparent-Browser.md +11 -0
- data/documentation/Troubleshooting.md +3 -2
- data/documentation/What-is-PWN.md +2 -2
- data/documentation/YubiKey-Vaults.md +183 -0
- data/documentation/YubiKey.md +339 -0
- data/documentation/diagrams/cron-scheduling.svg +130 -128
- data/documentation/diagrams/dot/cron-scheduling.dot +5 -5
- data/documentation/diagrams/dot/persistence-filesystem.dot +5 -3
- data/documentation/diagrams/dot/plugin-ecosystem.dot +2 -2
- data/documentation/diagrams/persistence-filesystem.svg +172 -142
- data/documentation/diagrams/plugin-ecosystem.svg +64 -64
- data/documentation/pwn-ai-Agent.md +10 -4
- data/etc/default_cron/session_cleanup.rb +1245 -22
- data/etc/default_skills/pwn/ai/agent/mission_namespace/SKILL.md +21 -0
- data/etc/default_skills/pwn/ai/agent/tools/browser_act/SKILL.md +45 -0
- data/etc/default_skills/pwn/ai/agent/tools/loot_query/SKILL.md +27 -26
- data/etc/default_skills/pwn/ai/agent/tools/recon_run/SKILL.md +25 -15
- data/etc/default_skills/pwn/ai/agent/tools/workspace_tool/SKILL.md +24 -0
- data/etc/default_skills/pwn/ai/server_tools/SKILL.md +25 -0
- data/etc/default_skills/pwn/backups/SKILL.md +54 -0
- data/etc/default_skills/pwn/plugins/recon/SKILL.md +12 -1
- data/etc/default_skills/pwn/plugins/yubikey/SKILL.md +157 -0
- data/etc/default_skills/pwn/plugins/yubikey/fido/SKILL.md +189 -0
- data/etc/default_skills/pwn/plugins/yubikey/hsmauth/SKILL.md +83 -0
- data/etc/default_skills/pwn/plugins/yubikey/management/SKILL.md +80 -0
- data/etc/default_skills/pwn/plugins/yubikey/oath/SKILL.md +115 -0
- data/etc/default_skills/pwn/plugins/yubikey/openpgp/SKILL.md +144 -0
- data/etc/default_skills/pwn/plugins/yubikey/otp/SKILL.md +97 -0
- data/etc/default_skills/pwn/plugins/yubikey/piv/SKILL.md +110 -0
- data/etc/default_skills/pwn/plugins/yubikey/protocol/SKILL.md +21 -0
- data/etc/default_skills/pwn/plugins/yubikey/security_domain/SKILL.md +75 -0
- data/etc/default_skills/pwn/plugins/yubikey/security_domain/scp/SKILL.md +46 -0
- data/etc/default_skills/pwn/plugins/yubikey/transport/SKILL.md +98 -0
- data/etc/default_skills/pwn/redaction/SKILL.md +13 -1
- data/lib/pwn/ai/agent/dispatch.rb +26 -15
- data/lib/pwn/ai/agent/extrospection.rb +6 -7
- data/lib/pwn/ai/agent/learning.rb +62 -25
- data/lib/pwn/ai/agent/loop.rb +17 -1
- data/lib/pwn/ai/agent/mission_namespace.rb +331 -0
- data/lib/pwn/ai/agent/policy.rb +5 -0
- data/lib/pwn/ai/agent/policy_evaluation.rb +8 -8
- data/lib/pwn/ai/agent/prompt_builder.rb +13 -2
- data/lib/pwn/ai/agent/request_runtime.rb +122 -18
- data/lib/pwn/ai/agent/reward.rb +5 -3
- data/lib/pwn/ai/agent/skill_consolidation.rb +3 -2
- data/lib/pwn/ai/agent/skill_review.rb +9 -5
- data/lib/pwn/ai/agent/swarm.rb +35 -16
- data/lib/pwn/ai/agent/tools/browser_act.rb +32 -0
- data/lib/pwn/ai/agent/tools/loot_query.rb +11 -1
- data/lib/pwn/ai/agent/tools/recon_run.rb +11 -1
- data/lib/pwn/ai/agent/tools/workspace_tool.rb +29 -0
- data/lib/pwn/ai/agent/turn_finalizer.rb +6 -2
- data/lib/pwn/ai/agent.rb +1 -0
- data/lib/pwn/ai/anthropic.rb +8 -33
- data/lib/pwn/ai/cli.rb +34 -5
- data/lib/pwn/ai/gemini.rb +1 -0
- data/lib/pwn/ai/grok.rb +39 -28
- data/lib/pwn/ai/ollama.rb +4 -2
- data/lib/pwn/ai/open_ai.rb +11 -32
- data/lib/pwn/ai/open_web_ui.rb +62 -4
- data/lib/pwn/ai/router.rb +53 -3
- data/lib/pwn/ai/server_tools.rb +231 -0
- data/lib/pwn/ai.rb +1 -0
- data/lib/pwn/backups.rb +347 -0
- data/lib/pwn/config.rb +70 -39
- data/lib/pwn/cron.rb +120 -49
- data/lib/pwn/migrate.rb +150 -68
- data/lib/pwn/module_skills.rb +7 -0
- data/lib/pwn/plugins/ai_sandbox.rb +538 -1
- data/lib/pwn/plugins/authentication_helper.rb +74 -4
- data/lib/pwn/plugins/basic_auth.rb +2 -1
- data/lib/pwn/plugins/binary_parser.rb +5 -4
- data/lib/pwn/plugins/ghidra.rb +4 -3
- data/lib/pwn/plugins/google_workspace.rb +4 -21
- data/lib/pwn/plugins/jobs.rb +1 -1
- data/lib/pwn/plugins/log.rb +21 -3
- data/lib/pwn/plugins/preflight_checker.rb +34 -9
- data/lib/pwn/plugins/recon.rb +4 -1
- data/lib/pwn/plugins/repl/ai.rb +305 -27
- data/lib/pwn/plugins/repl/ai_console.rb +273 -89
- data/lib/pwn/plugins/repl/ai_swarm.rb +181 -62
- data/lib/pwn/plugins/repl/mesh.rb +3 -16
- data/lib/pwn/plugins/repl/vault.rb +4 -15
- data/lib/pwn/plugins/repl.rb +1 -216
- data/lib/pwn/plugins/sandbox/driver.rb +4 -0
- data/lib/pwn/plugins/transparent_browser.rb +215 -0
- data/lib/pwn/plugins/vault.rb +412 -135
- data/lib/pwn/plugins/yubikey/fido.rb +1288 -0
- data/lib/pwn/plugins/yubikey/hsmauth.rb +439 -0
- data/lib/pwn/plugins/yubikey/management.rb +686 -0
- data/lib/pwn/plugins/yubikey/oath.rb +766 -0
- data/lib/pwn/plugins/yubikey/openpgp.rb +1063 -0
- data/lib/pwn/plugins/yubikey/otp.rb +897 -0
- data/lib/pwn/plugins/yubikey/piv.rb +1315 -0
- data/lib/pwn/plugins/yubikey/protocol.rb +341 -0
- data/lib/pwn/plugins/yubikey/security_domain/scp.rb +482 -0
- data/lib/pwn/plugins/yubikey/security_domain.rb +464 -0
- data/lib/pwn/plugins/yubikey/transport.rb +1017 -0
- data/lib/pwn/plugins/yubikey.rb +844 -0
- data/lib/pwn/plugins.rb +1 -0
- data/lib/pwn/redaction.rb +69 -12
- data/lib/pwn/session_trace.rb +4 -0
- data/lib/pwn/sessions.rb +58 -16
- data/lib/pwn/setup.rb +123 -41
- data/lib/pwn/version.rb +1 -1
- data/lib/pwn/www/app_cobalt_io.rb +16 -5
- data/lib/pwn/www/bug_crowd.rb +16 -5
- data/lib/pwn/www/coinbase_pro.rb +16 -5
- data/lib/pwn/www/github.rb +30 -14
- data/lib/pwn/www/paypal.rb +15 -4
- data/lib/pwn/www/synack.rb +16 -5
- data/lib/pwn/www/twitter.rb +17 -6
- data/lib/pwn.rb +1 -0
- data/scripts/build_gate.rb +58 -0
- data/spec/documentation/installation_md_spec.rb +1 -1
- data/spec/fixtures/yubikey/apdu_frames.json +162 -0
- data/spec/integration/authentication_helper_oath_spec.rb +117 -0
- data/spec/integration/build_gate_spec.rb +73 -0
- data/spec/integration/controller_spill_spec.rb +48 -0
- data/spec/integration/harness_process_spec.rb +97 -0
- data/spec/integration/mission_cancel_spec.rb +79 -0
- data/spec/integration/mission_curses_pty_spec.rb +44 -0
- data/spec/integration/persistence_roundtrip_spec.rb +1 -7
- data/spec/integration/readme_security_spec.rb +47 -0
- data/spec/integration/web_cache_deception_yubikey_spec.rb +159 -0
- data/spec/lib/pwn/ai/agent/dispatch_spec.rb +6 -3
- data/spec/lib/pwn/ai/agent/dispatch_vault_safety_spec.rb +86 -0
- data/spec/lib/pwn/ai/agent/injection_guard_spec.rb +2 -0
- data/spec/lib/pwn/ai/agent/learning_spec.rb +4 -4
- data/spec/lib/pwn/ai/agent/loop_artifacts_spec.rb +5 -0
- data/spec/lib/pwn/ai/agent/loop_spec.rb +104 -29
- data/spec/lib/pwn/ai/agent/mission_namespace_spec.rb +138 -0
- data/spec/lib/pwn/ai/agent/policy_spec.rb +27 -0
- data/spec/lib/pwn/ai/agent/request_runtime_spec.rb +48 -0
- data/spec/lib/pwn/ai/agent/shell_search_effect_spec.rb +44 -0
- data/spec/lib/pwn/ai/agent/solve_spec.rb +6 -4
- data/spec/lib/pwn/ai/agent/tools/browser_act_spec.rb +44 -0
- data/spec/lib/pwn/ai/agent/tools/loot_query_yubikey_spec.rb +55 -0
- data/spec/lib/pwn/ai/agent/tools/recon_run_yubikey_spec.rb +43 -0
- data/spec/lib/pwn/ai/agent/tools/workspace_tool_spec.rb +20 -0
- data/spec/lib/pwn/ai/agent/turn_finalizer_spec.rb +7 -0
- data/spec/lib/pwn/ai/anthropic_vault_enrollment_spec.rb +37 -0
- data/spec/lib/pwn/ai/cli_spec.rb +6 -9
- data/spec/lib/pwn/ai/oauth_vault_persistence_spec.rb +195 -0
- data/spec/lib/pwn/ai/open_ai_spec.rb +2 -2
- data/spec/lib/pwn/ai/open_web_ui_workspace_spec.rb +36 -0
- data/spec/lib/pwn/ai/router_spec.rb +31 -3
- data/spec/lib/pwn/ai/server_tools_spec.rb +39 -0
- data/spec/lib/pwn/backups_spec.rb +114 -0
- data/spec/lib/pwn/config_redaction_yubikey_spec.rb +28 -0
- data/spec/lib/pwn/config_yubikey_spec.rb +92 -0
- data/spec/lib/pwn/cron_spec.rb +113 -6
- data/spec/lib/pwn/migrate_spec.rb +108 -5
- data/spec/lib/pwn/migrate_yubikey_spec.rb +128 -0
- data/spec/lib/pwn/module_skills_spec.rb +8 -0
- data/spec/lib/pwn/plugins/ai_sandbox_spec.rb +230 -0
- data/spec/lib/pwn/plugins/authentication_helper_native_yubikey_spec.rb +40 -0
- data/spec/lib/pwn/plugins/authentication_helper_yubikey_spec.rb +90 -0
- data/spec/lib/pwn/plugins/capability_broker/daemon_spec.rb +1 -1
- data/spec/lib/pwn/plugins/google_workspace_spec.rb +17 -35
- data/spec/lib/pwn/plugins/log_spec.rb +7 -0
- data/spec/lib/pwn/plugins/log_yubikey_spec.rb +47 -0
- data/spec/lib/pwn/plugins/preflight_checker_native_yubikey_spec.rb +19 -0
- data/spec/lib/pwn/plugins/preflight_checker_yubikey_spec.rb +55 -0
- data/spec/lib/pwn/plugins/recon_yubikey_spec.rb +43 -0
- data/spec/lib/pwn/plugins/repl/ai_console_spec.rb +125 -78
- data/spec/lib/pwn/plugins/repl/ai_legacy_containment_spec.rb +200 -0
- data/spec/lib/pwn/plugins/repl/ai_swarm_spec.rb +30 -16
- data/spec/lib/pwn/plugins/repl/ai_vault_edit_yubikey_spec.rb +103 -0
- data/spec/lib/pwn/plugins/repl/ai_yubikey_spec.rb +48 -0
- data/spec/lib/pwn/plugins/repl/mesh_vault_persistence_spec.rb +149 -0
- data/spec/lib/pwn/plugins/repl/vault_yaml_serialization_spec.rb +11 -0
- data/spec/lib/pwn/plugins/repl/vault_yubikey_spec.rb +39 -0
- data/spec/lib/pwn/plugins/repl_spec.rb +56 -5
- data/spec/lib/pwn/plugins/transparent_browser_jev_spec.rb +46 -0
- data/spec/lib/pwn/plugins/vault_native_yubikey_spec.rb +29 -0
- data/spec/lib/pwn/plugins/vault_publication_safety_spec.rb +258 -0
- data/spec/lib/pwn/plugins/vault_store_safety_spec.rb +241 -0
- data/spec/lib/pwn/plugins/vault_yubikey_safety_spec.rb +168 -0
- data/spec/lib/pwn/plugins/vault_yubikey_spec.rb +128 -0
- data/spec/lib/pwn/plugins/vault_yubikey_vector_spec.rb +45 -0
- data/spec/lib/pwn/plugins/yubikey/fido_spec.rb +854 -0
- data/spec/lib/pwn/plugins/yubikey/hsmauth_spec.rb +235 -0
- data/spec/lib/pwn/plugins/yubikey/management_spec.rb +277 -0
- data/spec/lib/pwn/plugins/yubikey/oath_spec.rb +349 -0
- data/spec/lib/pwn/plugins/yubikey/openpgp_spec.rb +810 -0
- data/spec/lib/pwn/plugins/yubikey/otp_spec.rb +337 -0
- data/spec/lib/pwn/plugins/yubikey/piv_spec.rb +646 -0
- data/spec/lib/pwn/plugins/yubikey/protocol_secure_spec.rb +28 -0
- data/spec/lib/pwn/plugins/yubikey/protocol_spec.rb +149 -0
- data/spec/lib/pwn/plugins/yubikey/protocol_vectors_spec.rb +18 -0
- data/spec/lib/pwn/plugins/yubikey/security_domain/scp_spec.rb +256 -0
- data/spec/lib/pwn/plugins/yubikey/security_domain/support.rb +153 -0
- data/spec/lib/pwn/plugins/yubikey/security_domain_spec.rb +296 -0
- data/spec/lib/pwn/plugins/yubikey/transport_spec.rb +760 -0
- data/spec/lib/pwn/plugins/yubikey_security_spec.rb +260 -0
- data/spec/lib/pwn/plugins/yubikey_spec.rb +209 -0
- data/spec/lib/pwn/plugins/yubikey_terminal_spec.rb +147 -0
- data/spec/lib/pwn/redaction_spec.rb +13 -10
- data/spec/lib/pwn/redaction_yubikey_spec.rb +61 -0
- data/spec/lib/pwn/session_cleanup_concurrency_spec.rb +201 -0
- data/spec/lib/pwn/session_cleanup_default_spec.rb +167 -2
- data/spec/lib/pwn/session_cleanup_legacy_spec.rb +101 -0
- data/spec/lib/pwn/session_cleanup_payload_spec.rb +335 -2
- data/spec/lib/pwn/session_cleanup_recovery_spec.rb +403 -0
- data/spec/lib/pwn/sessions_spec.rb +112 -0
- data/spec/lib/pwn/setup_native_yubikey_spec.rb +66 -0
- data/spec/lib/pwn/setup_spec.rb +3 -1
- data/spec/lib/pwn/setup_vault_yubikey_spec.rb +61 -0
- data/spec/lib/pwn/setup_yubikey_spec.rb +40 -0
- data/spec/lib/pwn/www/login_yubikey_spec.rb +273 -0
- data/spec/spec_helper.rb +15 -0
- data/spec/support/gate_failure_formatter.rb +14 -0
- data/spec/support/harness/cli_faults.rb +13 -9
- data/spec/support/harness/controller_spill.rb +26 -0
- data/spec/support/harness/controller_state.rb +54 -0
- data/spec/support/harness/evaluate.rb +12 -1
- data/spec/support/harness/experiments.rb +1 -1
- data/spec/support/harness/fake_provider.rb +22 -6
- data/spec/support/harness/fault_preload.rb +3 -1
- data/spec/support/harness/mission_console.rb +54 -0
- data/spec/support/harness/mission_curses_pty.rb +41 -0
- data/spec/support/harness/mission_dispatch.rb +30 -0
- data/spec/support/harness/mission_fixture_preload.rb +24 -0
- data/spec/support/harness/mission_job_lifetime.rb +25 -0
- data/spec/support/harness/mission_legacy.rb +25 -0
- data/spec/support/harness/mission_normal_exit.rb +34 -0
- data/spec/support/harness/mission_swarm.rb +23 -0
- data/spec/support/harness/mission_unsupported.rb +29 -0
- data/spec/support/harness/package_acceptance.rb +33 -5
- data/spec/support/harness/session_acceptance.rb +4 -4
- data/spec/support/harness/swarm_acceptance.rb +1 -1
- data/spec/support/harness/swarm_control.rb +44 -0
- data/spec/support/mission_namespace.rb +14 -0
- data/spec/support/mission_pipe_queue.rb +33 -0
- data/third_party/pwn_rdoc.jsonl +599 -15
- metadata +140 -17
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# PWN managed default: session_cleanup
|
|
3
|
+
# PWN managed default: session_cleanup v10
|
|
4
4
|
# No provider calls or broad store GC. Unsummarized candidates are backed up first.
|
|
5
5
|
require 'pwn'
|
|
6
6
|
require 'json'
|
|
@@ -12,7 +12,1046 @@ require 'securerandom'
|
|
|
12
12
|
module PWN
|
|
13
13
|
# Provenance-based session cleanup, shared by seeded cron payloads.
|
|
14
14
|
module SessionCleanup
|
|
15
|
-
VERSION =
|
|
15
|
+
VERSION = 10
|
|
16
|
+
|
|
17
|
+
# Logical categories deliberately preserve existing literal evidence paths.
|
|
18
|
+
STORAGE = {
|
|
19
|
+
'sessions' => %w[sessions],
|
|
20
|
+
'learning' => %w[learning.jsonl learning lessons.json memory.json memory.archive.json memory.idx mistakes.json policy policy.json policy_traj.jsonl curriculum curriculum_kpi.jsonl extrospection.json finetune preferences.jsonl reward_sentinel.json skill_review.jsonl operator-learning-review.jsonl],
|
|
21
|
+
'artifacts' => %w[artifacts exports findings.jsonl engagements corpora binws redb ghidra sandbox_snapshots],
|
|
22
|
+
'state' => %w[swarm cron agents.yml open_goal.json jobs missions runs metrics metrics.json capabilities.json],
|
|
23
|
+
'logs' => %w[logs], 'cache' => %w[cache], 'tmp' => %w[workspaces/runtime],
|
|
24
|
+
'maintenance' => %w[maintenance session_cleanup_archive],
|
|
25
|
+
'backups' => %w[backup]
|
|
26
|
+
}.freeze
|
|
27
|
+
POLICY = { 'version' => 1, 'batch_size' => 100, 'scan_limit' => 100_000, 'work_items' => 100_000, 'work_seconds' => 30, 'work_bytes' => 67_108_864,
|
|
28
|
+
'retention_days' => { 'sessions' => 0, 'learning' => nil, 'artifacts' => nil, 'state' => nil,
|
|
29
|
+
'logs' => 30, 'cache' => 7, 'tmp' => 1, 'maintenance' => 90 },
|
|
30
|
+
'budget_bytes' => { 'logs' => 268_435_456, 'cache' => 536_870_912, 'tmp' => 268_435_456, 'maintenance' => 1_073_741_824 } }.freeze
|
|
31
|
+
|
|
32
|
+
# Versioned additive setup: no old path moves or permission rewrites.
|
|
33
|
+
def self.install(root: File.join(Dir.home, '.pwn'))
|
|
34
|
+
directory = private_directory(File.join(root, 'maintenance'))
|
|
35
|
+
File.open(File.join(directory, '.setup.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
36
|
+
lock.flock(File::LOCK_EX)
|
|
37
|
+
path = File.join(directory, 'policy.json')
|
|
38
|
+
raise IOError, 'Symlinked maintenance policy' if File.symlink?(path)
|
|
39
|
+
|
|
40
|
+
publish(path, JSON.pretty_generate(POLICY)) unless File.exist?(path)
|
|
41
|
+
private_directory(File.join(root, 'cache')) unless File.exist?(File.join(root, 'cache'))
|
|
42
|
+
private_directory(File.join(root, 'workspaces', 'runtime')) unless File.exist?(File.join(root, 'workspaces', 'runtime'))
|
|
43
|
+
layout = File.join(directory, 'layout.json')
|
|
44
|
+
raise IOError, 'Unsafe maintenance inventory' if File.symlink?(layout)
|
|
45
|
+
|
|
46
|
+
previous = File.file?(layout) ? JSON.parse(File.binread(layout)) : nil
|
|
47
|
+
raise IOError, 'Unknown maintenance inventory version' if previous && ![1, 2, 3].include?(previous['version'])
|
|
48
|
+
|
|
49
|
+
publish(layout, JSON.generate(version: 3, categories: STORAGE)) unless previous && previous['version'] == 3
|
|
50
|
+
end
|
|
51
|
+
{ version: 3, root: directory }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Startup performs only fixed-path checks and one coalesced request write.
|
|
55
|
+
# The existing cron worker owns all scanning, migration and deletion.
|
|
56
|
+
def self.request_background(root: File.join(Dir.home, '.pwn'))
|
|
57
|
+
directory = File.join(root, 'maintenance')
|
|
58
|
+
return { state: 'not_installed' } unless File.directory?(directory)
|
|
59
|
+
raise IOError, 'Unsafe maintenance directory' unless File.realpath(directory) == directory
|
|
60
|
+
|
|
61
|
+
File.open(File.join(directory, '.schedule.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
62
|
+
return { state: 'coalesced' } unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
63
|
+
|
|
64
|
+
path = File.join(directory, 'background.json')
|
|
65
|
+
raise IOError, 'Unsafe background request' if File.symlink?(path)
|
|
66
|
+
|
|
67
|
+
if File.file?(path)
|
|
68
|
+
prior = JSON.parse(File.binread(path))
|
|
69
|
+
return { state: 'coalesced' } if prior['state'] == 'pending' || prior.fetch('requested_at', 0) > Time.now.to_i - 86_400
|
|
70
|
+
end
|
|
71
|
+
publish(path, JSON.generate(version: 1, state: 'pending', requested_at: Time.now.to_i))
|
|
72
|
+
{ state: 'queued', path: path }
|
|
73
|
+
end
|
|
74
|
+
rescue StandardError => e
|
|
75
|
+
{ state: 'unhealthy', error: e.class.name }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def self.background_request(root: File.join(Dir.home, '.pwn'))
|
|
79
|
+
directory = File.join(root, 'maintenance')
|
|
80
|
+
path = File.join(directory, 'background.json')
|
|
81
|
+
return unless File.file?(path)
|
|
82
|
+
raise IOError, 'Unsafe background request' unless File.realpath(path) == path
|
|
83
|
+
|
|
84
|
+
File.open(File.join(directory, '.schedule.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
85
|
+
next unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
86
|
+
|
|
87
|
+
row = JSON.parse(File.binread(path))
|
|
88
|
+
next unless row['version'] == 1 && row['state'] == 'pending'
|
|
89
|
+
|
|
90
|
+
result = yield
|
|
91
|
+
next unless result
|
|
92
|
+
|
|
93
|
+
row['state'] = result[:status] == 'success' ? 'completed' : 'interrupted'
|
|
94
|
+
row['completed_at'] = Time.now.to_i
|
|
95
|
+
publish(path, JSON.generate(row))
|
|
96
|
+
result
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def self.policy
|
|
101
|
+
path = File.join(Dir.home, '.pwn', 'maintenance', 'policy.json')
|
|
102
|
+
return Marshal.load(Marshal.dump(POLICY)) unless File.exist?(path) || File.symlink?(path)
|
|
103
|
+
|
|
104
|
+
raise IOError, 'Unsafe maintenance policy' if File.symlink?(path)
|
|
105
|
+
|
|
106
|
+
value = JSON.parse(File.binread(path))
|
|
107
|
+
raise ArgumentError, 'Unknown maintenance policy' unless value.is_a?(Hash) && value['version'] == 1 && (value.keys - POLICY.keys).empty?
|
|
108
|
+
|
|
109
|
+
result = Marshal.load(Marshal.dump(POLICY)).merge(value)
|
|
110
|
+
%w[retention_days budget_bytes].each do |key|
|
|
111
|
+
raise ArgumentError, "Invalid #{key}" unless result[key].is_a?(Hash) && (result[key].keys - STORAGE.keys).empty? && result[key].values.all? { |n| n.nil? || (n.is_a?(Integer) && n >= 0) }
|
|
112
|
+
|
|
113
|
+
result[key] = POLICY[key].merge(result[key])
|
|
114
|
+
end
|
|
115
|
+
raise ArgumentError, 'Invalid read budget' unless result['work_bytes'].is_a?(Integer) && result['work_bytes'].positive?
|
|
116
|
+
raise ArgumentError, 'Invalid maintenance bounds' unless %w[batch_size scan_limit work_items work_seconds].all? { |key| result[key].is_a?(Integer) && result[key].between?(1, 100_000) }
|
|
117
|
+
|
|
118
|
+
result
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# No mkdir, locks, provider access, metadata rewrites or repair in status.
|
|
122
|
+
def self.status
|
|
123
|
+
config = policy
|
|
124
|
+
root = File.join(Dir.home, '.pwn')
|
|
125
|
+
errors = []
|
|
126
|
+
remaining = config['scan_limit']
|
|
127
|
+
categories = STORAGE.to_h do |category, names|
|
|
128
|
+
paths = names.map { |name| name == 'sessions' ? PWN::Sessions::SESSIONS_DIR : File.join(root, name) }
|
|
129
|
+
row = { paths: paths, files: 0, bytes: 0, unique_bytes: 0, symlinks: 0, retention_days: config['retention_days'][category], budget_bytes: config['budget_bytes'][category] }
|
|
130
|
+
inodes = {}
|
|
131
|
+
pending = paths.dup
|
|
132
|
+
until pending.empty? || remaining.zero? || stopping?
|
|
133
|
+
begin
|
|
134
|
+
entry = pending.pop
|
|
135
|
+
if entry.is_a?(Array)
|
|
136
|
+
begin
|
|
137
|
+
path = File.join(entry[0], entry[1].next)
|
|
138
|
+
pending << entry
|
|
139
|
+
rescue StopIteration
|
|
140
|
+
next
|
|
141
|
+
end
|
|
142
|
+
else
|
|
143
|
+
path = entry
|
|
144
|
+
end
|
|
145
|
+
next unless File.exist?(path) || File.symlink?(path)
|
|
146
|
+
|
|
147
|
+
remaining -= 1
|
|
148
|
+
stat = File.lstat(path)
|
|
149
|
+
if stat.symlink?
|
|
150
|
+
row[:symlinks] += 1
|
|
151
|
+
elsif stat.directory?
|
|
152
|
+
pending << [path, Dir.each_child(path)]
|
|
153
|
+
elsif stat.file?
|
|
154
|
+
row[:files] += 1
|
|
155
|
+
row[:bytes] += stat.size
|
|
156
|
+
key = [stat.dev, stat.ino]
|
|
157
|
+
row[:unique_bytes] += stat.size unless inodes[key]
|
|
158
|
+
inodes[key] = true
|
|
159
|
+
end
|
|
160
|
+
rescue SystemCallError => e
|
|
161
|
+
errors << { path: path, error: e.class.name }
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
row[:incomplete] = !pending.empty?
|
|
165
|
+
row[:over_budget] = row[:budget_bytes] && row[:bytes] > row[:budget_bytes]
|
|
166
|
+
row[:policy_mode] = %w[learning artifacts state backups].include?(category) ? 'protected_evidence_no_automatic_deletion' : 'verified_disposable_only'
|
|
167
|
+
[category, row]
|
|
168
|
+
end
|
|
169
|
+
transactions = Dir.glob(File.join(root, 'maintenance', 'runs', '*.json')).last(config['batch_size']).map do |path|
|
|
170
|
+
raise IOError, 'Unsafe transaction ledger' unless File.realpath(path) == path
|
|
171
|
+
|
|
172
|
+
row = JSON.parse(admitted_bytes(path))
|
|
173
|
+
{ ledger: path, state: row['state'], recovery: %w[prepared interrupted].include?(row['state']) ? 'inspect_child_ledgers_before_retry' : nil }
|
|
174
|
+
rescue StandardError => e
|
|
175
|
+
{ ledger: path, state: 'unknown', error: e.class.name }
|
|
176
|
+
end
|
|
177
|
+
{ version: VERSION, operation: 'status', categories: categories, errors: errors, transactions: transactions,
|
|
178
|
+
policy: config, reasons: ['Unknown files and immutable evidence are retained; budgets do not override evidence or leases.',
|
|
179
|
+
'Historical backup expiry requires explicit retention policy and verified current recovery coverage; pinned evidence remains.'] }
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Explicit legacy expiry, not broad GC. No payload quarantine or new per-file
|
|
183
|
+
# leases: old producers do not honor those. Permanent maintenance locking,
|
|
184
|
+
# closed-stream markers, process descriptors and exact byte identities fence
|
|
185
|
+
# adoption. References are conservatively matched even in retained transcripts.
|
|
186
|
+
def self.expire_legacy(dry_run: true, manifest: nil)
|
|
187
|
+
require 'zlib'
|
|
188
|
+
root = File.join(Dir.home, '.pwn')
|
|
189
|
+
raise IOError, 'Unsafe root' unless File.realpath(root) == root
|
|
190
|
+
raise ArgumentError, 'New external metadata manifest required' if !dry_run && (manifest.nil? || File.expand_path(manifest).start_with?("#{root}/") || File.exist?(manifest))
|
|
191
|
+
|
|
192
|
+
result = { removed: [], retained: [], candidates: [] }
|
|
193
|
+
File.open(File.join(root, 'sessions', '.maintenance.lock'), File::RDWR | File::NOFOLLOW) do |lock|
|
|
194
|
+
raise IOError, 'Maintenance active' unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
195
|
+
|
|
196
|
+
candidates = legacy_candidates(root) + legacy_recovery_candidates(root) + legacy_cache_candidates(root)
|
|
197
|
+
pins = legacy_references(root, candidates)
|
|
198
|
+
opened = legacy_open_files
|
|
199
|
+
candidates.each do |row|
|
|
200
|
+
reason = pins[row[:path]] ? 'evidence_reference' : nil
|
|
201
|
+
reason ||= 'open_descriptor' if opened.include?([row[:dev], row[:ino]])
|
|
202
|
+
(reason ? result[:retained] : result[:candidates]) << row.merge(reason: reason, references: pins[row[:path]])
|
|
203
|
+
end
|
|
204
|
+
return result if dry_run
|
|
205
|
+
|
|
206
|
+
# Publish metadata before irreversible expiry. An interrupted manifest is
|
|
207
|
+
# an intent ledger, never a claim that every planned removal completed.
|
|
208
|
+
result[:state] = 'prepared'
|
|
209
|
+
legacy_manifest(manifest, result)
|
|
210
|
+
raise IOError, 'Evidence references changed' unless pins == legacy_references(root, candidates)
|
|
211
|
+
|
|
212
|
+
result[:candidates].each do |row|
|
|
213
|
+
path = row[:path]
|
|
214
|
+
File.open(path, File::RDONLY | File::NOFOLLOW) do |file|
|
|
215
|
+
raise IOError, 'Candidate locked' unless file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
216
|
+
|
|
217
|
+
lease = File.open("#{path}.lease", File::RDWR | File::NOFOLLOW) if File.exist?("#{path}.lease") || File.symlink?("#{path}.lease") # rubocop:disable Style/FileOpen -- closed in ensure while optional producer lease is held
|
|
218
|
+
raise IOError, 'Producer lease active' if lease && !lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
219
|
+
raise IOError, 'Candidate changed' unless legacy_identity(path) == row.except(:category, :reason, :references, :proofs)
|
|
220
|
+
|
|
221
|
+
Array(row[:proofs]).each do |proof|
|
|
222
|
+
raise IOError, 'Recovery proof changed' unless legacy_identity(proof[:path]) == proof
|
|
223
|
+
end
|
|
224
|
+
raise IOError, 'Candidate became active' if legacy_open_files(except_fd: file.fileno).include?([row[:dev], row[:ino]])
|
|
225
|
+
|
|
226
|
+
File.unlink(path)
|
|
227
|
+
result[:removed] << row
|
|
228
|
+
ensure
|
|
229
|
+
lease&.close
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
result[:state] = 'completed'
|
|
233
|
+
legacy_manifest(manifest, result)
|
|
234
|
+
end
|
|
235
|
+
result
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def self.legacy_manifest(path, result)
|
|
239
|
+
File.open(path, File::WRONLY | File::CREAT | File::TRUNC | File::NOFOLLOW, 0o600) do |file|
|
|
240
|
+
gzip = Zlib::GzipWriter.new(file)
|
|
241
|
+
gzip.write(JSON.generate(result))
|
|
242
|
+
gzip.finish
|
|
243
|
+
file.flush
|
|
244
|
+
file.fsync
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def self.legacy_identity(path)
|
|
249
|
+
stat = File.lstat(path)
|
|
250
|
+
raise IOError, 'Unsafe legacy file' unless stat.file? && stat.uid == File.stat(File.join(Dir.home, '.pwn')).uid && stat.nlink == 1 && File.realpath(path) == path
|
|
251
|
+
|
|
252
|
+
{ path: path, dev: stat.dev, ino: stat.ino, size: stat.size, mtime: stat.mtime.to_r.to_s,
|
|
253
|
+
ctime: stat.ctime.to_r.to_s, sha256: Digest::SHA256.file(path).hexdigest }
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def self.legacy_candidates(root)
|
|
257
|
+
days = policy['retention_days']['logs']
|
|
258
|
+
return [] unless days.is_a?(Numeric) && days.positive?
|
|
259
|
+
|
|
260
|
+
Dir.glob(File.join(root, 'logs', '*.log')).filter_map do |path|
|
|
261
|
+
next unless File.basename(path).match?(/\Apwn-ai-DEBUG-[a-zA-Z0-9_-]+-R[1-9]\d*\.log\z/)
|
|
262
|
+
|
|
263
|
+
row = legacy_identity(path)
|
|
264
|
+
next unless File.lstat(path).mtime < Time.now - (days * 86_400)
|
|
265
|
+
|
|
266
|
+
bytes = File.binread(path)
|
|
267
|
+
next unless bytes.include?('debug session stop') || bytes.include?('footer iter=')
|
|
268
|
+
|
|
269
|
+
row.merge(category: 'logs')
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
# Historical learning/reward recovery is explicitly bounded to at least
|
|
274
|
+
# three parseable copies per exact producer family, plus all reference pins.
|
|
275
|
+
# A thirty-day floor excludes files an older daily producer might still use.
|
|
276
|
+
def self.legacy_recovery_candidates(root)
|
|
277
|
+
families = { 'learning.jsonl' => /\Alearning\.jsonl\.bak-lean-\d{8}\z/,
|
|
278
|
+
'preferences.jsonl' => /\Apreferences\.jsonl\.bak-p15-\d{14}\z/ }
|
|
279
|
+
families.flat_map do |source, pattern|
|
|
280
|
+
current = File.join(root, source)
|
|
281
|
+
next [] unless File.file?(current)
|
|
282
|
+
|
|
283
|
+
legacy_identity(current)
|
|
284
|
+
File.foreach(current) { |line| JSON.parse(line) unless line.strip.empty? }
|
|
285
|
+
copies = Dir.children(root).grep(pattern).sort.map { |name| File.join(root, name) }
|
|
286
|
+
valid = copies.select do |path|
|
|
287
|
+
legacy_identity(path)
|
|
288
|
+
File.foreach(path) { |line| JSON.parse(line) unless line.strip.empty? }
|
|
289
|
+
File.size(path).positive?
|
|
290
|
+
rescue JSON::ParserError
|
|
291
|
+
false
|
|
292
|
+
end
|
|
293
|
+
(valid - valid.last(3)).filter_map do |path|
|
|
294
|
+
next unless File.mtime(path) < Time.now - (30 * 86_400)
|
|
295
|
+
|
|
296
|
+
legacy_identity(path).merge(category: 'root_recovery', proofs: valid.last(3).map { |copy| legacy_identity(copy) })
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# BinaryParser persists exactly these cache bytes to ArtifactRegistry. Missing
|
|
302
|
+
# source binaries prevent recomputation; require a surviving byte-exact copy
|
|
303
|
+
# instead, never classify arbitrary decompiler stdout as reproducible.
|
|
304
|
+
def self.legacy_cache_candidates(root)
|
|
305
|
+
days = policy['retention_days']['cache']
|
|
306
|
+
return [] unless days.is_a?(Numeric) && days.positive?
|
|
307
|
+
|
|
308
|
+
copies = Dir.glob(File.join(root, 'artifacts', '**', '*')).select { |path| File.file?(path) && !File.symlink?(path) }
|
|
309
|
+
Dir.glob(File.join(root, 'cache', 'triage', '*.json')).filter_map do |path|
|
|
310
|
+
next unless File.basename(path).match?(/\A[a-f0-9]{64}\.json\z/) && File.mtime(path) < Time.now - (days * 86_400)
|
|
311
|
+
|
|
312
|
+
row = legacy_identity(path)
|
|
313
|
+
body = JSON.parse(File.binread(path))
|
|
314
|
+
next unless body['sha256'] == File.basename(path, '.json') && body['path'].is_a?(String)
|
|
315
|
+
|
|
316
|
+
copy = copies.find { |candidate| File.size(candidate) == row[:size] && Digest::SHA256.file(candidate).hexdigest == row[:sha256] }
|
|
317
|
+
next unless copy
|
|
318
|
+
|
|
319
|
+
row.merge(category: 'cache', proofs: [legacy_identity(copy)])
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def self.legacy_references(root, candidates)
|
|
324
|
+
names = candidates.to_h { |row| [File.basename(row[:path]), row[:path]] }
|
|
325
|
+
return {} if names.empty?
|
|
326
|
+
|
|
327
|
+
pattern = Regexp.union(names.keys)
|
|
328
|
+
references = {}
|
|
329
|
+
# Recovery generations are historical copies, not live evidence pins.
|
|
330
|
+
# Candidate payloads cannot pin themselves. Everything else is scanned
|
|
331
|
+
# without interpreting or displaying credential/transcript contents.
|
|
332
|
+
Dir.glob(File.join(root, '**', '*'), File::FNM_DOTMATCH).sort.each do |path|
|
|
333
|
+
relative = path.delete_prefix("#{root}/")
|
|
334
|
+
next if %w[backup logs cache].include?(relative.split('/').first)
|
|
335
|
+
next if names.value?(path) || !File.file?(path)
|
|
336
|
+
raise IOError, 'Symlink in reference scope' if File.symlink?(path)
|
|
337
|
+
|
|
338
|
+
File.binread(path).scan(pattern).uniq.each do |name|
|
|
339
|
+
(references[names.fetch(name)] ||= []) << path
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
references
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def self.legacy_open_files(except_fd: nil)
|
|
346
|
+
raise IOError, 'Process inventory unavailable' unless File.directory?('/proc/self/fd')
|
|
347
|
+
|
|
348
|
+
opened = []
|
|
349
|
+
Dir.glob('/proc/[0-9]*').each do |process|
|
|
350
|
+
next unless Process.uid.zero? || File.stat(process).uid == Process.uid
|
|
351
|
+
|
|
352
|
+
Dir.children("#{process}/fd").each do |fd|
|
|
353
|
+
next if File.basename(process).to_i == Process.pid && fd.to_i == except_fd
|
|
354
|
+
|
|
355
|
+
stat = File.stat("#{process}/fd/#{fd}")
|
|
356
|
+
opened << [stat.dev, stat.ino] if stat.file?
|
|
357
|
+
rescue Errno::ENOENT, Errno::ESRCH
|
|
358
|
+
next
|
|
359
|
+
end
|
|
360
|
+
rescue Errno::ENOENT, Errno::ESRCH
|
|
361
|
+
next
|
|
362
|
+
end
|
|
363
|
+
opened.uniq
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
# A producer registers only reproducible/disposable bytes, never evidence.
|
|
367
|
+
# Writers hold the permanent .lease exclusively while changing a file;
|
|
368
|
+
# consumers hold it shared for the whole read/use lifetime.
|
|
369
|
+
def self.register_disposable(path:, category:, expires_at: nil)
|
|
370
|
+
raise ArgumentError, 'Not a disposable category' unless %w[cache tmp logs].include?(category)
|
|
371
|
+
|
|
372
|
+
root = disposable_directory(category)
|
|
373
|
+
path = File.expand_path(path)
|
|
374
|
+
raise IOError, 'Disposable path escapes its category' unless path.start_with?("#{root}/") && File.realpath(path) == path
|
|
375
|
+
raise IOError, 'Disposable must be an owned regular file' unless File.file?(path) && File.stat(path).uid == Process.uid
|
|
376
|
+
|
|
377
|
+
days = policy['retention_days'][category]
|
|
378
|
+
raise ArgumentError, 'Retention disabled for category' if days.nil?
|
|
379
|
+
|
|
380
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
381
|
+
raise IOError, 'Disposable is active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
382
|
+
|
|
383
|
+
record_disposable(path: path, category: category, expires_at: expires_at || (Time.now.to_i + (days * 86_400)))
|
|
384
|
+
end
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
# Caller holds the permanent exclusive producer lease through publication.
|
|
388
|
+
def self.record_disposable(path:, category:, expires_at:)
|
|
389
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'disposable'))
|
|
390
|
+
record = { version: 1, path: path, category: category, expires_at: expires_at,
|
|
391
|
+
sha256: Digest::SHA256.file(path).hexdigest, bytes: File.size(path), inode: File.stat(path).ino }
|
|
392
|
+
publish(File.join(directory, "#{Digest::SHA256.hexdigest(path)}.json"), JSON.generate(record))
|
|
393
|
+
record
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
# Cache consumers use the same permanent inode as publication and eviction.
|
|
397
|
+
def self.cache(path:, bytes: nil)
|
|
398
|
+
root = File.join(Dir.home, '.pwn', 'cache')
|
|
399
|
+
raise IOError, 'Unsafe cache path' unless File.expand_path(path) == path && path.start_with?("#{root}/")
|
|
400
|
+
|
|
401
|
+
private_directory(File.dirname(path)) unless File.directory?(File.dirname(path))
|
|
402
|
+
raise IOError, 'Unsafe cache directory' unless File.realpath(File.dirname(path)) == File.dirname(path)
|
|
403
|
+
|
|
404
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
405
|
+
lease.flock(bytes ? File::LOCK_EX : File::LOCK_SH)
|
|
406
|
+
if bytes
|
|
407
|
+
publish(path, bytes)
|
|
408
|
+
days = policy['retention_days']['cache']
|
|
409
|
+
if days
|
|
410
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'disposable'))
|
|
411
|
+
row = { version: 1, path: path, category: 'cache', expires_at: Time.now.to_i + (days * 86_400),
|
|
412
|
+
sha256: Digest::SHA256.file(path).hexdigest, bytes: File.size(path), inode: File.stat(path).ino }
|
|
413
|
+
publish(File.join(directory, "#{Digest::SHA256.hexdigest(path)}.json"), JSON.generate(row))
|
|
414
|
+
end
|
|
415
|
+
bytes
|
|
416
|
+
elsif File.file?(path) && !File.symlink?(path)
|
|
417
|
+
File.binread(path)
|
|
418
|
+
end
|
|
419
|
+
end
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
# Known disposable input producer. The permanent lease covers publication,
|
|
423
|
+
# consumption and unlink; a killed consumer leaves a registered exact file.
|
|
424
|
+
def self.temporary(bytes:)
|
|
425
|
+
directory = private_directory(disposable_directory('tmp'))
|
|
426
|
+
path = File.join(directory, "input-#{SecureRandom.hex(12)}.txt")
|
|
427
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
428
|
+
lease.flock(File::LOCK_EX)
|
|
429
|
+
begin
|
|
430
|
+
publish(path, bytes)
|
|
431
|
+
days = policy['retention_days']['tmp']
|
|
432
|
+
record_disposable(path: path, category: 'tmp', expires_at: Time.now.to_i + (days * 86_400)) if days
|
|
433
|
+
yield path
|
|
434
|
+
ensure
|
|
435
|
+
File.unlink(path) if File.file?(path) && !File.symlink?(path)
|
|
436
|
+
end
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
# Rotate ordinary diagnostics under the same lease as append and eviction.
|
|
441
|
+
# Request debug streams rotate at request boundaries and hold a lifetime lease.
|
|
442
|
+
def self.append_log(path:, bytes:, rotate_bytes: 1_024_000)
|
|
443
|
+
root = File.join(Dir.home, '.pwn', 'logs')
|
|
444
|
+
raise IOError, 'Unsafe log path' unless File.expand_path(path) == path && path.start_with?("#{root}/")
|
|
445
|
+
|
|
446
|
+
private_directory(File.dirname(path)) unless File.directory?(File.dirname(path))
|
|
447
|
+
raise IOError, 'Unsafe log directory' unless File.realpath(File.dirname(path)) == File.dirname(path)
|
|
448
|
+
|
|
449
|
+
File.open("#{path}.lease", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
450
|
+
lease.flock(File::LOCK_EX)
|
|
451
|
+
raise IOError, 'Symlinked log' if File.symlink?(path)
|
|
452
|
+
|
|
453
|
+
if File.file?(path) && File.size(path) >= rotate_bytes
|
|
454
|
+
rotated = "#{path}.#{SecureRandom.hex(12)}"
|
|
455
|
+
File.rename(path, rotated)
|
|
456
|
+
register_disposable(path: rotated, category: 'logs') unless policy['retention_days']['logs'].nil?
|
|
457
|
+
end
|
|
458
|
+
File.open(path, File::WRONLY | File::CREAT | File::APPEND | File::NOFOLLOW, 0o600) do |file|
|
|
459
|
+
file.write(bytes)
|
|
460
|
+
file.flush
|
|
461
|
+
file.fsync
|
|
462
|
+
end
|
|
463
|
+
days = policy['retention_days']['logs']
|
|
464
|
+
record_disposable(path: path, category: 'logs', expires_at: Time.now.to_i + (days * 86_400)) if days
|
|
465
|
+
end
|
|
466
|
+
path
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# Disposable category tmp lives in the runtime workspace, not ~/.pwn/tmp.
|
|
470
|
+
def self.disposable_directory(category)
|
|
471
|
+
raise ArgumentError, 'Not a disposable category' unless %w[cache tmp logs].include?(category)
|
|
472
|
+
|
|
473
|
+
base = File.join(Dir.home, '.pwn')
|
|
474
|
+
category == 'tmp' ? File.join(base, 'workspaces', 'runtime') : File.join(base, category)
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
def self.private_directory(path)
|
|
478
|
+
ancestor = File.expand_path(path)
|
|
479
|
+
loop do
|
|
480
|
+
raise IOError, 'Symlinked maintenance directory' if File.symlink?(ancestor)
|
|
481
|
+
break if ancestor == File.dirname(ancestor)
|
|
482
|
+
|
|
483
|
+
ancestor = File.dirname(ancestor)
|
|
484
|
+
end
|
|
485
|
+
FileUtils.mkdir_p(path, mode: 0o700)
|
|
486
|
+
stat = File.stat(path)
|
|
487
|
+
raise IOError, 'Unsafe maintenance directory permissions' unless stat.uid == Process.uid && stat.mode.nobits?(0o077)
|
|
488
|
+
|
|
489
|
+
path
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
# Exact-size reads cannot grow without bound if an uncooperative writer appends.
|
|
493
|
+
def self.exact_bytes(path, size)
|
|
494
|
+
bytes = File.binread(path, size + 1)
|
|
495
|
+
raise IOError, 'File size changed' unless bytes.bytesize == size
|
|
496
|
+
|
|
497
|
+
bytes
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def self.admitted_bytes(path)
|
|
501
|
+
size = File.size(path)
|
|
502
|
+
raise IOError, 'Maintenance read budget exhausted' unless admit?(size + 1)
|
|
503
|
+
|
|
504
|
+
exact_bytes(path, size)
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
def self.disposable_reason(row, pressure: false, verify: true)
|
|
508
|
+
return 'unknown_schema' unless row.is_a?(Hash) && row['version'] == 1 && %w[cache tmp logs].include?(row['category']) && row['expires_at'].is_a?(Integer)
|
|
509
|
+
|
|
510
|
+
path = row['path']
|
|
511
|
+
root = disposable_directory(row['category'])
|
|
512
|
+
return 'unsafe_path' unless path.is_a?(String) && path.start_with?("#{root}/") && File.expand_path(path) == path
|
|
513
|
+
return 'missing' unless File.exist?(path)
|
|
514
|
+
return 'unsafe_path' unless File.realpath(path) == path && File.file?(path) && File.stat(path).uid == Process.uid
|
|
515
|
+
return 'changed' unless File.stat(path).ino == row['inode'] && File.size(path) == row['bytes']
|
|
516
|
+
return 'changed' if verify && Digest::SHA256.hexdigest(exact_bytes(path, row['bytes'])) != row['sha256']
|
|
517
|
+
return 'retention_disabled' if policy['retention_days'][row['category']].nil?
|
|
518
|
+
return 'not_expired' if !pressure && row['expires_at'] > Time.now.to_i
|
|
519
|
+
return 'missing_lease' unless File.file?("#{path}.lease") && !File.symlink?("#{path}.lease")
|
|
520
|
+
|
|
521
|
+
nil
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
def self.clean_disposables(dry_run: true, stop: nil)
|
|
525
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
526
|
+
result = { operation: 'routine', state: 'planned', dry_run: dry_run, eligible: [], deleted: [], retained: [], records: [] }
|
|
527
|
+
root = File.join(Dir.home, '.pwn', 'maintenance')
|
|
528
|
+
usage = status[:categories]
|
|
529
|
+
# A lexical round-robin cursor is only scheduling state, never deletion
|
|
530
|
+
# authority. Manifests stay put, so concurrent re-registration cannot
|
|
531
|
+
# be removed or overwritten by cleanup completion.
|
|
532
|
+
cursor_path = File.join(root, 'disposable.cursor')
|
|
533
|
+
cursor = File.file?(cursor_path) && !File.symlink?(cursor_path) ? File.binread(cursor_path, 70) : ''
|
|
534
|
+
cursor = '' unless cursor.match?(/\A[a-f0-9]{64}\.json\z/)
|
|
535
|
+
files = Dir.glob(File.join(root, 'disposable', '*.json'))
|
|
536
|
+
after, before = files.partition { |file| File.basename(file) > cursor }
|
|
537
|
+
files = after + before
|
|
538
|
+
result[:incomplete] = true if files.length > policy['scan_limit']
|
|
539
|
+
files = files.first(policy['scan_limit'])
|
|
540
|
+
files.each do |manifest|
|
|
541
|
+
if !admit? || stop&.call || result[:deleted].length >= policy['batch_size']
|
|
542
|
+
result[:incomplete] = true
|
|
543
|
+
break
|
|
544
|
+
end
|
|
545
|
+
|
|
546
|
+
publish(cursor_path, File.basename(manifest)) unless dry_run
|
|
547
|
+
raise IOError, 'Unsafe disposable manifest' unless File.realpath(manifest) == manifest
|
|
548
|
+
|
|
549
|
+
manifest_size = File.size(manifest)
|
|
550
|
+
break unless admit?((manifest_size + 1) * 2)
|
|
551
|
+
|
|
552
|
+
manifest_bytes = exact_bytes(manifest, manifest_size)
|
|
553
|
+
row = JSON.parse(manifest_bytes)
|
|
554
|
+
category_usage = usage[row['category']] || {}
|
|
555
|
+
budget = category_usage[:budget_bytes]
|
|
556
|
+
pressure = budget && category_usage[:bytes] > budget
|
|
557
|
+
reason = disposable_reason(row, pressure: pressure, verify: false)
|
|
558
|
+
if reason
|
|
559
|
+
result[:retained] << { manifest: manifest, reason: reason }
|
|
560
|
+
next
|
|
561
|
+
end
|
|
562
|
+
path = row['path']
|
|
563
|
+
File.open("#{path}.lease", File::RDWR | File::NOFOLLOW) do |lease|
|
|
564
|
+
unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
565
|
+
result[:retained] << { path: path, reason: 'active' }
|
|
566
|
+
next
|
|
567
|
+
end
|
|
568
|
+
if exact_bytes(manifest, manifest_size) != manifest_bytes
|
|
569
|
+
result[:retained] << { path: path, reason: 'registration_changed' }
|
|
570
|
+
next
|
|
571
|
+
end
|
|
572
|
+
# Reserve all payload reads, backup write and verification before commit.
|
|
573
|
+
unless admit?((row['bytes'] + 1) * (dry_run ? 1 : 5))
|
|
574
|
+
result[:retained] << { path: path, reason: 'work_budget' }
|
|
575
|
+
next
|
|
576
|
+
end
|
|
577
|
+
if (reason = disposable_reason(row, pressure: pressure))
|
|
578
|
+
result[:retained] << { path: path, reason: reason }
|
|
579
|
+
next
|
|
580
|
+
end
|
|
581
|
+
result[:eligible] << path
|
|
582
|
+
category_usage[:bytes] -= row['bytes']
|
|
583
|
+
next if dry_run
|
|
584
|
+
|
|
585
|
+
unless result[:ledger]
|
|
586
|
+
directory = PWN::Backups.directory(label: 'disposable')
|
|
587
|
+
result[:ledger] = File.join(directory, 'ledger.json')
|
|
588
|
+
end
|
|
589
|
+
backup_path = File.join(File.dirname(result[:ledger]), "#{Digest::SHA256.hexdigest(path)}.bin")
|
|
590
|
+
publish(backup_path, exact_bytes(path, row['bytes']))
|
|
591
|
+
raise IOError, 'Disposable backup mismatch' unless Digest::SHA256.hexdigest(exact_bytes(backup_path, row['bytes'])) == row['sha256']
|
|
592
|
+
|
|
593
|
+
record = row.merge('backup' => backup_path, 'state' => 'prepared')
|
|
594
|
+
result[:records] << record
|
|
595
|
+
result[:state] = 'prepared'
|
|
596
|
+
publish(result[:ledger], JSON.pretty_generate(result))
|
|
597
|
+
raise IOError, 'Disposable changed after backup' if disposable_reason(row, pressure: pressure)
|
|
598
|
+
|
|
599
|
+
File.unlink(path)
|
|
600
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
601
|
+
result[:deleted] << path
|
|
602
|
+
record['state'] = 'committed'
|
|
603
|
+
publish(result[:ledger], JSON.pretty_generate(result))
|
|
604
|
+
# Keep permanent lease inode and manifest for audit/recovery.
|
|
605
|
+
end
|
|
606
|
+
rescue StandardError => e
|
|
607
|
+
result[:retained] << { manifest: manifest, reason: e.class.name }
|
|
608
|
+
end
|
|
609
|
+
unless dry_run
|
|
610
|
+
result[:state] = stopping? || stop&.call ? 'stopped' : 'committed'
|
|
611
|
+
result[:state] = 'interrupted' if result[:records].any? { |row| row['state'] == 'prepared' }
|
|
612
|
+
end
|
|
613
|
+
publish(result[:ledger], JSON.pretty_generate(result)) if result[:ledger]
|
|
614
|
+
result
|
|
615
|
+
end
|
|
616
|
+
end
|
|
617
|
+
|
|
618
|
+
# Only disposable quarantine expires. Learning/session/retirement backups
|
|
619
|
+
# are evidence, and are deliberately outside this pruning protocol.
|
|
620
|
+
def self.prune_quarantine(dry_run: true)
|
|
621
|
+
config = policy
|
|
622
|
+
result = { eligible: [], deleted: [], retained: [] }
|
|
623
|
+
days = config['retention_days']['maintenance']
|
|
624
|
+
return result if days.nil?
|
|
625
|
+
|
|
626
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
627
|
+
remaining_bytes = status[:categories]['maintenance'][:unique_bytes]
|
|
628
|
+
budget = config['budget_bytes']['maintenance']
|
|
629
|
+
Dir.glob(File.join(Dir.home, '.pwn', 'maintenance', 'quarantine', '*', 'ledger.json')).each do |ledger|
|
|
630
|
+
break if stopping? || result[:deleted].length >= config['batch_size']
|
|
631
|
+
|
|
632
|
+
raise IOError, 'Unsafe quarantine ledger' unless File.realpath(ledger) == ledger
|
|
633
|
+
|
|
634
|
+
data = JSON.parse(admitted_bytes(ledger))
|
|
635
|
+
next if data['state'] == 'expired'
|
|
636
|
+
next if data['state'] != 'expiry_prepared' && File.mtime(ledger) > Time.now - (days * 86_400) && !(budget && remaining_bytes > budget)
|
|
637
|
+
raise IOError, 'Uncommitted quarantine retained' unless data['operation'] == 'routine' && %w[committed expiry_prepared].include?(data['state']) && data['records'].is_a?(Array) && data['records'].all? { |row| row['state'] == 'committed' }
|
|
638
|
+
|
|
639
|
+
backups = data['records'].filter_map do |row|
|
|
640
|
+
path = row.fetch('backup')
|
|
641
|
+
raise IOError, 'Unsafe quarantine backup' unless File.dirname(path) == File.dirname(ledger) && File.expand_path(path) == path
|
|
642
|
+
next if data['state'] == 'expiry_prepared' && !File.exist?(path) && !File.symlink?(path)
|
|
643
|
+
|
|
644
|
+
raise IOError, 'Unknown quarantine bytes' unless File.dirname(path) == File.dirname(ledger) && File.realpath(path) == path && Digest::SHA256.hexdigest(admitted_bytes(path)) == row['sha256']
|
|
645
|
+
|
|
646
|
+
path
|
|
647
|
+
end
|
|
648
|
+
raise IOError, 'Unknown quarantine contents' unless (Dir.children(File.dirname(ledger)) - ['ledger.json', *backups.map { |path| File.basename(path) }]).empty?
|
|
649
|
+
|
|
650
|
+
result[:eligible].concat(backups)
|
|
651
|
+
remaining_bytes -= backups.sum { |path| File.size(path) }
|
|
652
|
+
next if dry_run
|
|
653
|
+
|
|
654
|
+
data['state'] = 'expiry_prepared'
|
|
655
|
+
publish(ledger, JSON.pretty_generate(data))
|
|
656
|
+
backups.each do |path|
|
|
657
|
+
break if !admit? || result[:deleted].length >= config['batch_size']
|
|
658
|
+
|
|
659
|
+
File.unlink(path)
|
|
660
|
+
result[:deleted] << path
|
|
661
|
+
end
|
|
662
|
+
File.open(File.dirname(ledger), File::RDONLY, &:fsync)
|
|
663
|
+
data['state'] = backups.any? { |path| File.exist?(path) } ? 'expiry_prepared' : 'expired'
|
|
664
|
+
publish(ledger, JSON.pretty_generate(data))
|
|
665
|
+
rescue StandardError => e
|
|
666
|
+
result[:retained] << { ledger: ledger, reason: e.class.name }
|
|
667
|
+
end
|
|
668
|
+
end
|
|
669
|
+
result
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
# Prune duplicate physical bytes, not recovery names: hardlink replacement
|
|
673
|
+
# preserves every store-specific rollback API and never loses the last copy.
|
|
674
|
+
# Only committed writer-defined backup rows enter this inventory.
|
|
675
|
+
def self.legacy_backups
|
|
676
|
+
root = File.join(Dir.home, '.pwn')
|
|
677
|
+
patterns = ['session_cleanup_archive/*/ledger.json', 'learning/migrations/*.json', 'maintenance/retirements/*/ledger.json']
|
|
678
|
+
rows = []
|
|
679
|
+
patterns.each do |pattern|
|
|
680
|
+
Dir.glob(File.join(root, pattern)).first(policy['scan_limit']).each do |ledger|
|
|
681
|
+
break unless admit?
|
|
682
|
+
next unless File.realpath(ledger) == ledger
|
|
683
|
+
|
|
684
|
+
data = JSON.parse(admitted_bytes(ledger))
|
|
685
|
+
candidates = []
|
|
686
|
+
if ledger.start_with?("#{File.join(root, 'learning', 'migrations')}/")
|
|
687
|
+
hash = File.basename(ledger, '.json')
|
|
688
|
+
next unless data['version'] == 1 && data['state'] == 'committed' && data['before_sha256'] == hash && hash.match?(/\A[a-f0-9]{64}\z/)
|
|
689
|
+
|
|
690
|
+
candidates << [File.join(File.dirname(ledger), "#{hash}.backup.jsonl"), hash]
|
|
691
|
+
elsif ledger.start_with?("#{File.join(root, 'maintenance', 'retirements')}/")
|
|
692
|
+
next unless data['state'] == 'committed' && data['session_id'].to_s.match?(PWN::Sessions::SESSION_ID)
|
|
693
|
+
|
|
694
|
+
candidates << [data['backup'], data['sha256']]
|
|
695
|
+
elsif data['dry_run'] == false && data['records'].is_a?(Array) && data['deleted_ids'].is_a?(Array)
|
|
696
|
+
data['records'].each do |row|
|
|
697
|
+
next unless row['deleted'] == true && data['deleted_ids'].include?(row['id']) && row['id'].to_s.match?(PWN::Sessions::SESSION_ID)
|
|
698
|
+
|
|
699
|
+
row.fetch('files', {}).each do |name, info|
|
|
700
|
+
next unless info && ["#{row['id']}.jsonl", "#{row['id']}.meta.json"].include?(name)
|
|
701
|
+
|
|
702
|
+
candidates << [File.join(File.dirname(ledger), name), info['sha256']]
|
|
703
|
+
end
|
|
704
|
+
end
|
|
705
|
+
Array(data['dependency_removals']).each do |row|
|
|
706
|
+
candidates << [row['backup'], row['sha256']] if row['state'] == 'committed'
|
|
707
|
+
end
|
|
708
|
+
end
|
|
709
|
+
candidates.each do |path, hash|
|
|
710
|
+
next unless path.is_a?(String) && File.dirname(path) == File.dirname(ledger)
|
|
711
|
+
next unless verified_backup?(path, hash)
|
|
712
|
+
|
|
713
|
+
rows << { path: path, sha256: hash, ledger: ledger }
|
|
714
|
+
end
|
|
715
|
+
rescue StandardError
|
|
716
|
+
# Unknown, partial and unreadable ledgers remain untouched.
|
|
717
|
+
next
|
|
718
|
+
end
|
|
719
|
+
end
|
|
720
|
+
rows.uniq { |row| row[:path] }
|
|
721
|
+
end
|
|
722
|
+
|
|
723
|
+
def self.verified_backup?(path, hash, admitted: false, size: nil)
|
|
724
|
+
hash.is_a?(String) && hash.match?(/\A[a-f0-9]{64}\z/) && File.file?(path) &&
|
|
725
|
+
File.realpath(path) == path && File.stat(path).uid == Process.uid &&
|
|
726
|
+
Digest::SHA256.hexdigest(admitted ? exact_bytes(path, size || File.size(path)) : admitted_bytes(path)) == hash
|
|
727
|
+
end
|
|
728
|
+
|
|
729
|
+
def self.prune_legacy_backups(dry_run: true)
|
|
730
|
+
result = { operation: 'backup_compaction', state: 'planned', compacted: [], retained: [], records: [] }
|
|
731
|
+
days = policy['retention_days']['maintenance']
|
|
732
|
+
return result if days.nil?
|
|
733
|
+
|
|
734
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
735
|
+
# Learning and continuation producers also serialize with these locks.
|
|
736
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
737
|
+
groups = legacy_backups.group_by { |row| row[:sha256] }
|
|
738
|
+
usage = status[:categories]['maintenance'][:unique_bytes]
|
|
739
|
+
budget = policy['budget_bytes']['maintenance']
|
|
740
|
+
groups.each_value do |rows|
|
|
741
|
+
canonical = rows.first
|
|
742
|
+
rows.drop(1).each do |row|
|
|
743
|
+
break unless admit?
|
|
744
|
+
break if result[:compacted].length >= policy['batch_size']
|
|
745
|
+
next if File.identical?(canonical[:path], row[:path])
|
|
746
|
+
next if File.mtime(row[:ledger]) > Time.now - (days * 86_400) && !(budget && usage > budget)
|
|
747
|
+
|
|
748
|
+
size = File.size(row[:path])
|
|
749
|
+
break unless admit?((size + 1) * 5)
|
|
750
|
+
|
|
751
|
+
usage -= size
|
|
752
|
+
record = row.merge(canonical: canonical[:path], bytes: size, state: 'prepared')
|
|
753
|
+
result[:records] << record
|
|
754
|
+
next if dry_run
|
|
755
|
+
|
|
756
|
+
unless result[:ledger]
|
|
757
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'compactions'))
|
|
758
|
+
result[:ledger] = File.join(directory, "#{SecureRandom.hex(12)}.json")
|
|
759
|
+
end
|
|
760
|
+
result[:state] = 'prepared'
|
|
761
|
+
publish(result[:ledger], JSON.generate(result))
|
|
762
|
+
compact_backup(record)
|
|
763
|
+
record[:state] = 'committed'
|
|
764
|
+
result[:compacted] << row[:path]
|
|
765
|
+
publish(result[:ledger], JSON.generate(result))
|
|
766
|
+
rescue StandardError => e
|
|
767
|
+
result[:retained] << { path: row[:path], reason: e.class.name }
|
|
768
|
+
end
|
|
769
|
+
end
|
|
770
|
+
result[:state] = stopping? || result[:records].any? { |row| row[:state] == 'prepared' } ? 'interrupted' : 'committed' unless dry_run
|
|
771
|
+
publish(result[:ledger], JSON.generate(result)) if result[:ledger]
|
|
772
|
+
end
|
|
773
|
+
end
|
|
774
|
+
result
|
|
775
|
+
end
|
|
776
|
+
|
|
777
|
+
def self.compact_backup(row)
|
|
778
|
+
path, canonical, hash = row.values_at(:path, :canonical, :sha256)
|
|
779
|
+
size = row[:bytes] || File.size(path)
|
|
780
|
+
raise IOError, 'Changed backup' unless verified_backup?(path, hash, admitted: true, size: size) && verified_backup?(canonical, hash, admitted: true, size: size)
|
|
781
|
+
return if File.identical?(path, canonical)
|
|
782
|
+
|
|
783
|
+
temporary = "#{path}.compact-#{SecureRandom.hex(12)}"
|
|
784
|
+
begin
|
|
785
|
+
File.link(canonical, temporary)
|
|
786
|
+
raise IOError, 'Backup readback mismatch' unless exact_bytes(temporary, size) == exact_bytes(path, size)
|
|
787
|
+
|
|
788
|
+
File.rename(temporary, path)
|
|
789
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
790
|
+
raise IOError, 'Compacted backup mismatch' unless verified_backup?(path, hash, admitted: true, size: size)
|
|
791
|
+
ensure
|
|
792
|
+
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
793
|
+
end
|
|
794
|
+
end
|
|
795
|
+
|
|
796
|
+
# Status is not rollback: broad maintenance has independently committed
|
|
797
|
+
# children. Restoring a ledger alone never undoes deletion or migration.
|
|
798
|
+
def self.recovery_status(ledger:)
|
|
799
|
+
root = File.join(Dir.home, '.pwn')
|
|
800
|
+
prefixes = %w[backup maintenance session_cleanup_archive learning/migrations].map { |name| "#{File.join(root, name)}/" }
|
|
801
|
+
raise IOError, 'Unsafe recovery ledger' unless prefixes.any? { |prefix| ledger.start_with?(prefix) } && File.realpath(ledger) == ledger
|
|
802
|
+
|
|
803
|
+
data = JSON.parse(File.binread(ledger))
|
|
804
|
+
operation = data['operation']
|
|
805
|
+
operation = 'session_cleanup' if ledger.include?('/backup/') && data['records'].is_a?(Array) && data['deleted_ids'].is_a?(Array)
|
|
806
|
+
operation = 'continuation_retirement' if ledger.include?('/backup/') && data['backup'] == File.join(File.dirname(ledger), 'personas.json')
|
|
807
|
+
operation = 'session_layout' if ledger.include?('/backup/') && File.basename(ledger) == 'layout.json'
|
|
808
|
+
operation = 'session_layout' if File.basename(ledger) == 'layout.json' && ledger.start_with?("#{File.join(root, 'session_cleanup_archive')}/") && data['records'].is_a?(Array)
|
|
809
|
+
operation = 'learning_migration' if ledger.start_with?("#{File.join(root, 'learning', 'migrations')}/") && data['version'] == 1 && data['before_sha256'] == File.basename(ledger, '.json')
|
|
810
|
+
operation = 'session_cleanup' if ledger.start_with?("#{File.join(root, 'session_cleanup_archive')}/") && data['dry_run'] == false && data['records'].is_a?(Array) && data['deleted_ids'].is_a?(Array)
|
|
811
|
+
operation = 'continuation_retirement' if ledger.start_with?("#{File.join(root, 'maintenance', 'retirements')}/") && %w[prepared committed rolled_back].include?(data['state']) && data['backup'] == File.join(File.dirname(ledger), 'personas.json')
|
|
812
|
+
actions = { 'session_layout' => ['resume'], 'session_cleanup' => ['restore'], 'continuation_retirement' => ['rollback'], 'backup_compaction' => ['resume'], 'maintenance' => ['resume'], 'learning_migration' => ['rollback'], 'routine' => ['restore'] }.fetch(operation, ['inspect_store_specific_recovery'])
|
|
813
|
+
actions << 'rollback_dependencies' if operation == 'session_cleanup' && data['dependency_removals'].is_a?(Array)
|
|
814
|
+
{ ledger: ledger, state: data['state'] || 'legacy_per_record', operation: operation,
|
|
815
|
+
actions: actions,
|
|
816
|
+
data: data, warning: 'Ledger restoration does not undo data changes; restore source sessions before learning rollback.' }
|
|
817
|
+
end
|
|
818
|
+
|
|
819
|
+
# Recovery is explicit repair, never startup/cron behavior. Both prepared
|
|
820
|
+
# and committed rows retain exact backups; an existing destination wins.
|
|
821
|
+
def self.recover(ledger:, dry_run: true, action: 'restore', writers_stopped: false)
|
|
822
|
+
report = recovery_status(ledger: ledger)
|
|
823
|
+
raise ArgumentError, 'Unsupported recovery action; inspect recovery_status' unless report[:actions].include?(action)
|
|
824
|
+
|
|
825
|
+
return reorganize(dry_run: dry_run, writers_stopped: writers_stopped) if report[:operation] == 'session_layout'
|
|
826
|
+
|
|
827
|
+
return recover_dependencies(ledger: ledger, data: report[:data], dry_run: dry_run) if action == 'rollback_dependencies'
|
|
828
|
+
return recover_sessions(ledger: ledger, data: report[:data], dry_run: dry_run) if report[:operation] == 'session_cleanup'
|
|
829
|
+
return recover_continuation(ledger: ledger, data: report[:data], dry_run: dry_run) if report[:operation] == 'continuation_retirement'
|
|
830
|
+
if report[:operation] == 'learning_migration'
|
|
831
|
+
return with_maintenance_lock(dry_run: dry_run) do
|
|
832
|
+
PWN::AI::Agent::Learning.rollback_evidence(migration: File.basename(ledger, '.json'), dry_run: dry_run)
|
|
833
|
+
end
|
|
834
|
+
end
|
|
835
|
+
return dry_run ? report.merge(dry_run: true) : maintain(dry_run: false) if report[:operation] == 'maintenance'
|
|
836
|
+
if report[:operation] == 'backup_compaction'
|
|
837
|
+
return with_maintenance_lock(dry_run: dry_run) do
|
|
838
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
839
|
+
known = legacy_backups.to_h { |row| [row[:path], row[:sha256]] }
|
|
840
|
+
data = report[:data]
|
|
841
|
+
data.fetch('records').each do |row|
|
|
842
|
+
raise IOError, 'Unknown compaction source' unless [row['path'], row['canonical']].all? { |path| known[path] == row['sha256'] }
|
|
843
|
+
next if dry_run
|
|
844
|
+
|
|
845
|
+
compact_backup(row.transform_keys(&:to_sym))
|
|
846
|
+
row['state'] = 'committed'
|
|
847
|
+
publish(ledger, JSON.generate(data))
|
|
848
|
+
end
|
|
849
|
+
data['state'] = 'committed' unless dry_run
|
|
850
|
+
publish(ledger, JSON.generate(data)) unless dry_run
|
|
851
|
+
{ dry_run: dry_run, state: data['state'], operation: 'backup_compaction' }
|
|
852
|
+
end
|
|
853
|
+
end
|
|
854
|
+
end
|
|
855
|
+
root = File.join(Dir.home, '.pwn', 'maintenance', 'quarantine')
|
|
856
|
+
raise IOError, 'Unsafe recovery ledger' unless [root, File.join(Dir.home, '.pwn', 'backup')].any? { |base| ledger.start_with?("#{base}/") } && File.realpath(ledger) == ledger
|
|
857
|
+
|
|
858
|
+
data = JSON.parse(File.binread(ledger))
|
|
859
|
+
raise IOError, 'Unknown recovery ledger' unless data['operation'] == 'routine' && data['records'].is_a?(Array)
|
|
860
|
+
|
|
861
|
+
result = { operation: 'repair', dry_run: dry_run, restored: [] }
|
|
862
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
863
|
+
data['records'].each do |row|
|
|
864
|
+
path = row.fetch('path')
|
|
865
|
+
category = row.fetch('category')
|
|
866
|
+
target_root = disposable_directory(category)
|
|
867
|
+
backup_path = row.fetch('backup')
|
|
868
|
+
raise IOError, 'Unsafe recovery path' unless %w[cache tmp logs].include?(category) && path.start_with?("#{target_root}/") && File.expand_path(path) == path && File.realpath(File.dirname(path)) == File.dirname(path)
|
|
869
|
+
raise IOError, 'Unsafe recovery backup' unless File.dirname(backup_path) == File.dirname(ledger) && File.realpath(backup_path) == backup_path && Digest::SHA256.file(backup_path).hexdigest == row['sha256']
|
|
870
|
+
raise IOError, 'Refusing existing recovery destination' if File.exist?(path) || File.symlink?(path)
|
|
871
|
+
next if dry_run
|
|
872
|
+
|
|
873
|
+
File.open("#{path}.lease", File::RDWR | File::NOFOLLOW) do |lease|
|
|
874
|
+
raise IOError, 'Recovery destination active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
875
|
+
|
|
876
|
+
temporary = "#{path}.restore-#{SecureRandom.hex(12)}"
|
|
877
|
+
begin
|
|
878
|
+
publish(temporary, File.binread(backup_path))
|
|
879
|
+
File.link(temporary, path) # atomic no-clobber; never replace a reader's new file
|
|
880
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
881
|
+
result[:restored] << path
|
|
882
|
+
row['recovered_at'] = Time.now.utc.iso8601
|
|
883
|
+
publish(ledger, JSON.pretty_generate(data))
|
|
884
|
+
ensure
|
|
885
|
+
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
886
|
+
end
|
|
887
|
+
end
|
|
888
|
+
end
|
|
889
|
+
end
|
|
890
|
+
result
|
|
891
|
+
end
|
|
892
|
+
|
|
893
|
+
def self.recover_dependencies(ledger:, data:, dry_run:)
|
|
894
|
+
result = { dry_run: dry_run, restored: [] }
|
|
895
|
+
root = File.join(Dir.home, '.pwn')
|
|
896
|
+
allowed = %w[learning.jsonl memory.json memory.archive.json mistakes.json].map { |name| File.join(root, name) }
|
|
897
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
898
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
899
|
+
data.fetch('dependency_removals').reverse_each do |row|
|
|
900
|
+
path, backup_path, id = row.values_at('path', 'backup', 'session_id')
|
|
901
|
+
raise IOError, 'Unsafe dependency recovery target' unless allowed.include?(path) && File.realpath(path) == path
|
|
902
|
+
raise IOError, 'Invalid dependency recovery backup' unless File.dirname(backup_path) == File.dirname(ledger) && verified_backup?(backup_path, row['sha256'])
|
|
903
|
+
raise IOError, 'Restore source session before dependency rollback' unless id.is_a?(String) && id.match?(PWN::Sessions::SESSION_ID) && File.file?(File.join(PWN::Sessions.sessions_dir, "#{id}.jsonl"))
|
|
904
|
+
|
|
905
|
+
current = Digest::SHA256.file(path).hexdigest
|
|
906
|
+
next if current == row['sha256']
|
|
907
|
+
raise IOError, 'Dependency store changed since removal' unless current == row['updated_sha256']
|
|
908
|
+
next if dry_run
|
|
909
|
+
|
|
910
|
+
publish(path, File.binread(backup_path))
|
|
911
|
+
raise IOError, 'Dependency restore mismatch' unless Digest::SHA256.file(path).hexdigest == row['sha256']
|
|
912
|
+
|
|
913
|
+
row['state'] = 'rolled_back'
|
|
914
|
+
publish(ledger, JSON.generate(data))
|
|
915
|
+
result[:restored] << path
|
|
916
|
+
end
|
|
917
|
+
end
|
|
918
|
+
end
|
|
919
|
+
result
|
|
920
|
+
end
|
|
921
|
+
|
|
922
|
+
def self.restore_copy(backup_path, path, hash)
|
|
923
|
+
raise IOError, 'Invalid recovery backup' unless verified_backup?(backup_path, hash)
|
|
924
|
+
raise IOError, 'Refusing existing recovery destination' if File.exist?(path) || File.symlink?(path)
|
|
925
|
+
raise IOError, 'Unsafe recovery parent' unless File.realpath(File.dirname(path)) == File.dirname(path)
|
|
926
|
+
|
|
927
|
+
temporary = "#{path}.restore-#{SecureRandom.hex(12)}"
|
|
928
|
+
begin
|
|
929
|
+
publish(temporary, File.binread(backup_path))
|
|
930
|
+
raise IOError, 'Restore readback mismatch' unless Digest::SHA256.file(temporary).hexdigest == hash
|
|
931
|
+
|
|
932
|
+
File.link(temporary, path)
|
|
933
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
934
|
+
ensure
|
|
935
|
+
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
936
|
+
end
|
|
937
|
+
end
|
|
938
|
+
|
|
939
|
+
def self.recover_sessions(ledger:, data:, dry_run:)
|
|
940
|
+
result = { dry_run: dry_run, restored: [], warning: 'Session bytes only; dependency remediations and learning migration are separate transactions.' }
|
|
941
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
942
|
+
data.fetch('records').each do |row|
|
|
943
|
+
id = row['id']
|
|
944
|
+
next unless id.is_a?(String) && id.match?(PWN::Sessions::SESSION_ID)
|
|
945
|
+
|
|
946
|
+
# A crash can follow unlink before deleted_ids was journaled. The
|
|
947
|
+
# exact prepared snapshot is also eligible, but never replaces data.
|
|
948
|
+
next unless row['files'].is_a?(Hash)
|
|
949
|
+
|
|
950
|
+
row['files'].each do |name, info|
|
|
951
|
+
next unless info && ["#{id}.jsonl", "#{id}.meta.json"].include?(name)
|
|
952
|
+
|
|
953
|
+
backup_path = File.join(File.dirname(ledger), name)
|
|
954
|
+
next unless File.file?(backup_path)
|
|
955
|
+
raise IOError, 'Invalid session recovery backup' unless verified_backup?(backup_path, info['sha256'])
|
|
956
|
+
|
|
957
|
+
path = name.end_with?('.meta.json') ? PWN::Sessions.support_path(kind: :metadata, session_id: id) : File.join(PWN::Sessions.sessions_dir, name)
|
|
958
|
+
next if verified_backup?(path, info['sha256'])
|
|
959
|
+
raise IOError, 'Refusing existing recovery destination' if File.exist?(path) || File.symlink?(path)
|
|
960
|
+
next if dry_run
|
|
961
|
+
|
|
962
|
+
File.open(PWN::Sessions.support_path(kind: :lease, session_id: id), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
963
|
+
raise IOError, 'Session recovery active' unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
964
|
+
|
|
965
|
+
restore_copy(backup_path, path, info['sha256'])
|
|
966
|
+
result[:restored] << path
|
|
967
|
+
(data['restored_paths'] ||= []) << path
|
|
968
|
+
publish(ledger, JSON.generate(data))
|
|
969
|
+
end
|
|
970
|
+
end
|
|
971
|
+
end
|
|
972
|
+
end
|
|
973
|
+
result
|
|
974
|
+
end
|
|
975
|
+
|
|
976
|
+
def self.recover_continuation(ledger:, data:, dry_run:)
|
|
977
|
+
root = File.join(Dir.home, '.pwn')
|
|
978
|
+
path, id = data.values_at('path', 'session_id')
|
|
979
|
+
raise IOError, 'Unsafe continuation recovery path' unless path.is_a?(String) && path.match?(%r{\A#{Regexp.escape(root)}/swarm/\d{8}_\d{6}_[a-f0-9]{6}/personas\.json\z}) && File.realpath(path) == path
|
|
980
|
+
raise IOError, 'Invalid continuation recovery identity' unless id.is_a?(String) && id.match?(PWN::Sessions::SESSION_ID)
|
|
981
|
+
raise IOError, 'Invalid continuation backup' unless verified_backup?(data['backup'], data['sha256'])
|
|
982
|
+
|
|
983
|
+
before = File.binread(data['backup'])
|
|
984
|
+
map = JSON.parse(before)
|
|
985
|
+
raise IOError, 'Invalid continuation backup schema' unless map.is_a?(Hash) && map[data['name']] == id
|
|
986
|
+
|
|
987
|
+
expected = map.reject { |name, _| name == data['name'] }
|
|
988
|
+
result = { dry_run: dry_run, restored: [] }
|
|
989
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
990
|
+
with_dependency_lock(nil, dry_run: dry_run) do
|
|
991
|
+
current = JSON.parse(File.binread(path))
|
|
992
|
+
raise IOError, 'Continuation map changed after retirement' unless current == expected || current == map
|
|
993
|
+
next if dry_run || current == map
|
|
994
|
+
|
|
995
|
+
File.open(PWN::Sessions.support_path(kind: :lease, session_id: id), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
996
|
+
raise IOError, 'Continuation recovery active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
997
|
+
raise IOError, 'Restore source session before continuation' unless File.file?(File.join(PWN::Sessions.sessions_dir, "#{id}.jsonl"))
|
|
998
|
+
|
|
999
|
+
publish(path, before)
|
|
1000
|
+
raise IOError, 'Continuation restore readback mismatch' unless File.binread(path) == before
|
|
1001
|
+
|
|
1002
|
+
data['state'] = 'rolled_back'
|
|
1003
|
+
publish(ledger, JSON.generate(data))
|
|
1004
|
+
result[:restored] << path
|
|
1005
|
+
end
|
|
1006
|
+
end
|
|
1007
|
+
end
|
|
1008
|
+
result
|
|
1009
|
+
end
|
|
1010
|
+
|
|
1011
|
+
def self.with_continuation_lock(root: File.join(Dir.home, '.pwn'))
|
|
1012
|
+
File.open(File.join(root, '.swarm-continuations.lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
|
|
1013
|
+
lock.flock(File::LOCK_EX)
|
|
1014
|
+
yield
|
|
1015
|
+
end
|
|
1016
|
+
end
|
|
1017
|
+
|
|
1018
|
+
def self.retire_continuation(root:, swarm_id:, name:)
|
|
1019
|
+
raise ArgumentError, 'Invalid swarm identifier' unless swarm_id.match?(/\A\d{8}_\d{6}_[a-f0-9]{6}\z/)
|
|
1020
|
+
|
|
1021
|
+
path = File.join(root, swarm_id, 'personas.json')
|
|
1022
|
+
with_maintenance_lock(dry_run: false) do
|
|
1023
|
+
with_continuation_lock(root: File.dirname(root)) do
|
|
1024
|
+
raise IOError, 'Unsafe continuation map' unless File.realpath(path) == path
|
|
1025
|
+
|
|
1026
|
+
bytes = File.binread(path)
|
|
1027
|
+
map = JSON.parse(bytes)
|
|
1028
|
+
raise IOError, 'Unknown continuation map' unless map.is_a?(Hash) && map.values.all? { |id| id.is_a?(String) && id.match?(/\A\d{8}_\d{6}_[a-f0-9]{8}\z/) }
|
|
1029
|
+
|
|
1030
|
+
id = map[name]
|
|
1031
|
+
next { state: 'absent' } unless id
|
|
1032
|
+
|
|
1033
|
+
File.open(PWN::Sessions.support_path(kind: :lease, session_id: id), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lease|
|
|
1034
|
+
raise IOError, 'Swarm continuation active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
1035
|
+
|
|
1036
|
+
directory = PWN::Backups.directory(label: 'continuation')
|
|
1037
|
+
backup_path = File.join(directory, 'personas.json')
|
|
1038
|
+
publish(backup_path, bytes)
|
|
1039
|
+
raise IOError, 'Continuation backup mismatch' unless File.binread(backup_path) == bytes
|
|
1040
|
+
|
|
1041
|
+
result = { state: 'prepared', session_id: id, path: path, name: name, backup: backup_path, sha256: Digest::SHA256.hexdigest(bytes) }
|
|
1042
|
+
ledger = File.join(directory, 'ledger.json')
|
|
1043
|
+
publish(ledger, JSON.pretty_generate(result))
|
|
1044
|
+
map.delete(name)
|
|
1045
|
+
publish(path, JSON.pretty_generate(map))
|
|
1046
|
+
raise IOError, 'Continuation retirement mismatch' unless JSON.parse(File.binread(path)) == map
|
|
1047
|
+
|
|
1048
|
+
result[:state] = 'committed'
|
|
1049
|
+
publish(ledger, JSON.pretty_generate(result))
|
|
1050
|
+
result
|
|
1051
|
+
end
|
|
1052
|
+
end
|
|
1053
|
+
end
|
|
1054
|
+
end
|
|
16
1055
|
|
|
17
1056
|
def self.classify(path, id)
|
|
18
1057
|
return :unknown unless path == File.join(PWN::Sessions.sessions_dir, "#{id}.jsonl")
|
|
@@ -113,28 +1152,77 @@ module PWN
|
|
|
113
1152
|
|
|
114
1153
|
def self.publish(path, bytes)
|
|
115
1154
|
temporary = "#{path}.#{SecureRandom.hex(8)}.tmp"
|
|
1155
|
+
root = File.join(Dir.home, '.pwn')
|
|
1156
|
+
category = %w[cache tmp].find { |name| path.start_with?("#{disposable_directory(name)}/") }
|
|
1157
|
+
if category
|
|
1158
|
+
directory = private_directory(File.join(root, 'maintenance', 'atomic'))
|
|
1159
|
+
journal = File.join(directory, "#{SecureRandom.hex(12)}.json")
|
|
1160
|
+
end
|
|
116
1161
|
File.open(temporary, File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW, 0o600) do |file|
|
|
1162
|
+
file.flock(File::LOCK_EX)
|
|
1163
|
+
if journal
|
|
1164
|
+
row = { version: 1, operation: 'atomic_disposable', category: category, path: temporary, target: path,
|
|
1165
|
+
inode: file.stat.ino, device: file.stat.dev, sha256: Digest::SHA256.hexdigest(bytes), created_at: Time.now.to_i }
|
|
1166
|
+
publish(journal, JSON.generate(row))
|
|
1167
|
+
end
|
|
117
1168
|
file.write(bytes)
|
|
118
1169
|
file.flush
|
|
119
1170
|
file.fsync
|
|
1171
|
+
File.rename(temporary, path)
|
|
1172
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
120
1173
|
end
|
|
121
|
-
File.rename(temporary, path)
|
|
122
|
-
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
123
1174
|
ensure
|
|
124
1175
|
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
1176
|
+
File.unlink(journal) if journal && File.exist?(journal)
|
|
125
1177
|
end
|
|
126
1178
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
1179
|
+
# Only registered disposable atomic outputs are recoverable automatically.
|
|
1180
|
+
# Missing journals, partial bytes, wrong inode, active writers and evidence
|
|
1181
|
+
# atomic outputs are preserved, regardless of filename or apparent age.
|
|
1182
|
+
def self.recover_atomic_temps(dry_run: true, expired_only: false)
|
|
1183
|
+
result = { deleted: [], retained: [] }
|
|
1184
|
+
root = File.join(Dir.home, '.pwn')
|
|
1185
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
1186
|
+
Dir.glob(File.join(root, 'maintenance', 'atomic', '*.json')).first(policy['scan_limit']).each do |ledger|
|
|
1187
|
+
break unless admit?
|
|
1188
|
+
break if result[:deleted].length >= policy['batch_size']
|
|
1189
|
+
raise IOError, 'Unsafe atomic ledger' unless File.realpath(ledger) == ledger
|
|
1190
|
+
|
|
1191
|
+
row = JSON.parse(admitted_bytes(ledger))
|
|
1192
|
+
raise IOError, 'Unknown atomic owner' unless row['version'] == 1 && row['operation'] == 'atomic_disposable' && %w[cache tmp].include?(row['category'])
|
|
1193
|
+
|
|
1194
|
+
days = policy['retention_days'][row['category']]
|
|
1195
|
+
next if days.nil? || (expired_only && row.fetch('created_at') > Time.now.to_i - (days * 86_400))
|
|
1196
|
+
|
|
1197
|
+
path, target = row.values_at('path', 'target')
|
|
1198
|
+
base = "#{disposable_directory(row['category'])}/"
|
|
1199
|
+
raise IOError, 'Unsafe atomic path' unless [path, target].all? { |name| name.is_a?(String) && name.start_with?(base) && File.expand_path(name) == name } && File.dirname(path) == File.dirname(target) && path.match?(/\A#{Regexp.escape(target)}\.[a-f0-9]{16}\.tmp\z/)
|
|
1200
|
+
next unless File.exist?(path) || File.symlink?(path)
|
|
1201
|
+
raise IOError, 'Changed atomic bytes' unless verified_backup?(path, row['sha256']) && File.stat(path).ino == row['inode'] && File.stat(path).dev == row['device']
|
|
1202
|
+
|
|
1203
|
+
File.open("#{target}.lease", File::RDWR | File::NOFOLLOW) do |lease|
|
|
1204
|
+
raise IOError, 'Atomic producer active' unless lease.flock(File::LOCK_EX | File::LOCK_NB)
|
|
1205
|
+
|
|
1206
|
+
File.open(path, File::RDONLY | File::NOFOLLOW) do |file|
|
|
1207
|
+
raise IOError, 'Atomic file active or changed' unless file.flock(File::LOCK_EX | File::LOCK_NB) && File.identical?(file, path) && file.stat.ino == row['inode'] && Digest::SHA256.hexdigest(admitted_bytes(path)) == row['sha256']
|
|
1208
|
+
next if dry_run
|
|
1209
|
+
|
|
1210
|
+
File.unlink(path)
|
|
1211
|
+
File.open(File.dirname(path), File::RDONLY, &:fsync)
|
|
1212
|
+
row['state'] = 'recovered'
|
|
1213
|
+
publish(ledger, JSON.generate(row))
|
|
1214
|
+
result[:deleted] << path
|
|
1215
|
+
end
|
|
1216
|
+
end
|
|
1217
|
+
rescue StandardError => e
|
|
1218
|
+
result[:retained] << { ledger: ledger, reason: e.class.name }
|
|
1219
|
+
end
|
|
137
1220
|
end
|
|
1221
|
+
result
|
|
1222
|
+
end
|
|
1223
|
+
|
|
1224
|
+
def self.backup(path, expected, result)
|
|
1225
|
+
result[:archive_dir] ||= PWN::Backups.directory(label: 'session-cleanup')
|
|
138
1226
|
expected.each do |name, record|
|
|
139
1227
|
next unless record
|
|
140
1228
|
|
|
@@ -175,17 +1263,22 @@ module PWN
|
|
|
175
1263
|
state
|
|
176
1264
|
end
|
|
177
1265
|
|
|
178
|
-
def self.with_dependency_lock(id)
|
|
1266
|
+
def self.with_dependency_lock(id, dry_run: false)
|
|
1267
|
+
return yield if dry_run
|
|
1268
|
+
|
|
179
1269
|
rows = invalid_dependencies(id)
|
|
180
1270
|
raise 'Unresolved dependency remediation' unless rows.empty? || remediable?(id)
|
|
181
1271
|
|
|
182
1272
|
# Lock even absent stores: a concurrent writer can introduce the first
|
|
183
1273
|
# reference between the final index and unlink. These inodes are permanent.
|
|
184
1274
|
root = File.join(Dir.home, '.pwn')
|
|
185
|
-
paths = %w[.learning-write.lock memory.json.write.lock mistakes.json.write.lock].map { |name| File.join(root, name) }.sort
|
|
1275
|
+
paths = %w[.learning-write.lock .swarm-continuations.lock memory.json.write.lock mistakes.json.write.lock].map { |name| File.join(root, name) }.sort
|
|
186
1276
|
locks = []
|
|
187
1277
|
begin
|
|
188
1278
|
paths.each do |path|
|
|
1279
|
+
owner = Thread.current[:pwn_learning_write_lock]
|
|
1280
|
+
next if File.basename(path) == '.learning-write.lock' && owner && owner[:pid] == Process.pid && owner[:thread] == Thread.current
|
|
1281
|
+
|
|
189
1282
|
lock = File.open(path, File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) # rubocop:disable Style/FileOpen -- closed together in ensure
|
|
190
1283
|
locks << lock
|
|
191
1284
|
raise 'Dependency writer active' unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
@@ -240,7 +1333,7 @@ module PWN
|
|
|
240
1333
|
# Legacy writers block all mutations, including evidence migration.
|
|
241
1334
|
# One permanent inode serializes CLI, curses and cron maintenance writers.
|
|
242
1335
|
# Read-only previews never acquire a writer lock or create lock files.
|
|
243
|
-
def self.with_maintenance_lock(dry_run
|
|
1336
|
+
def self.with_maintenance_lock(dry_run:, &block)
|
|
244
1337
|
return yield if dry_run
|
|
245
1338
|
|
|
246
1339
|
root = PWN::Sessions.sessions_dir
|
|
@@ -255,22 +1348,127 @@ module PWN
|
|
|
255
1348
|
|
|
256
1349
|
begin
|
|
257
1350
|
Thread.current[:pwn_session_maintenance_lock] = { pid: Process.pid, thread: Thread.current, path: path, file: file }
|
|
258
|
-
|
|
1351
|
+
learning = PWN::AI::Agent::Learning.send(:learning_lock, nonblocking: true)
|
|
1352
|
+
if learning == :busy
|
|
1353
|
+
Thread.current[:pwn_backup_learning_busy] = true
|
|
1354
|
+
yield
|
|
1355
|
+
else
|
|
1356
|
+
PWN::Backups.transaction(&block)
|
|
1357
|
+
end
|
|
259
1358
|
ensure
|
|
1359
|
+
PWN::AI::Agent::Learning.send(:learning_unlock, lock: learning) unless learning == :busy
|
|
1360
|
+
Thread.current[:pwn_backup_learning_busy] = nil
|
|
260
1361
|
Thread.current[:pwn_session_maintenance_lock] = owner
|
|
261
1362
|
end
|
|
262
1363
|
end
|
|
263
1364
|
end
|
|
264
1365
|
|
|
1366
|
+
def self.stopping?
|
|
1367
|
+
control = Thread.current[:pwn_steering_input]
|
|
1368
|
+
budget = Thread.current[:pwn_maintenance_budget]
|
|
1369
|
+
(control.respond_to?(:stopping?) && control.stopping?) || (budget && (budget[:remaining] <= 0 || budget[:bytes] <= 0 || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= budget[:deadline]))
|
|
1370
|
+
end
|
|
1371
|
+
|
|
1372
|
+
# Fresh schedules use one coordinator. Existing split jobs keep maintain
|
|
1373
|
+
# until an operator explicitly replaces their schedules after deployment.
|
|
1374
|
+
# Preserve evidence before retention can remove its last source edge.
|
|
1375
|
+
def self.coordinate(dry_run: false, current_session_id: nil, proc_root: '/proc')
|
|
1376
|
+
with_maintenance_lock(dry_run: dry_run) do
|
|
1377
|
+
result = { state: 'prepared', dry_run: dry_run, phases: {} }
|
|
1378
|
+
unless dry_run
|
|
1379
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'runs'))
|
|
1380
|
+
ledger = File.join(directory, "coordinator-#{SecureRandom.hex(12)}.json")
|
|
1381
|
+
publish(ledger, JSON.generate(result))
|
|
1382
|
+
end
|
|
1383
|
+
begin
|
|
1384
|
+
maintenance = maintain(dry_run: dry_run, current_session_id: current_session_id, proc_root: proc_root)
|
|
1385
|
+
result[:phases][:maintenance] = maintenance
|
|
1386
|
+
if maintenance[:apply_blocked] || maintenance[:incomplete] || stopping?
|
|
1387
|
+
result[:state] = maintenance[:apply_blocked] ? 'blocked' : 'interrupted'
|
|
1388
|
+
else
|
|
1389
|
+
result[:phases][:memory_consolidation] = dry_run ? { planned: true } : PWN::AI::Agent::Learning.consolidate
|
|
1390
|
+
result[:phases][:learning_retention] = PWN::AI::Agent::Learning.prune_outcomes!(dry_run: dry_run)
|
|
1391
|
+
result[:phases][:mistakes_retention] = PWN::AI::Agent::Mistakes.lean!(dry_run: dry_run)
|
|
1392
|
+
result[:phases][:policy_retention] = PWN::AI::Agent::Policy.lean!(dry_run: dry_run)
|
|
1393
|
+
result[:state] = dry_run ? 'planned' : 'committed'
|
|
1394
|
+
end
|
|
1395
|
+
result[:transaction_ledger] = ledger if ledger
|
|
1396
|
+
publish(ledger, JSON.generate(result)) if ledger
|
|
1397
|
+
result
|
|
1398
|
+
rescue StandardError, Interrupt, SystemExit => e
|
|
1399
|
+
result[:state] = 'interrupted'
|
|
1400
|
+
publish(ledger, JSON.generate(result.merge(error: e.class.name))) if ledger
|
|
1401
|
+
raise
|
|
1402
|
+
end
|
|
1403
|
+
end
|
|
1404
|
+
end
|
|
1405
|
+
|
|
265
1406
|
def self.maintain(dry_run: false, current_session_id: nil, proc_root: '/proc')
|
|
1407
|
+
config = policy # validate policy before any mutation or migration
|
|
1408
|
+
prior_budget = Thread.current[:pwn_maintenance_budget]
|
|
1409
|
+
prior_admission = Thread.current[:pwn_maintenance_admission]
|
|
1410
|
+
Thread.current[:pwn_maintenance_budget] = { remaining: config['work_items'], bytes: config['work_bytes'], deadline: Process.clock_gettime(Process::CLOCK_MONOTONIC) + config['work_seconds'] }
|
|
1411
|
+
Thread.current[:pwn_maintenance_admission] = method(:admit?)
|
|
266
1412
|
with_maintenance_lock(dry_run: dry_run) do
|
|
267
|
-
|
|
1413
|
+
unless dry_run
|
|
1414
|
+
directory = private_directory(File.join(Dir.home, '.pwn', 'maintenance', 'runs'))
|
|
1415
|
+
ledger = File.join(directory, "#{Time.now.utc.strftime('%Y%m%dT%H%M%S')}-#{SecureRandom.hex(8)}.json")
|
|
1416
|
+
publish(ledger, JSON.generate(version: 1, state: 'prepared', operation: 'maintenance'))
|
|
1417
|
+
end
|
|
1418
|
+
begin
|
|
1419
|
+
result = maintenance_transaction(dry_run: dry_run, current_session_id: current_session_id, proc_root: proc_root)
|
|
1420
|
+
result[:storage] = clean_disposables(dry_run: dry_run || result[:apply_blocked] == true)
|
|
1421
|
+
result[:quarantine] = prune_quarantine(dry_run: dry_run || result[:apply_blocked] == true)
|
|
1422
|
+
unless stopping?
|
|
1423
|
+
result[:atomic_temps] = recover_atomic_temps(dry_run: dry_run || result[:apply_blocked] == true, expired_only: true)
|
|
1424
|
+
result[:legacy_backups] = prune_legacy_backups(dry_run: dry_run || result[:apply_blocked] == true)
|
|
1425
|
+
end
|
|
1426
|
+
result[:migration_backups] = PWN::Migrate.prune_backups(dry_run: dry_run || result[:apply_blocked] == true, admission: method(:admit?)) unless stopping?
|
|
1427
|
+
result[:usage] = status unless stopping?
|
|
1428
|
+
result[:incomplete] = true if stopping? || result[:storage][:incomplete]
|
|
1429
|
+
result[:state] = if dry_run
|
|
1430
|
+
'planned'
|
|
1431
|
+
elsif result[:incomplete] || result[:storage][:state] == 'interrupted'
|
|
1432
|
+
'interrupted'
|
|
1433
|
+
elsif result[:apply_blocked]
|
|
1434
|
+
'blocked'
|
|
1435
|
+
else
|
|
1436
|
+
'committed'
|
|
1437
|
+
end
|
|
1438
|
+
result[:transaction_ledger] = ledger if ledger
|
|
1439
|
+
publish(ledger, JSON.generate(version: 1, state: result[:state], operation: 'maintenance', result: result)) if ledger
|
|
1440
|
+
result
|
|
1441
|
+
rescue StandardError, Interrupt, SystemExit => e
|
|
1442
|
+
publish(ledger, JSON.generate(version: 1, state: 'interrupted', operation: 'maintenance', error: e.class.name, result: result)) if ledger
|
|
1443
|
+
raise
|
|
1444
|
+
end
|
|
268
1445
|
end
|
|
1446
|
+
ensure
|
|
1447
|
+
Thread.current[:pwn_maintenance_budget] = prior_budget
|
|
1448
|
+
Thread.current[:pwn_maintenance_admission] = prior_admission
|
|
1449
|
+
end
|
|
1450
|
+
|
|
1451
|
+
# Admission is cooperative: an admitted atomic write is never interrupted.
|
|
1452
|
+
def self.admit?(bytes = 0)
|
|
1453
|
+
return false if stopping?
|
|
1454
|
+
|
|
1455
|
+
budget = Thread.current[:pwn_maintenance_budget]
|
|
1456
|
+
if budget
|
|
1457
|
+
budget[:remaining] -= 1
|
|
1458
|
+
if bytes > budget[:bytes]
|
|
1459
|
+
budget[:bytes] = 0
|
|
1460
|
+
return false
|
|
1461
|
+
end
|
|
1462
|
+
budget[:bytes] -= bytes
|
|
1463
|
+
end
|
|
1464
|
+
true
|
|
269
1465
|
end
|
|
270
1466
|
|
|
271
1467
|
def self.maintenance_transaction(dry_run:, current_session_id:, proc_root:)
|
|
272
1468
|
preflight = sweep(dry_run: true, current_session_id: current_session_id, proc_root: proc_root)
|
|
273
|
-
|
|
1469
|
+
return preflight.merge(apply_blocked: true, evidence_migration: { deferred: 'learning_writer_active', migrated_rows: 0 }) if Thread.current[:pwn_backup_learning_busy] && !dry_run
|
|
1470
|
+
|
|
1471
|
+
blocked = stopping? || !preflight[:legacy_owners].empty? || Array(preflight[:unresolved_store_remediation]).any?
|
|
274
1472
|
unless dry_run || blocked
|
|
275
1473
|
# Private metadata-only preflight survives an interrupted migration.
|
|
276
1474
|
backup(File.join(PWN::Sessions.sessions_dir, 'inventory.jsonl'), {}, preflight)
|
|
@@ -278,7 +1476,10 @@ module PWN
|
|
|
278
1476
|
publish(ledger, JSON.pretty_generate(state: 'prepared', inventory: preflight))
|
|
279
1477
|
blocked = !legacy_owners(proc_root).empty?
|
|
280
1478
|
end
|
|
281
|
-
|
|
1479
|
+
return preflight.merge(incomplete: true, apply_blocked: true) if stopping?
|
|
1480
|
+
|
|
1481
|
+
migration = PWN::AI::Agent::Learning.migrate_evidence(dry_run: dry_run || blocked, max_sessions: policy['batch_size'], stop: method(:stopping?), nonblocking: true)
|
|
1482
|
+
blocked = true if migration[:deferred]
|
|
282
1483
|
result = if dry_run || blocked
|
|
283
1484
|
preflight
|
|
284
1485
|
else
|
|
@@ -404,7 +1605,7 @@ module PWN
|
|
|
404
1605
|
{ version: VERSION, dry_run: result[:dry_run], apply_blocked: result[:apply_blocked] == true,
|
|
405
1606
|
scanned: result[:scanned], candidates: result[:candidate_ids].length, deleted: result[:deleted_ids].length,
|
|
406
1607
|
remaining: result[:remaining_sessions], visible: result[:visible_sessions], skipped: result[:skipped],
|
|
407
|
-
evidence_migration: result[:evidence_migration], archive_dir: result[:archive_dir],
|
|
1608
|
+
evidence_migration: result[:evidence_migration], archive_dir: result[:archive_dir], storage: result[:storage], usage: result[:usage], quarantine: result[:quarantine],
|
|
408
1609
|
unresolved_dependency_stores: result[:dependency_errors] || [], legacy_owners: result[:legacy_owners] }
|
|
409
1610
|
end
|
|
410
1611
|
|
|
@@ -430,6 +1631,7 @@ module PWN
|
|
|
430
1631
|
legacy = legacy_owners(proc_root)
|
|
431
1632
|
result = { version: VERSION, dry_run: dry_run, scanned: 0, unnamed: 0, candidate_ids: [], deleted_ids: [], records: [], skipped: Hash.new(0), legacy_owners: legacy }
|
|
432
1633
|
references = PWN::Sessions.dependency_index
|
|
1634
|
+
result[:incomplete] = true if references[:errors].include?('incomplete_dependency_index')
|
|
433
1635
|
unless references[:errors].empty?
|
|
434
1636
|
result[:skipped][:unknown_pins] = 1
|
|
435
1637
|
result[:dependency_errors] = references[:errors]
|
|
@@ -439,6 +1641,10 @@ module PWN
|
|
|
439
1641
|
dependencies = references[:dependencies]
|
|
440
1642
|
pins = dependencies.keys
|
|
441
1643
|
Dir.glob(File.join(root, '*.jsonl')).each do |path|
|
|
1644
|
+
unless admit?
|
|
1645
|
+
result[:incomplete] = true
|
|
1646
|
+
break
|
|
1647
|
+
end
|
|
442
1648
|
id = File.basename(path, '.jsonl')
|
|
443
1649
|
result[:scanned] += 1
|
|
444
1650
|
state = cleanup_state(path, id, references)
|
|
@@ -457,6 +1663,12 @@ module PWN
|
|
|
457
1663
|
result[:skipped][:unresolved_dependency_remediation] += 1
|
|
458
1664
|
next
|
|
459
1665
|
end
|
|
1666
|
+
days = policy['retention_days']['sessions']
|
|
1667
|
+
if days.nil? || File.mtime(path) > Time.now - (days * 86_400)
|
|
1668
|
+
record[:skip] = :retention
|
|
1669
|
+
result[:skipped][:retention] += 1
|
|
1670
|
+
next
|
|
1671
|
+
end
|
|
460
1672
|
expected = snapshot(path)
|
|
461
1673
|
expected_dependencies = invalid_dependencies(id, references)
|
|
462
1674
|
record[:dependency_records] = expected_dependencies
|
|
@@ -484,6 +1696,17 @@ module PWN
|
|
|
484
1696
|
record[:apply_blocked_by] = :unknown_legacy_ownership unless legacy.empty?
|
|
485
1697
|
next if dry_run
|
|
486
1698
|
|
|
1699
|
+
if stopping?
|
|
1700
|
+
record[:skip] = :stopped
|
|
1701
|
+
result[:skipped][:stopped] += 1
|
|
1702
|
+
next
|
|
1703
|
+
end
|
|
1704
|
+
if result[:deleted_ids].length >= policy['batch_size']
|
|
1705
|
+
record[:skip] = :batch_limit
|
|
1706
|
+
result[:skipped][:batch_limit] += 1
|
|
1707
|
+
next
|
|
1708
|
+
end
|
|
1709
|
+
|
|
487
1710
|
unless legacy.empty? && legacy_owners(proc_root).empty?
|
|
488
1711
|
record[:skip] = :unknown_legacy_ownership
|
|
489
1712
|
result[:skipped][:unknown_legacy_ownership] += 1
|