pwn 0.5.784 → 0.5.787

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (254) hide show
  1. checksums.yaml +4 -4
  2. data/.ruby-version +1 -1
  3. data/Gemfile +6 -6
  4. data/README.md +37 -0
  5. data/bin/pwn +2 -1
  6. data/bin/pwn_web_cache_deception +35 -7
  7. data/build_gem.sh +1 -1
  8. data/documentation/AI-Integration.md +7 -5
  9. data/documentation/Agent-Tool-Registry.md +2 -2
  10. data/documentation/Backup-Retention.md +69 -15
  11. data/documentation/Configuration.md +10 -6
  12. data/documentation/Contributing.md +1 -1
  13. data/documentation/Cron.md +2 -2
  14. data/documentation/Diagrams.md +2 -2
  15. data/documentation/General-PWN-Usage.md +4 -3
  16. data/documentation/Harness-Evaluation.md +1 -1
  17. data/documentation/Home.md +6 -5
  18. data/documentation/How-PWN-Works.md +6 -5
  19. data/documentation/Installation.md +12 -9
  20. data/documentation/Mission-Cancellation.md +134 -0
  21. data/documentation/Operability.md +18 -0
  22. data/documentation/Persistence.md +7 -5
  23. data/documentation/Plugin-Degradation.md +14 -0
  24. data/documentation/Plugins.md +3 -2
  25. data/documentation/Policy-Benchmark.md +32 -33
  26. data/documentation/Runtime-Source-Protection.md +162 -0
  27. data/documentation/Transparent-Browser.md +11 -0
  28. data/documentation/Troubleshooting.md +3 -2
  29. data/documentation/What-is-PWN.md +2 -2
  30. data/documentation/YubiKey-Vaults.md +185 -0
  31. data/documentation/YubiKey.md +218 -0
  32. data/documentation/diagrams/cron-scheduling.svg +130 -130
  33. data/documentation/diagrams/dot/cron-scheduling.dot +3 -3
  34. data/documentation/diagrams/dot/persistence-filesystem.dot +5 -3
  35. data/documentation/diagrams/dot/plugin-ecosystem.dot +2 -2
  36. data/documentation/diagrams/dot/pwn-ai-feedback-learning-loop.dot +1 -1
  37. data/documentation/diagrams/dot/reinforcement-learning.dot +2 -2
  38. data/documentation/diagrams/persistence-filesystem.svg +172 -142
  39. data/documentation/diagrams/plugin-ecosystem.svg +64 -64
  40. data/documentation/diagrams/pwn-ai-feedback-learning-loop.svg +1 -1
  41. data/documentation/diagrams/reinforcement-learning.svg +89 -89
  42. data/documentation/pwn-ai-Agent.md +10 -4
  43. data/etc/default_cron/session_cleanup.rb +43 -27
  44. data/etc/default_skills/pwn/ai/agent/mission_namespace/SKILL.md +21 -0
  45. data/etc/default_skills/pwn/ai/agent/tools/browser_act/SKILL.md +45 -0
  46. data/etc/default_skills/pwn/ai/agent/tools/loot_query/SKILL.md +27 -26
  47. data/etc/default_skills/pwn/ai/agent/tools/recon_run/SKILL.md +25 -15
  48. data/etc/default_skills/pwn/ai/agent/tools/workspace_tool/SKILL.md +24 -0
  49. data/etc/default_skills/pwn/ai/server_tools/SKILL.md +25 -0
  50. data/etc/default_skills/pwn/backups/SKILL.md +54 -0
  51. data/etc/default_skills/pwn/plugins/recon/SKILL.md +12 -1
  52. data/etc/default_skills/pwn/plugins/yubikey/SKILL.md +157 -0
  53. data/etc/default_skills/pwn/plugins/yubikey/fido/SKILL.md +189 -0
  54. data/etc/default_skills/pwn/plugins/yubikey/hsmauth/SKILL.md +83 -0
  55. data/etc/default_skills/pwn/plugins/yubikey/management/SKILL.md +80 -0
  56. data/etc/default_skills/pwn/plugins/yubikey/oath/SKILL.md +115 -0
  57. data/etc/default_skills/pwn/plugins/yubikey/openpgp/SKILL.md +144 -0
  58. data/etc/default_skills/pwn/plugins/yubikey/otp/SKILL.md +97 -0
  59. data/etc/default_skills/pwn/plugins/yubikey/piv/SKILL.md +110 -0
  60. data/etc/default_skills/pwn/plugins/yubikey/protocol/SKILL.md +21 -0
  61. data/etc/default_skills/pwn/plugins/yubikey/security_domain/SKILL.md +75 -0
  62. data/etc/default_skills/pwn/plugins/yubikey/security_domain/scp/SKILL.md +46 -0
  63. data/etc/default_skills/pwn/plugins/yubikey/transport/SKILL.md +98 -0
  64. data/etc/default_skills/pwn/redaction/SKILL.md +13 -1
  65. data/lib/pwn/ai/agent/dispatch.rb +26 -15
  66. data/lib/pwn/ai/agent/learning.rb +14 -18
  67. data/lib/pwn/ai/agent/loop.rb +17 -1
  68. data/lib/pwn/ai/agent/mission_namespace.rb +331 -0
  69. data/lib/pwn/ai/agent/policy_evaluation.rb +334 -36
  70. data/lib/pwn/ai/agent/prompt_builder.rb +13 -2
  71. data/lib/pwn/ai/agent/request_runtime.rb +122 -18
  72. data/lib/pwn/ai/agent/reward.rb +5 -3
  73. data/lib/pwn/ai/agent/skill_consolidation.rb +3 -2
  74. data/lib/pwn/ai/agent/skill_review.rb +9 -5
  75. data/lib/pwn/ai/agent/swarm.rb +4 -0
  76. data/lib/pwn/ai/agent/tools/browser_act.rb +32 -0
  77. data/lib/pwn/ai/agent/tools/loot_query.rb +11 -1
  78. data/lib/pwn/ai/agent/tools/recon_run.rb +11 -1
  79. data/lib/pwn/ai/agent/tools/workspace_tool.rb +29 -0
  80. data/lib/pwn/ai/agent/turn_finalizer.rb +6 -2
  81. data/lib/pwn/ai/agent.rb +1 -0
  82. data/lib/pwn/ai/anthropic.rb +8 -33
  83. data/lib/pwn/ai/cli.rb +34 -5
  84. data/lib/pwn/ai/gemini.rb +1 -0
  85. data/lib/pwn/ai/grok.rb +39 -28
  86. data/lib/pwn/ai/ollama.rb +4 -2
  87. data/lib/pwn/ai/open_ai.rb +11 -32
  88. data/lib/pwn/ai/open_web_ui.rb +62 -4
  89. data/lib/pwn/ai/router.rb +8 -1
  90. data/lib/pwn/ai/server_tools.rb +231 -0
  91. data/lib/pwn/ai.rb +1 -0
  92. data/lib/pwn/backups.rb +349 -0
  93. data/lib/pwn/config.rb +70 -39
  94. data/lib/pwn/migrate.rb +85 -280
  95. data/lib/pwn/module_skills.rb +7 -0
  96. data/lib/pwn/plugins/ai_sandbox.rb +538 -1
  97. data/lib/pwn/plugins/authentication_helper.rb +74 -4
  98. data/lib/pwn/plugins/basic_auth.rb +2 -1
  99. data/lib/pwn/plugins/google_workspace.rb +4 -21
  100. data/lib/pwn/plugins/jobs.rb +1 -1
  101. data/lib/pwn/plugins/log.rb +3 -1
  102. data/lib/pwn/plugins/preflight_checker.rb +34 -9
  103. data/lib/pwn/plugins/recon.rb +4 -1
  104. data/lib/pwn/plugins/repl/ai.rb +293 -26
  105. data/lib/pwn/plugins/repl/ai_console.rb +268 -88
  106. data/lib/pwn/plugins/repl/ai_swarm.rb +177 -58
  107. data/lib/pwn/plugins/repl/mesh.rb +14 -19
  108. data/lib/pwn/plugins/repl/vault.rb +4 -15
  109. data/lib/pwn/plugins/repl.rb +9 -216
  110. data/lib/pwn/plugins/sandbox/driver.rb +4 -0
  111. data/lib/pwn/plugins/transparent_browser.rb +215 -0
  112. data/lib/pwn/plugins/vault.rb +412 -135
  113. data/lib/pwn/plugins/yubikey/fido.rb +1288 -0
  114. data/lib/pwn/plugins/yubikey/hsmauth.rb +439 -0
  115. data/lib/pwn/plugins/yubikey/management.rb +686 -0
  116. data/lib/pwn/plugins/yubikey/oath.rb +766 -0
  117. data/lib/pwn/plugins/yubikey/openpgp.rb +1063 -0
  118. data/lib/pwn/plugins/yubikey/otp.rb +897 -0
  119. data/lib/pwn/plugins/yubikey/piv.rb +1315 -0
  120. data/lib/pwn/plugins/yubikey/protocol.rb +341 -0
  121. data/lib/pwn/plugins/yubikey/security_domain/scp.rb +482 -0
  122. data/lib/pwn/plugins/yubikey/security_domain.rb +464 -0
  123. data/lib/pwn/plugins/yubikey/transport.rb +1017 -0
  124. data/lib/pwn/plugins/yubikey.rb +844 -0
  125. data/lib/pwn/plugins.rb +1 -0
  126. data/lib/pwn/redaction.rb +69 -12
  127. data/lib/pwn/session_trace.rb +4 -0
  128. data/lib/pwn/setup.rb +114 -24
  129. data/lib/pwn/version.rb +1 -1
  130. data/lib/pwn/www/app_cobalt_io.rb +16 -5
  131. data/lib/pwn/www/bug_crowd.rb +16 -5
  132. data/lib/pwn/www/coinbase_pro.rb +16 -5
  133. data/lib/pwn/www/github.rb +30 -14
  134. data/lib/pwn/www/paypal.rb +15 -4
  135. data/lib/pwn/www/synack.rb +16 -5
  136. data/lib/pwn/www/twitter.rb +17 -6
  137. data/lib/pwn.rb +1 -0
  138. data/spec/documentation/installation_md_spec.rb +1 -1
  139. data/spec/fixtures/yubikey/apdu_frames.json +162 -0
  140. data/spec/integration/authentication_helper_oath_spec.rb +117 -0
  141. data/spec/integration/controller_spill_spec.rb +48 -0
  142. data/spec/integration/harness_process_spec.rb +97 -0
  143. data/spec/integration/mission_cancel_spec.rb +79 -0
  144. data/spec/integration/mission_curses_pty_spec.rb +44 -0
  145. data/spec/integration/readme_security_spec.rb +47 -0
  146. data/spec/integration/web_cache_deception_yubikey_spec.rb +159 -0
  147. data/spec/lib/pwn/ai/agent/dispatch_spec.rb +6 -3
  148. data/spec/lib/pwn/ai/agent/dispatch_vault_safety_spec.rb +86 -0
  149. data/spec/lib/pwn/ai/agent/injection_guard_spec.rb +2 -0
  150. data/spec/lib/pwn/ai/agent/learning_spec.rb +4 -4
  151. data/spec/lib/pwn/ai/agent/loop_artifacts_spec.rb +5 -0
  152. data/spec/lib/pwn/ai/agent/loop_spec.rb +104 -29
  153. data/spec/lib/pwn/ai/agent/mission_namespace_spec.rb +138 -0
  154. data/spec/lib/pwn/ai/agent/policy_evaluation_spec.rb +10 -23
  155. data/spec/lib/pwn/ai/agent/request_runtime_spec.rb +48 -0
  156. data/spec/lib/pwn/ai/agent/shell_search_effect_spec.rb +44 -0
  157. data/spec/lib/pwn/ai/agent/solve_spec.rb +6 -4
  158. data/spec/lib/pwn/ai/agent/tools/browser_act_spec.rb +44 -0
  159. data/spec/lib/pwn/ai/agent/tools/loot_query_yubikey_spec.rb +55 -0
  160. data/spec/lib/pwn/ai/agent/tools/recon_run_yubikey_spec.rb +43 -0
  161. data/spec/lib/pwn/ai/agent/tools/workspace_tool_spec.rb +20 -0
  162. data/spec/lib/pwn/ai/agent/turn_finalizer_spec.rb +7 -0
  163. data/spec/lib/pwn/ai/anthropic_vault_enrollment_spec.rb +37 -0
  164. data/spec/lib/pwn/ai/cli_spec.rb +8 -16
  165. data/spec/lib/pwn/ai/oauth_vault_persistence_spec.rb +195 -0
  166. data/spec/lib/pwn/ai/open_ai_spec.rb +2 -2
  167. data/spec/lib/pwn/ai/open_web_ui_workspace_spec.rb +36 -0
  168. data/spec/lib/pwn/ai/router_spec.rb +1 -1
  169. data/spec/lib/pwn/ai/server_tools_spec.rb +39 -0
  170. data/spec/lib/pwn/backups_spec.rb +129 -0
  171. data/spec/lib/pwn/config_redaction_yubikey_spec.rb +28 -0
  172. data/spec/lib/pwn/config_yubikey_spec.rb +92 -0
  173. data/spec/lib/pwn/migrate_spec.rb +29 -81
  174. data/spec/lib/pwn/migrate_yubikey_spec.rb +128 -0
  175. data/spec/lib/pwn/module_skills_spec.rb +8 -0
  176. data/spec/lib/pwn/plugins/ai_sandbox_spec.rb +230 -0
  177. data/spec/lib/pwn/plugins/authentication_helper_native_yubikey_spec.rb +40 -0
  178. data/spec/lib/pwn/plugins/authentication_helper_yubikey_spec.rb +90 -0
  179. data/spec/lib/pwn/plugins/capability_broker/daemon_spec.rb +1 -1
  180. data/spec/lib/pwn/plugins/google_workspace_spec.rb +17 -35
  181. data/spec/lib/pwn/plugins/log_yubikey_spec.rb +47 -0
  182. data/spec/lib/pwn/plugins/preflight_checker_native_yubikey_spec.rb +19 -0
  183. data/spec/lib/pwn/plugins/preflight_checker_yubikey_spec.rb +55 -0
  184. data/spec/lib/pwn/plugins/recon_yubikey_spec.rb +43 -0
  185. data/spec/lib/pwn/plugins/repl/ai_console_spec.rb +110 -75
  186. data/spec/lib/pwn/plugins/repl/ai_legacy_containment_spec.rb +200 -0
  187. data/spec/lib/pwn/plugins/repl/ai_swarm_spec.rb +19 -16
  188. data/spec/lib/pwn/plugins/repl/ai_vault_edit_yubikey_spec.rb +103 -0
  189. data/spec/lib/pwn/plugins/repl/ai_yubikey_spec.rb +48 -0
  190. data/spec/lib/pwn/plugins/repl/mesh_vault_persistence_spec.rb +149 -0
  191. data/spec/lib/pwn/plugins/repl/vault_yaml_serialization_spec.rb +11 -0
  192. data/spec/lib/pwn/plugins/repl/vault_yubikey_spec.rb +39 -0
  193. data/spec/lib/pwn/plugins/repl_mesh_layout_spec.rb +2 -0
  194. data/spec/lib/pwn/plugins/repl_spec.rb +56 -5
  195. data/spec/lib/pwn/plugins/transparent_browser_jev_spec.rb +46 -0
  196. data/spec/lib/pwn/plugins/vault_native_yubikey_spec.rb +29 -0
  197. data/spec/lib/pwn/plugins/vault_publication_safety_spec.rb +258 -0
  198. data/spec/lib/pwn/plugins/vault_store_safety_spec.rb +241 -0
  199. data/spec/lib/pwn/plugins/vault_yubikey_safety_spec.rb +168 -0
  200. data/spec/lib/pwn/plugins/vault_yubikey_spec.rb +128 -0
  201. data/spec/lib/pwn/plugins/vault_yubikey_vector_spec.rb +45 -0
  202. data/spec/lib/pwn/plugins/yubikey/fido_spec.rb +854 -0
  203. data/spec/lib/pwn/plugins/yubikey/hsmauth_spec.rb +235 -0
  204. data/spec/lib/pwn/plugins/yubikey/management_spec.rb +277 -0
  205. data/spec/lib/pwn/plugins/yubikey/oath_spec.rb +349 -0
  206. data/spec/lib/pwn/plugins/yubikey/openpgp_spec.rb +810 -0
  207. data/spec/lib/pwn/plugins/yubikey/otp_spec.rb +337 -0
  208. data/spec/lib/pwn/plugins/yubikey/piv_spec.rb +646 -0
  209. data/spec/lib/pwn/plugins/yubikey/protocol_secure_spec.rb +28 -0
  210. data/spec/lib/pwn/plugins/yubikey/protocol_spec.rb +149 -0
  211. data/spec/lib/pwn/plugins/yubikey/protocol_vectors_spec.rb +18 -0
  212. data/spec/lib/pwn/plugins/yubikey/security_domain/scp_spec.rb +256 -0
  213. data/spec/lib/pwn/plugins/yubikey/security_domain/support.rb +153 -0
  214. data/spec/lib/pwn/plugins/yubikey/security_domain_spec.rb +296 -0
  215. data/spec/lib/pwn/plugins/yubikey/transport_spec.rb +760 -0
  216. data/spec/lib/pwn/plugins/yubikey_security_spec.rb +260 -0
  217. data/spec/lib/pwn/plugins/yubikey_spec.rb +209 -0
  218. data/spec/lib/pwn/plugins/yubikey_terminal_spec.rb +147 -0
  219. data/spec/lib/pwn/redaction_yubikey_spec.rb +61 -0
  220. data/spec/lib/pwn/session_cleanup_default_spec.rb +7 -5
  221. data/spec/lib/pwn/session_cleanup_payload_spec.rb +11 -11
  222. data/spec/lib/pwn/session_cleanup_recovery_spec.rb +34 -20
  223. data/spec/lib/pwn/setup_native_yubikey_spec.rb +66 -0
  224. data/spec/lib/pwn/setup_vault_yubikey_spec.rb +61 -0
  225. data/spec/lib/pwn/setup_yubikey_spec.rb +40 -0
  226. data/spec/lib/pwn/www/login_yubikey_spec.rb +273 -0
  227. data/spec/spec_helper.rb +15 -0
  228. data/spec/support/harness/cli_faults.rb +13 -9
  229. data/spec/support/harness/controller_spill.rb +26 -0
  230. data/spec/support/harness/controller_state.rb +54 -0
  231. data/spec/support/harness/evaluate.rb +12 -1
  232. data/spec/support/harness/experiments.rb +1 -1
  233. data/spec/support/harness/fake_provider.rb +22 -6
  234. data/spec/support/harness/fault_preload.rb +3 -1
  235. data/spec/support/harness/mission_console.rb +54 -0
  236. data/spec/support/harness/mission_curses_pty.rb +41 -0
  237. data/spec/support/harness/mission_dispatch.rb +30 -0
  238. data/spec/support/harness/mission_fixture_preload.rb +24 -0
  239. data/spec/support/harness/mission_job_lifetime.rb +25 -0
  240. data/spec/support/harness/mission_legacy.rb +25 -0
  241. data/spec/support/harness/mission_normal_exit.rb +34 -0
  242. data/spec/support/harness/mission_swarm.rb +23 -0
  243. data/spec/support/harness/mission_unsupported.rb +29 -0
  244. data/spec/support/harness/package_acceptance.rb +33 -5
  245. data/spec/support/harness/session_acceptance.rb +4 -4
  246. data/spec/support/harness/swarm_acceptance.rb +1 -1
  247. data/spec/support/harness/swarm_control.rb +44 -0
  248. data/spec/support/mission_namespace.rb +14 -0
  249. data/spec/support/mission_pipe_queue.rb +33 -0
  250. data/third_party/pwn_rdoc.jsonl +605 -19
  251. metadata +129 -16
  252. data/scripts/benchmark_policy.rb +0 -376
  253. data/scripts/build_gate.rb +0 -58
  254. data/spec/integration/build_gate_spec.rb +0 -73
@@ -0,0 +1,273 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+ require 'tty-prompt'
5
+
6
+ RSpec.shared_context 'offline MFA browser' do |selectors|
7
+ let(:browser) { double('offline browser') }
8
+ let(:browser_obj) { { browser: browser } }
9
+ let(:credential_field) { double('credential field', set: nil, present?: true) }
10
+ let(:otp_field) { double('OTP field', present?: true, exist?: true) }
11
+ let(:absent_field) { double('absent field', present?: false, exist?: false) }
12
+ let(:submit) { double('submit control', exist?: true, present?: true) }
13
+ let(:yubikey) { { account: 'Example:tester@example.invalid', confirm: true, serial: '123456', device_path: '/fixture/native-device', timeout: 30 }.freeze }
14
+ let(:login_opts) { { browser_obj: browser_obj, username: 'tester@example.invalid', password: 'fake-password', yubikey: yubikey } }
15
+
16
+ before do
17
+ @authenticated = false
18
+ @otp_written = false
19
+ @form_ready = false
20
+ stub_const('PWN::Plugins::YubiKey', Module.new)
21
+ allow(PWN::Plugins::YubiKey).to receive(:oath_code) do
22
+ raise 'OATH requested before the MFA form was ready' unless @form_ready
23
+
24
+ '012345'
25
+ end
26
+ allow(TTY::Prompt).to receive(:new).and_raise('unexpected interactive prompt')
27
+ allow(browser).to receive(:goto)
28
+ allow(browser).to receive(:url) { @authenticated ? selectors[:success] : selectors[:challenge] }
29
+ allow(browser).to receive(:text_field).and_return(credential_field)
30
+ allow(credential_field).to receive(:wait_until).and_return(credential_field)
31
+ allow(browser).to receive(:text_field).with(selectors[:otp]).and_return(otp_field)
32
+ Array(selectors[:absent]).each do |selector|
33
+ allow(browser).to receive(:text_field).with(selector).and_return(absent_field)
34
+ end
35
+ allow(otp_field).to receive(:wait_until) do
36
+ @form_ready = true
37
+ otp_field
38
+ end
39
+ allow(otp_field).to receive(:set) { @otp_written = true }
40
+ allow(browser).to receive_messages(button: submit, input: submit, span: submit)
41
+ allow(submit).to receive(:wait_until).and_return(submit)
42
+ allow(submit).to receive(:click)
43
+ allow(submit).to receive(:click!) { @authenticated = true if @otp_written }
44
+ end
45
+ end
46
+
47
+ RSpec.shared_examples 'YubiKey browser MFA' do |selectors|
48
+ include_context 'offline MFA browser', selectors
49
+ include_examples 'manual browser MFA'
50
+
51
+ it 'documents explicit consent in the YubiKey example and explains native OATH effects' do
52
+ expect { described_class.help }.to output(/Example yubikey: \{ account: 'EXACT_OATH_NAME', confirm: true.*HOTP.*authenticat/m).to_stdout
53
+ expect { described_class.help }.not_to output(/ykman/).to_stdout
54
+ end
55
+
56
+ [nil, false].each do |mfa|
57
+ it "types a leading-zero OATH code at the MFA form when mfa is #{mfa.inspect}" do
58
+ login_opts[:mfa] = mfa unless mfa.nil?
59
+ expect(otp_field).to receive(:set).with('012345') { @otp_written = true }
60
+ expect(described_class.login(login_opts)).to equal(browser_obj)
61
+ expect(PWN::Plugins::YubiKey).to have_received(:oath_code).with(yubikey).once
62
+ end
63
+ end
64
+
65
+ it 'does not read a YubiKey when login completes without an MFA challenge' do
66
+ @authenticated = true
67
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
68
+ expect(otp_field).not_to receive(:set)
69
+ expect(described_class.login(login_opts)).to equal(browser_obj)
70
+ end
71
+
72
+ it 'does not read a YubiKey if the MFA field never becomes present' do
73
+ allow(otp_field).to receive(:wait_until).and_raise(IOError, 'MFA form unavailable')
74
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
75
+ expect(otp_field).not_to receive(:set)
76
+ expect { described_class.login(login_opts) }.to raise_error(IOError, 'MFA form unavailable')
77
+ end
78
+
79
+ it 'propagates a YubiKey failure without typing a token or prompting' do
80
+ allow(PWN::Plugins::YubiKey).to receive(:oath_code).and_raise(IOError, 'device unavailable')
81
+ expect(otp_field).not_to receive(:set)
82
+ expect { described_class.login(login_opts) }.to raise_error(IOError, 'device unavailable')
83
+ end
84
+ end
85
+
86
+ RSpec.shared_examples 'manual browser MFA' do
87
+ it 'preserves the manual MFA prompt without a YubiKey option' do
88
+ login_opts.delete(:yubikey)
89
+ login_opts[:mfa] = true
90
+ prompt = instance_double(TTY::Prompt)
91
+ allow(TTY::Prompt).to receive(:new).and_return(prompt)
92
+ expect(prompt).to receive(:ask).with('enter mfa token: ').and_return('012345')
93
+ expect(otp_field).to receive(:set).with('012345') { @otp_written = true }
94
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
95
+ expect(described_class.login(login_opts)).to equal(browser_obj)
96
+ end
97
+
98
+ it 'preserves disabled MFA when neither a token source nor mfa is selected' do
99
+ login_opts.delete(:yubikey)
100
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
101
+ expect(otp_field).not_to receive(:set)
102
+ expect(described_class.login(login_opts)).to equal(browser_obj)
103
+ end
104
+ end
105
+
106
+ RSpec.shared_examples 'MFA source validation' do
107
+ [nil, false, 'true'].each do |confirm|
108
+ it "rejects missing or nonliteral OATH consent #{confirm.inspect} before password prompts or browser navigation" do
109
+ login_opts.delete(:password)
110
+ unconfirmed = yubikey.except(:confirm)
111
+ unconfirmed[:confirm] = confirm unless confirm.nil?
112
+ expect(browser).not_to receive(:goto)
113
+ expect(PWN::Plugins::AuthenticationHelper).not_to receive(:mask_password)
114
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
115
+ expect { described_class.login(login_opts.merge(yubikey: unconfirmed)) }.to raise_error(ArgumentError, /confirm: true/)
116
+ end
117
+ end
118
+
119
+ it 'documents the optional YubiKey configuration and explicit-token conflict' do
120
+ expect { described_class.help }.to output(/yubikey:.*exact.*account.*serial.*reader.*device_path.*connection.*timeout.*conflict/m).to_stdout
121
+ end
122
+
123
+ [{ mfa: '654321' }, { mfa_token: '654321' }, { mfa: '654321', mfa_token: ' ' }].each do |conflict|
124
+ it "rejects YubiKey with explicit token #{conflict.keys.inspect} before login or prompting" do
125
+ login_opts.delete(:password)
126
+ expect(browser).not_to receive(:goto)
127
+ expect(PWN::Plugins::AuthenticationHelper).not_to receive(:mask_password)
128
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
129
+ expect { described_class.login(login_opts.merge(conflict)) }.to raise_error(ArgumentError, /yubikey.*(?:mfa|token)/i)
130
+ end
131
+ end
132
+
133
+ it 'rejects malformed YubiKey configuration before submitting a password' do
134
+ expect(browser).not_to receive(:goto)
135
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
136
+ expect { described_class.login(login_opts.merge(yubikey: {})) }.to raise_error(ArgumentError, /account/)
137
+ end
138
+ end
139
+
140
+ RSpec.describe PWN::WWW::GitHub do
141
+ include_examples 'MFA source validation'
142
+ include_examples 'YubiKey browser MFA',
143
+ otp: { name: 'app_otp' },
144
+ challenge: 'https://github.com/sessions/two-factor',
145
+ success: 'https://github.com/'
146
+
147
+ [
148
+ { mfa: '012345' },
149
+ { mfa_token: '012345' },
150
+ { mfa: '654321', mfa_token: '012345' }
151
+ ].each do |token_opts|
152
+ it "preserves explicit tokens and mfa_token precedence for #{token_opts.keys.inspect}" do
153
+ login_opts.delete(:yubikey)
154
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
155
+ expect(otp_field).to receive(:set).with('012345') { @otp_written = true }
156
+ expect(described_class.login(login_opts.merge(token_opts))).to equal(browser_obj)
157
+ end
158
+ end
159
+
160
+ [{ id: 'app_totp' }, { name: 'otp' }].each do |selector|
161
+ it "supports the alternate authenticator field #{selector.inspect}" do
162
+ [{ name: 'app_otp' }, { id: 'app_totp' }, { name: 'otp' }].each do |absent|
163
+ allow(browser).to receive(:text_field).with(absent).and_return(absent_field)
164
+ end
165
+ allow(browser).to receive(:text_field).with(selector).and_return(otp_field)
166
+ expect(otp_field).to receive(:set).with('012345') { @otp_written = true }
167
+ expect(described_class.login(login_opts)).to equal(browser_obj)
168
+ end
169
+ end
170
+
171
+ it 'still accepts explicit SMS tokens without YubiKey' do
172
+ login_opts.delete(:yubikey)
173
+ [{ name: 'app_otp' }, { id: 'app_totp' }, { name: 'otp' }].each do |selector|
174
+ allow(browser).to receive(:text_field).with(selector).and_return(absent_field)
175
+ end
176
+ allow(browser).to receive(:text_field).with(name: 'sms_otp').and_return(otp_field)
177
+ expect(otp_field).to receive(:set).with('012345') { @otp_written = true }
178
+ expect(described_class.login(login_opts.merge(mfa_token: '012345'))).to equal(browser_obj)
179
+ end
180
+
181
+ it 'consumes an explicit token once then prompts again on a repeated MFA form' do
182
+ login_opts.delete(:yubikey)
183
+ prompt = instance_double(TTY::Prompt)
184
+ allow(TTY::Prompt).to receive(:new).and_return(prompt)
185
+ expect(prompt).to receive(:ask).with('enter mfa token: ').once.and_return('012345')
186
+ expect(otp_field).to(receive(:set).with('654321').ordered { @otp_written = false })
187
+ expect(otp_field).to(receive(:set).with('012345').ordered { @otp_written = true })
188
+ expect(described_class.login(login_opts.merge(mfa_token: '654321'))).to equal(browser_obj)
189
+ end
190
+
191
+ it 'refuses to type an OATH code into an SMS-only challenge' do
192
+ %i[app_otp otp].each do |name|
193
+ allow(browser).to receive(:text_field).with(name: name.to_s).and_return(absent_field)
194
+ end
195
+ allow(browser).to receive(:text_field).with(id: 'app_totp').and_return(absent_field)
196
+ allow(browser).to receive(:text_field).with(name: 'sms_otp').and_return(otp_field)
197
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
198
+ expect(otp_field).not_to receive(:set)
199
+ expect { described_class.login(login_opts) }.to raise_error(ArgumentError, /OATH.*authenticator/)
200
+ end
201
+ end
202
+
203
+ RSpec.describe PWN::WWW::Synack do
204
+ include_examples 'MFA source validation'
205
+ include_examples 'YubiKey browser MFA',
206
+ otp: { name: 'authy_token' },
207
+ challenge: 'https://login.synack.com/mfa',
208
+ success: 'https://platform.synack.com'
209
+ end
210
+
211
+ RSpec.describe PWN::WWW::Twitter do
212
+ include_examples 'MFA source validation'
213
+ include_examples 'YubiKey browser MFA',
214
+ otp: { id: 'challenge_response' },
215
+ challenge: 'https://twitter.com/login/challenge',
216
+ success: 'https://twitter.com/home'
217
+ end
218
+
219
+ RSpec.describe PWN::WWW::BugCrowd do
220
+ include_examples 'MFA source validation'
221
+ include_examples 'YubiKey browser MFA',
222
+ otp: { name: 'otp_attempt' },
223
+ challenge: 'https://bugcrowd.com/user/two_factor',
224
+ success: 'https://bugcrowd.com/programs'
225
+ end
226
+
227
+ RSpec.describe PWN::WWW::Paypal do
228
+ include_context 'offline MFA browser',
229
+ otp: { id: 'security-code' },
230
+ challenge: 'https://www.paypal.com/signin/challenge',
231
+ success: 'https://www.paypal.com/myaccount/home'
232
+ include_examples 'MFA source validation'
233
+ include_examples 'manual browser MFA'
234
+
235
+ [nil, false, true].each do |mfa|
236
+ it "rejects unsupported OATH before login or prompting when mfa is #{mfa.inspect}" do
237
+ login_opts.delete(:password)
238
+ login_opts[:mfa] = mfa unless mfa.nil?
239
+ expect(browser).not_to receive(:goto)
240
+ expect(PWN::Plugins::AuthenticationHelper).not_to receive(:mask_password)
241
+ expect(PWN::Plugins::YubiKey).not_to receive(:oath_code)
242
+ expect { described_class.login(login_opts) }.to raise_error(ArgumentError, /PayPal.*OATH.*unsupported.*SMS/)
243
+ end
244
+ end
245
+
246
+ it 'documents the SMS-only adapter limitation instead of claiming TOTP support' do
247
+ expect { described_class.help }.to output(/OATH.*unsupported.*SMS/m).to_stdout
248
+ expect { described_class.help }.to output(/unsupported even with confirm: true/m).to_stdout
249
+ end
250
+
251
+ it 'retains the SMS selection in manual mode' do
252
+ login_opts.delete(:yubikey)
253
+ allow(PWN::Plugins::AuthenticationHelper).to receive(:mfa).and_return('012345')
254
+ expect(browser).to receive(:button).with(id: 'btnSelectSoftToken').and_return(submit)
255
+ expect(described_class.login(login_opts.merge(mfa: true))).to equal(browser_obj)
256
+ end
257
+ end
258
+
259
+ RSpec.describe PWN::WWW::CoinbasePro do
260
+ include_examples 'MFA source validation'
261
+ include_examples 'YubiKey browser MFA',
262
+ otp: { name: 'token' },
263
+ challenge: 'https://www.coinbase.com/signin/challenge',
264
+ success: 'https://pro.coinbase.com'
265
+ end
266
+
267
+ RSpec.describe PWN::WWW::AppCobaltIO do
268
+ include_examples 'MFA source validation'
269
+ include_examples 'YubiKey browser MFA',
270
+ otp: { id: 'code' },
271
+ challenge: 'https://app.cobalt.io/users/two_factor',
272
+ success: 'https://app.cobalt.io/dashboard'
273
+ end
data/spec/spec_helper.rb CHANGED
@@ -35,6 +35,21 @@ RSpec.configure do |config|
35
35
  config.filter_run_excluding :combo_nation_mcp unless ENV['PWN_TEST_COMBO_NATION_MCP'] == '1'
36
36
  config.filter_run_excluding :x86_64_binary unless RbConfig::CONFIG['host_cpu'].to_s.match?(/amd64|x86_64/)
37
37
 
38
+ config.before(:each) do
39
+ @pwn_spec_backup_root = Dir.mktmpdir('pwn-backup-spec-')
40
+ Thread.current[:pwn_backup_real_home] = Dir.home
41
+ Thread.current[:pwn_backup_root] = @pwn_spec_backup_root
42
+ end
43
+
44
+ config.after(:each) do
45
+ root = @pwn_spec_backup_root
46
+ Thread.current[:pwn_backup_root] = nil
47
+ Thread.current[:pwn_backup_real_home] = nil
48
+ @pwn_spec_backup_root = nil
49
+ # FileUtils.remove_entry calls Dir.children, which examples stub with a fixed path.
50
+ system('rm', '-rf', '--', root) if root.is_a?(String) && root.start_with?('/') && root.include?('/pwn-backup-spec-') && !root.include?('..')
51
+ end
52
+
38
53
  if ENV['PWN_RAKE_SPEC'] == '1'
39
54
  module SilenceRakeFilterBanner
40
55
  def report_filter_message(message)
@@ -11,11 +11,11 @@ module HarnessCLIFaults
11
11
  end
12
12
 
13
13
  def self.invoke(root, fixture, prompt, options)
14
- FileUtils.mkdir_p(File.join(root, 'home'))
14
+ FileUtils.mkdir_p("#{root}-home")
15
15
  path = File.join(root, 'provider.json')
16
16
  File.write(path, JSON.generate(fixture))
17
17
  source = options[:source_root] || HarnessEvaluation::ROOT
18
- env = { 'HOME' => File.join(root, 'home'), 'PATH' => HarnessEvaluation.isolated_path, 'LANG' => 'C.UTF-8',
18
+ env = { 'HOME' => "#{root}-home", 'PATH' => HarnessEvaluation.isolated_path, 'LANG' => 'C.UTF-8',
19
19
  'GEM_HOME' => options[:gem_home] || Gem.dir, 'GEM_PATH' => options[:gem_path] || Gem.path.join(File::PATH_SEPARATOR),
20
20
  'PWN_HARNESS_FIXTURE' => path, 'PWN_HARNESS_EVENTS' => File.join(root, 'events.jsonl'),
21
21
  'PWN_HARNESS_FINGERPRINT' => File.join(root, 'fingerprint.json'), 'PWN_HARNESS_RUNTIME' => File.join(root, 'runtime'),
@@ -27,9 +27,9 @@ module HarnessCLIFaults
27
27
  ready_path = fixture[:await_cap_ready] ? env.fetch('PWN_HARNESS_CAP_READY') : nil
28
28
  startup = options[:executable] ? 180 : 45
29
29
  report = if ready_path
30
- HarnessEvaluation.capture(env, command, root, fixture.fetch(:wall_seconds, 35), ready_path: ready_path, startup_seconds: startup)
30
+ HarnessEvaluation.capture(env, command, root, fixture.fetch(:wall_seconds, 120), ready_path: ready_path, startup_seconds: startup)
31
31
  else
32
- HarnessEvaluation.agent_capture(env, command, root, fixture.fetch(:wall_seconds, 35), options)
32
+ HarnessEvaluation.agent_capture(env, command, root, fixture.fetch(:wall_seconds, 120), options)
33
33
  end
34
34
  phase = fixture[:resume] ? 'resumed' : 'initial'
35
35
  File.write(File.join(root, "#{phase}-process.json"), JSON.pretty_generate(report.merge(command: command)))
@@ -66,7 +66,9 @@ module HarnessCLIFaults
66
66
  end
67
67
  result = invoke(root, fixture, prompt, options)
68
68
  first = result.dup
69
- if %w[checkpoint-restart resource-cap-restart].include?(name)
69
+ restart_ready = (name == 'checkpoint-restart' && first[:exit_status] == 73) ||
70
+ (name == 'resource-cap-restart' && first[:resource_cap] && first[:ready_observed])
71
+ if restart_ready
70
72
  fixture = { resume: true, responses: [write, final] }
71
73
  result = invoke(root, fixture, 'continue', options)
72
74
  end
@@ -78,8 +80,8 @@ module HarnessCLIFaults
78
80
  bytes = File.file?(File.join(root, 'result.txt')) ? File.read(File.join(root, 'result.txt')) : nil
79
81
  passed = case name
80
82
  when 'compaction-steering'
81
- result[:exit_status]&.zero? && bytes == "alpha\n" && File.read(File.join(root, 'latest.txt')) == "beta\n" &&
82
- effects == %w[result-write latest-write] && observed.fetch('effective').include?('Also create latest.txt') &&
83
+ result[:exit_status]&.zero? && bytes == "alpha\n" && File.file?(File.join(root, 'latest.txt')) && File.read(File.join(root, 'latest.txt')) == "beta\n" &&
84
+ effects == %w[result-write latest-write] && observed.fetch('effective', '').include?('Also create latest.txt') &&
83
85
  trace.any? { |row| JSON.generate(row['messages']).include?('ENGAGEMENT MEMORY') }
84
86
  when 'provider-retry'
85
87
  result[:exit_status]&.zero? && bytes == "alpha\n" && effects == ['result-write'] && trace.length == 4
@@ -90,7 +92,7 @@ module HarnessCLIFaults
90
92
  (name == 'checkpoint-restart' ? first[:exit_status] == 73 : first[:resource_cap] && first[:ready_observed]) &&
91
93
  result[:exit_status] == 1 && effects == ['uncertain-write'] && bytes.nil? &&
92
94
  observed.dig('outcome', 'status') != 'complete' &&
93
- observed.fetch('events').one? { |row| row['phase'] == 'start' && row['tool'] == 'shell' }
95
+ observed.fetch('events', []).one? { |row| row['phase'] == 'start' && row['tool'] == 'shell' }
94
96
  end
95
97
  outcome = if result[:resource_cap]
96
98
  'INCOMPLETE'
@@ -100,7 +102,9 @@ module HarnessCLIFaults
100
102
  rows << { id: name, outcome: outcome,
101
103
  task_outcome: observed['outcome'], exit_status: result[:exit_status], initial_exit_status: first[:exit_status],
102
104
  initial_resource_cap: first[:resource_cap], initial_outcome: first[:resource_cap] ? 'INCOMPLETE' : nil,
103
- cap_ready_observed: first[:ready_observed], cap_startup_bound_seconds: name == 'resource-cap-restart' ? 35 : nil,
105
+ cap_ready_observed: first[:ready_observed], cap_startup_bound_seconds: name == 'resource-cap-restart' ? first[:startup_bound_seconds] : nil,
106
+ initial_process: first.slice(:exit_status, :signal, :resource_cap, :ready_observed, :timeout_phase),
107
+ final_process: result.slice(:exit_status, :signal, :resource_cap, :ready_observed, :timeout_phase),
104
108
  effects: effects, repeated_effects: effects.tally.values.sum { |n| n - 1 }, provider_calls: trace.length,
105
109
  artifact_root: root, tokens: nil, cost: nil,
106
110
  boundary: 'actual CLI with test-injected public Loop runtime/steering controls; provider fake, tools real' }
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'pwn'
4
+ require 'json'
5
+
6
+ source, work, mode = ARGV
7
+ protected_source = PWN::Plugins::AISandbox.protected_roots
8
+ PWN::Plugins::AISandbox.define_singleton_method(:protected_roots) { [source, *protected_source] }
9
+ runtime = PWN::AI::Agent::RequestRuntime.new(request: 'Create a report', session_id: 'spillboundary', artifact_root: File.join(work, 'state'), trace: false, retrieve: false)
10
+ Thread.current[:pwn_request_runtime] = runtime
11
+ spill = if mode == 'transcript'
12
+ File.join(Dir.home, '.pwn', 'artifacts', 'transcripts')
13
+ else
14
+ File.join(work, 'state', 'spillboundary')
15
+ end
16
+ FileUtils.mkdir_p(spill) if mode == 'transcript'
17
+ PWN::AI::Agent::Registry.register(name: 'spill_boundary_fixture', toolset: 'fixture', schema: { name: 'spill_boundary_fixture', parameters: { type: 'object' } }, handler: lambda { |_args|
18
+ Dir.rmdir(spill) if mode == 'transcript'
19
+ File.symlink(source, spill)
20
+ { redirected: true }
21
+ })
22
+ call = { function: { name: 'spill_boundary_fixture', arguments: '{}' } }
23
+ result = JSON.parse(PWN::AI::Agent::Dispatch.call(tool_call: call))
24
+ runtime.tool_result(call, 'synthetic output ' * 1024)
25
+ PWN::AI::Agent::Loop.send(:spill_tool_body, text: 'synthetic transcript ' * 1024) if mode == 'transcript'
26
+ puts JSON.generate(changed: !Dir.empty?(source), names: Dir.children(source), dispatch_success: result['success'], dispatch_error: JSON.generate(result))
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'pwn'
4
+ require 'json'
5
+
6
+ source, work, mode = ARGV
7
+ original_roots = PWN::Plugins::AISandbox.protected_roots
8
+ PWN::Plugins::AISandbox.define_singleton_method(:protected_roots) { [source, *original_roots] }
9
+ runtime = PWN::AI::Agent::RequestRuntime.new(request: 'Audit controller isolation', session_id: 'stateboundary', artifact_root: File.join(Dir.home, '.pwn', 'requests'), trace: false, retrieve: false)
10
+ Thread.current[:pwn_request_runtime] = runtime
11
+ # Independent finite writer inventory: include stores that do not exist yet,
12
+ # existing state, and writable-looking parent directories beneath HOME.
13
+ directories = %w[sessions artifacts logs learning metrics skills engagements missions runs curriculum finetune]
14
+ files = %w[learning.jsonl lessons.json policy.json policy_traj.jsonl metrics.json mistakes.json preferences.jsonl reward_sentinel.json open_goal.json skill_review.jsonl memory.json memory.archive.json memory.idx curriculum_kpi.jsonl pwn.yaml pwn.yaml.decryptor vault-secrets.json]
15
+ base = File.join(Dir.home, '.pwn')
16
+ directories.each { |name| FileUtils.mkdir_p(File.join(base, name)) }
17
+ if mode == 'aliased'
18
+ path = File.join(base, 'sessions')
19
+ target = File.join(File.dirname(work), 'session-state')
20
+ FileUtils.mv(path, target)
21
+ File.symlink(target, path)
22
+ end
23
+ files.each { |name| File.write(File.join(base, name), '{}') }
24
+ output = File.join(PWN::Plugins::AISandbox.workspace, 'allowed.txt')
25
+ PWN::AI::Agent::Registry.register(name: 'state_boundary_fixture', toolset: 'fixture', schema: { name: 'state_boundary_fixture', parameters: { type: 'object' } }, handler: lambda { |_args|
26
+ attempts = {}
27
+ directories.each do |name|
28
+ path = File.join(base, name, 'redirect')
29
+ attempts[name] = begin
30
+ File.symlink(source, path)
31
+ 'writable'
32
+ rescue Errno::EACCES, Errno::EPERM
33
+ 'denied'
34
+ end
35
+ end
36
+ files.each do |name|
37
+ attempts[name] = begin
38
+ File.write(File.join(base, name), 'mutated')
39
+ 'writable'
40
+ rescue Errno::EACCES, Errno::EPERM
41
+ 'denied'
42
+ end
43
+ end
44
+ attempts['missing-policy-temp'] = begin
45
+ File.symlink(source, File.join(base, '.policy-host-temp'))
46
+ 'writable'
47
+ rescue Errno::EACCES, Errno::EPERM
48
+ 'denied'
49
+ end
50
+ File.write(output, 'output remains writable')
51
+ attempts
52
+ })
53
+ result = JSON.parse(PWN::AI::Agent::Dispatch.call(tool_call: { function: { name: 'state_boundary_fixture', arguments: '{}' } }))
54
+ puts JSON.generate(result: result, source_unchanged: Dir.empty?(source), output: File.read(output))
@@ -74,6 +74,14 @@ module HarnessEvaluation
74
74
  status = nil
75
75
  capped = false
76
76
  ready_observed = false
77
+ phase = ready_path ? 'startup' : 'execution'
78
+ timeout_phase = nil
79
+ # A restart reuses the workspace, not the previous process's readiness.
80
+ FileUtils.rm_f(ready_path) if ready_path
81
+ # Source-tree fixtures need the scratch-built extension. Packaged fixtures
82
+ # use their isolated GEM_HOME and must load their own installed extension.
83
+ env = env.merge('TMPDIR' => ENV['TMPDIR']) if ENV['TMPDIR']
84
+ env = env.merge('RUBYLIB' => ENV['RUBYLIB']) if env['GEM_HOME'] == Gem.dir && ENV['RUBYLIB']
77
85
  Open3.popen3(env, *command, chdir: cwd, unsetenv_others: true, pgroup: true) do |input, output, error, waiter|
78
86
  input.close
79
87
  output.set_encoding(Encoding::UTF_8)
@@ -86,12 +94,14 @@ module HarnessEvaluation
86
94
  end
87
95
  ready_observed = File.file?(ready_path)
88
96
  end
97
+ phase = 'execution' if !ready_path || ready_observed
89
98
  Timeout.timeout(seconds) do
90
99
  status = waiter.value
91
100
  stdout, stderr = readers.map(&:value)
92
101
  end
93
102
  rescue Timeout::Error
94
103
  capped = true
104
+ timeout_phase = phase
95
105
  begin
96
106
  Process.kill('KILL', -waiter.pid)
97
107
  rescue Errno::ESRCH
@@ -103,6 +113,7 @@ module HarnessEvaluation
103
113
  end
104
114
  { stdout: stdout, stderr: stderr, exit_status: status.exitstatus,
105
115
  signal: status.termsig, resource_cap: capped, ready_observed: ready_observed,
116
+ timeout_phase: timeout_phase, startup_bound_seconds: ready_path ? startup_seconds : nil, execution_bound_seconds: seconds,
106
117
  latency_seconds: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started }
107
118
  end
108
119
 
@@ -130,7 +141,7 @@ module HarnessEvaluation
130
141
  ruby = RbConfig.ruby
131
142
  rows = manifest.fetch('tasks').map do |task|
132
143
  root = Dir.mktmpdir("#{task.fetch('id')}-", output_root)
133
- home = File.join(root, 'home')
144
+ home = "#{root}-home"
134
145
  FileUtils.mkdir_p(home)
135
146
  fixture = File.join(root, 'provider.json')
136
147
  File.write(fixture, JSON.generate(responses: task.fetch('responses')))
@@ -28,7 +28,7 @@ module HarnessExperiments
28
28
 
29
29
  def self.trial(task, arm, gold, snapshot, options)
30
30
  root = Dir.mktmpdir("#{task.fetch('id')}-#{arm}-", options.fetch(:output))
31
- home = File.join(root, 'home')
31
+ home = "#{root}-home"
32
32
  FileUtils.mkdir_p(home)
33
33
  FileUtils.cp_r(File.join(snapshot, '.'), home) if snapshot && arm != 'single-cold'
34
34
  # The immutable source snapshot is never mounted as a writable live store.
@@ -3,10 +3,11 @@
3
3
  # Test-only preload: never selected by the production CLI or configuration.
4
4
  require 'json'
5
5
  require 'pwn'
6
+ PWN::AI::Agent::MissionNamespace.bootstrap_requires = [File.expand_path(__FILE__)]
6
7
  require 'net/http'
7
8
  require 'socket'
8
9
 
9
- raise 'isolated HOME required' unless Dir.home.start_with?('/tmp/')
10
+ raise 'isolated HOME required' unless ['/tmp/', "#{ENV.fetch('TMPDIR', '/tmp')}/"].any? { |root| Dir.home.start_with?(root) }
10
11
  raise 'fixture required' unless ENV['PWN_HARNESS_FIXTURE']
11
12
 
12
13
  # Defense in depth: no provider HTTP/socket work in this offline lane.
@@ -43,9 +44,17 @@ PWN::AI::Agent::Dispatch.singleton_class.prepend(Module.new do
43
44
  end
44
45
  end)
45
46
  provider = PWN::AI::OpenAI
46
- calls = 0
47
+ cursor = "#{ENV.fetch('PWN_HARNESS_EVENTS')}.cursor-#{Process.pid}"
47
48
  provider.define_singleton_method(:chat_with_tools) do |opts = {}|
48
- calls += 1
49
+ # Worker isolation must not replay the parent's in-memory fixture cursor.
50
+ calls = File.open(cursor, File::RDWR | File::CREAT, 0o600) do |io|
51
+ io.flock(File::LOCK_EX)
52
+ value = io.read.to_i + 1
53
+ io.rewind
54
+ io.truncate(0)
55
+ io.write(value.to_s)
56
+ value
57
+ end
49
58
  row = fixture.fetch(:responses)[calls - 1]
50
59
  raise "fixture exhausted at provider call #{calls}" unless row
51
60
 
@@ -75,8 +84,15 @@ provider.define_singleton_method(:chat_with_tools) do |opts = {}|
75
84
  { choices: [{ message: row }] }
76
85
  end
77
86
  provider.define_singleton_method(:chat) { |*_args| raise 'unexpected text-only provider path' }
87
+ # Capture actual loaded components before crash/kill stimuli can bypass at_exit.
88
+ # Refresh on normal exit to include modules loaded during execution.
89
+ fingerprint = lambda do
90
+ path = ENV.fetch('PWN_HARNESS_FINGERPRINT')
91
+ stage = "#{path}.#{Process.pid}.tmp"
92
+ File.write(stage, JSON.pretty_generate(PWN::RuntimeFingerprint.report))
93
+ File.rename(stage, path)
94
+ end
95
+ fingerprint.call
96
+ at_exit { fingerprint.call }
78
97
  ready = ENV['PWN_HARNESS_READY'].to_s
79
98
  File.write(ready, 'ready') unless ready.empty?
80
- at_exit do
81
- File.write(ENV.fetch('PWN_HARNESS_FINGERPRINT'), JSON.pretty_generate(PWN::RuntimeFingerprint.report))
82
- end
@@ -5,6 +5,8 @@
5
5
  # replaced. This adapter injects steering/runtime controls as an interactive
6
6
  # caller does; public session selection is tested separately.
7
7
  require_relative 'fake_provider'
8
+ # The exec-backed mission must restore these controls, not only the provider.
9
+ PWN::AI::Agent::MissionNamespace.bootstrap_requires = [File.expand_path(__FILE__)]
8
10
  require 'stringio'
9
11
 
10
12
  fault = JSON.parse(File.read(ENV.fetch('PWN_HARNESS_FIXTURE')), symbolize_names: true)
@@ -19,7 +21,7 @@ PWN::AI::Agent::Loop.singleton_class.prepend(Module.new do
19
21
  end
20
22
  FileUtils.mkdir_p(root)
21
23
  File.write(identity_path, JSON.generate(identity))
22
- runtime_options = identity.merge(artifact_root: root, resume: !!fault[:resume], trace: false, retrieve: false,
24
+ runtime_options = identity.merge(artifact_root: File.join(Dir.home, 'runtime'), resume: !!fault[:resume], trace: false, retrieve: false,
23
25
  window: fault.fetch(:window, 128_000), enabled_toolsets: ['shell'])
24
26
  runtime = PWN::AI::Agent::RequestRuntime.new(runtime_options)
25
27
  control = PWN::AI::Agent::Loop::Steering.new(input: $stdin, output: $stderr)
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'pwn'
4
+ require 'pry'
5
+ require 'json'
6
+ require 'timeout'
7
+
8
+ work = ARGV.fetch(0)
9
+ PWN::AI::Agent::MissionNamespace.force_exec = true
10
+ ENV['PWN_MISSION_FIXTURE_WORK'] = work
11
+ ENV['PWN_MISSION_FIXTURE_COMMAND'] = ARGV[1] == 'command' ? '1' : '0'
12
+ PWN::AI::Agent::MissionNamespace.bootstrap_requires = [File.expand_path('mission_fixture_preload.rb', __dir__)]
13
+ PWN::Env[:ai] = { active: :openai, openai: { model: 'offline' }, agent: {} }
14
+ pry = Pry.new
15
+ pry.config.pwn_ai_session_id = 'offline-cancel'
16
+ input, tool_input = IO.pipe
17
+ console = PWN::Plugins::REPL::AIConsole::Console.new(pry: pry, input: input, curses: nil, getch: nil)
18
+ console.instance_variable_set(:@tool_input, tool_input)
19
+ PWN::Sessions.define_singleton_method(:provisional_title) { |**_opts| nil }
20
+ PWN::Sessions.define_singleton_method(:refresh_title) { |**_opts| nil }
21
+ console.define_singleton_method(:queue_title_update) { |*_args| nil }
22
+ PWN::AI::Agent::Loop.define_singleton_method(:run) do |opts|
23
+ if opts[:request] == 'second'
24
+ 'second completed'
25
+ else
26
+ Process.spawn(RbConfig.ruby, '-e', 'Process.setsid; File.write(ARGV[0], "ready"); sleep 1; File.write(ARGV[1], "late")', File.join(work, 'ready'), File.join(work, 'late'), out: File::NULL, err: File::NULL)
27
+ ARGV[1] == 'command' ? sleep(1.5) : loop { sleep 0.1 }
28
+ end
29
+ end
30
+ begin
31
+ if ARGV[1] == 'command'
32
+ console.start_request('/mcp test', command: true)
33
+ else
34
+ console.start_request('first')
35
+ end
36
+ generation = console.instance_variable_get(:@mission_generation)
37
+ Timeout.timeout(10) { sleep 0.01 until File.exist?(File.join(work, 'ready')) }
38
+ console.cancel
39
+ console.drain
40
+ ready = !console.busy?
41
+ console.start_request('second')
42
+ console.instance_variable_get(:@events) << [:assistant, 'stale generation', generation]
43
+ Timeout.timeout(10) do
44
+ while console.busy?
45
+ console.drain
46
+ sleep 0.01
47
+ end
48
+ end
49
+ sleep 1.2
50
+ rows = console.instance_variable_get(:@timeline)
51
+ puts JSON.generate(ready: ready, late: File.exist?(File.join(work, 'late')), second: rows.any? { |row| row[1].include?('second completed') }, fenced: rows.none? { |row| row[1].include?('stale generation') })
52
+ ensure
53
+ console.stop_worker
54
+ end