arachni 1.4 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (748) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +195 -0
  3. data/Gemfile +4 -4
  4. data/LICENSE.md +1 -1
  5. data/README.md +7 -3
  6. data/Rakefile +1 -43
  7. data/arachni.gemspec +35 -30
  8. data/bin/arachni +1 -1
  9. data/bin/arachni_console +1 -1
  10. data/bin/arachni_multi +6 -1
  11. data/bin/arachni_reporter +1 -1
  12. data/bin/arachni_reproduce +12 -0
  13. data/bin/arachni_rest_server +1 -1
  14. data/bin/arachni_restore +1 -1
  15. data/bin/arachni_rpc +6 -1
  16. data/bin/arachni_rpcd +1 -1
  17. data/bin/arachni_rpcd_monitor +6 -1
  18. data/bin/arachni_script +1 -1
  19. data/components/checks/active/code_injection.rb +1 -1
  20. data/components/checks/active/code_injection_php_input_wrapper.rb +1 -1
  21. data/components/checks/active/code_injection_timing.rb +1 -1
  22. data/components/checks/active/csrf.rb +20 -75
  23. data/components/checks/active/file_inclusion.rb +1 -1
  24. data/components/checks/active/ldap_injection.rb +1 -1
  25. data/components/checks/active/no_sql_injection.rb +1 -1
  26. data/components/checks/active/no_sql_injection_differential.rb +3 -3
  27. data/components/checks/active/os_cmd_injection.rb +1 -1
  28. data/components/checks/active/os_cmd_injection_timing.rb +1 -1
  29. data/components/checks/active/path_traversal.rb +3 -3
  30. data/components/checks/active/response_splitting.rb +1 -1
  31. data/components/checks/active/rfi.rb +1 -1
  32. data/components/checks/active/session_fixation.rb +1 -1
  33. data/components/checks/active/source_code_disclosure.rb +1 -1
  34. data/components/checks/active/sql_injection/regexps/hsqldb.yaml +1 -0
  35. data/components/checks/active/sql_injection/substrings/hsqldb +1 -0
  36. data/components/checks/active/sql_injection/substrings/java +4 -0
  37. data/components/checks/active/sql_injection/substrings/oracle +0 -1
  38. data/components/checks/active/sql_injection/substrings/sqlite +1 -0
  39. data/components/checks/active/sql_injection.rb +1 -1
  40. data/components/checks/active/sql_injection_differential.rb +3 -3
  41. data/components/checks/active/sql_injection_timing.rb +1 -1
  42. data/components/checks/active/trainer.rb +1 -1
  43. data/components/checks/active/unvalidated_redirect.rb +34 -11
  44. data/components/checks/active/unvalidated_redirect_dom.rb +4 -4
  45. data/components/checks/active/xpath_injection.rb +1 -1
  46. data/components/checks/active/xss.rb +54 -29
  47. data/components/checks/active/xss_dom.rb +15 -11
  48. data/components/checks/active/xss_dom_script_context.rb +4 -6
  49. data/components/checks/active/xss_event.rb +46 -34
  50. data/components/checks/active/xss_path.rb +9 -6
  51. data/components/checks/active/xss_script_context.rb +100 -47
  52. data/components/checks/active/xss_tag.rb +41 -15
  53. data/components/checks/active/xxe.rb +1 -1
  54. data/components/checks/passive/allowed_methods.rb +1 -1
  55. data/components/checks/passive/backdoors.rb +1 -1
  56. data/components/checks/passive/backup_directories.rb +15 -3
  57. data/components/checks/passive/backup_files.rb +39 -6
  58. data/components/checks/passive/common_admin_interfaces/admin-panels.txt +1 -0
  59. data/components/checks/passive/common_admin_interfaces.rb +1 -1
  60. data/components/checks/passive/common_directories/directories.txt +1 -0
  61. data/components/checks/passive/common_directories.rb +1 -1
  62. data/components/checks/passive/common_files.rb +1 -1
  63. data/components/checks/passive/directory_listing.rb +1 -1
  64. data/components/checks/passive/grep/captcha.rb +8 -9
  65. data/components/checks/passive/grep/cookie_set_for_parent_domain.rb +1 -1
  66. data/components/checks/passive/grep/credit_card.rb +1 -1
  67. data/components/checks/passive/grep/cvs_svn_users.rb +1 -1
  68. data/components/checks/passive/grep/emails.rb +1 -1
  69. data/components/checks/passive/grep/form_upload.rb +3 -5
  70. data/components/checks/passive/grep/hsts.rb +1 -1
  71. data/components/checks/passive/grep/html_objects.rb +1 -1
  72. data/components/checks/passive/grep/http_only_cookies.rb +1 -1
  73. data/components/checks/passive/grep/insecure_cookies.rb +5 -5
  74. data/components/checks/passive/grep/insecure_cors_policy.rb +1 -1
  75. data/components/checks/passive/grep/mixed_resource.rb +4 -4
  76. data/components/checks/passive/grep/password_autocomplete.rb +1 -1
  77. data/components/checks/passive/grep/private_ip.rb +1 -1
  78. data/components/checks/passive/grep/ssn.rb +1 -1
  79. data/components/checks/passive/grep/unencrypted_password_forms.rb +3 -3
  80. data/components/checks/passive/grep/x_frame_options.rb +4 -4
  81. data/components/checks/passive/htaccess_limit.rb +1 -1
  82. data/components/checks/passive/http_put.rb +1 -1
  83. data/components/checks/passive/insecure_client_access_policy.rb +2 -2
  84. data/components/checks/passive/insecure_cross_domain_policy_access.rb +2 -2
  85. data/components/checks/passive/insecure_cross_domain_policy_headers.rb +2 -2
  86. data/components/checks/passive/interesting_responses.rb +1 -1
  87. data/components/checks/passive/localstart_asp.rb +1 -1
  88. data/components/checks/passive/origin_spoof_access_restriction_bypass.rb +1 -1
  89. data/components/checks/passive/webdav.rb +1 -1
  90. data/components/checks/passive/xst.rb +10 -12
  91. data/components/fingerprinters/frameworks/aspx_mvc.rb +1 -1
  92. data/components/fingerprinters/frameworks/cakephp.rb +1 -1
  93. data/components/fingerprinters/frameworks/cherrypy.rb +1 -1
  94. data/components/fingerprinters/frameworks/django.rb +1 -1
  95. data/components/fingerprinters/frameworks/jsf.rb +1 -1
  96. data/components/fingerprinters/frameworks/nette.rb +1 -1
  97. data/components/fingerprinters/frameworks/rack.rb +1 -1
  98. data/components/fingerprinters/frameworks/rails.rb +1 -1
  99. data/components/fingerprinters/frameworks/symfony.rb +1 -1
  100. data/components/fingerprinters/languages/asp.rb +1 -1
  101. data/components/fingerprinters/languages/aspx.rb +1 -1
  102. data/components/fingerprinters/languages/java.rb +1 -1
  103. data/components/fingerprinters/languages/php.rb +1 -1
  104. data/components/fingerprinters/languages/python.rb +1 -1
  105. data/components/fingerprinters/languages/ruby.rb +1 -1
  106. data/components/fingerprinters/os/bsd.rb +1 -1
  107. data/components/fingerprinters/os/linux.rb +1 -1
  108. data/components/fingerprinters/os/solaris.rb +1 -1
  109. data/components/fingerprinters/os/unix.rb +1 -1
  110. data/components/fingerprinters/os/windows.rb +1 -1
  111. data/components/fingerprinters/servers/apache.rb +1 -1
  112. data/components/fingerprinters/servers/gunicorn.rb +1 -1
  113. data/components/fingerprinters/servers/iis.rb +1 -1
  114. data/components/fingerprinters/servers/jetty.rb +1 -1
  115. data/components/fingerprinters/servers/nginx.rb +1 -1
  116. data/components/fingerprinters/servers/tomcat.rb +1 -1
  117. data/components/path_extractors/anchors.rb +3 -5
  118. data/components/path_extractors/areas.rb +3 -4
  119. data/components/path_extractors/comments.rb +4 -5
  120. data/components/path_extractors/data_url.rb +4 -5
  121. data/components/path_extractors/forms.rb +3 -4
  122. data/components/path_extractors/frames.rb +3 -5
  123. data/components/path_extractors/generic.rb +3 -1
  124. data/components/path_extractors/links.rb +3 -4
  125. data/components/path_extractors/meta_refresh.rb +11 -17
  126. data/components/path_extractors/scripts.rb +18 -15
  127. data/components/plugins/autologin.rb +3 -2
  128. data/components/plugins/beep_notify.rb +1 -1
  129. data/components/plugins/content_types.rb +1 -1
  130. data/components/plugins/cookie_collector.rb +1 -1
  131. data/components/plugins/debug/browser_cluster_job_monitor.rb +60 -0
  132. data/components/plugins/defaults/autothrottle.rb +1 -1
  133. data/components/plugins/defaults/healthmap.rb +3 -1
  134. data/components/plugins/defaults/meta/remedies/discovery.rb +1 -1
  135. data/components/plugins/defaults/meta/remedies/timing_attacks.rb +1 -1
  136. data/components/plugins/defaults/meta/uniformity.rb +1 -1
  137. data/components/plugins/email_notify.rb +26 -9
  138. data/components/plugins/exec.rb +1 -1
  139. data/components/plugins/form_dicattack.rb +3 -4
  140. data/components/plugins/headers_collector.rb +1 -1
  141. data/components/plugins/http_dicattack.rb +4 -5
  142. data/components/plugins/login_script.rb +2 -2
  143. data/components/plugins/metrics.rb +44 -18
  144. data/components/plugins/page_dump.rb +60 -0
  145. data/components/plugins/proxy/panel/verify_login_sequence.html.erb +1 -1
  146. data/components/plugins/proxy/template_scope.rb +6 -1
  147. data/components/plugins/proxy.rb +44 -31
  148. data/components/plugins/rate_limiter.rb +80 -0
  149. data/components/plugins/restrict_to_dom_state.rb +1 -1
  150. data/components/plugins/script.rb +1 -1
  151. data/components/plugins/uncommon_headers.rb +1 -1
  152. data/components/plugins/vector_collector.rb +1 -1
  153. data/components/plugins/vector_feed.rb +1 -1
  154. data/components/plugins/waf_detector.rb +3 -3
  155. data/components/plugins/webhook_notify.rb +99 -0
  156. data/components/reporters/ap.rb +1 -1
  157. data/components/reporters/html/default/configuration.erb +2 -0
  158. data/components/reporters/html/default.erb +3 -2
  159. data/components/reporters/html.rb +5 -8
  160. data/components/reporters/json.rb +1 -1
  161. data/components/reporters/marshal.rb +1 -1
  162. data/components/reporters/plugin_formatters/html/autologin.rb +1 -1
  163. data/components/reporters/plugin_formatters/html/content_types.rb +1 -1
  164. data/components/reporters/plugin_formatters/html/cookie_collector.rb +1 -1
  165. data/components/reporters/plugin_formatters/html/exec.rb +1 -1
  166. data/components/reporters/plugin_formatters/html/form_dicattack.rb +1 -1
  167. data/components/reporters/plugin_formatters/html/healthmap.rb +1 -1
  168. data/components/reporters/plugin_formatters/html/http_dicattack.rb +1 -1
  169. data/components/reporters/plugin_formatters/html/login_script.rb +1 -1
  170. data/components/reporters/plugin_formatters/html/metrics.rb +46 -1
  171. data/components/reporters/plugin_formatters/html/uncommon_headers.rb +1 -1
  172. data/components/reporters/plugin_formatters/html/uniformity.rb +1 -1
  173. data/components/reporters/plugin_formatters/html/vector_collector.rb +1 -1
  174. data/components/reporters/plugin_formatters/html/waf_detector.rb +1 -1
  175. data/components/reporters/plugin_formatters/stdout/autologin.rb +1 -1
  176. data/components/reporters/plugin_formatters/stdout/content_types.rb +1 -1
  177. data/components/reporters/plugin_formatters/stdout/cookie_collector.rb +1 -1
  178. data/components/reporters/plugin_formatters/stdout/exec.rb +1 -1
  179. data/components/reporters/plugin_formatters/stdout/form_dicattack.rb +1 -1
  180. data/components/reporters/plugin_formatters/stdout/healthmap.rb +1 -1
  181. data/components/reporters/plugin_formatters/stdout/http_dicattack.rb +1 -1
  182. data/components/reporters/plugin_formatters/stdout/login_script.rb +1 -1
  183. data/components/reporters/plugin_formatters/stdout/metrics.rb +11 -1
  184. data/components/reporters/plugin_formatters/stdout/uncommon_headers.rb +1 -1
  185. data/components/reporters/plugin_formatters/stdout/uniformity.rb +1 -1
  186. data/components/reporters/plugin_formatters/stdout/vector_collector.rb +1 -1
  187. data/components/reporters/plugin_formatters/stdout/waf_detector.rb +1 -1
  188. data/components/reporters/plugin_formatters/xml/autologin.rb +1 -1
  189. data/components/reporters/plugin_formatters/xml/content_types.rb +10 -7
  190. data/components/reporters/plugin_formatters/xml/cookie_collector.rb +6 -3
  191. data/components/reporters/plugin_formatters/xml/exec.rb +1 -1
  192. data/components/reporters/plugin_formatters/xml/form_dicattack.rb +1 -1
  193. data/components/reporters/plugin_formatters/xml/healthmap.rb +1 -1
  194. data/components/reporters/plugin_formatters/xml/http_dicattack.rb +1 -1
  195. data/components/reporters/plugin_formatters/xml/login_script.rb +1 -1
  196. data/components/reporters/plugin_formatters/xml/metrics.rb +1 -1
  197. data/components/reporters/plugin_formatters/xml/uncommon_headers.rb +5 -2
  198. data/components/reporters/plugin_formatters/xml/uniformity.rb +1 -1
  199. data/components/reporters/plugin_formatters/xml/vector_collector.rb +8 -5
  200. data/components/reporters/plugin_formatters/xml/waf_detector.rb +1 -1
  201. data/components/reporters/stdout.rb +3 -2
  202. data/components/reporters/txt.rb +1 -1
  203. data/components/reporters/xml/schema.xsd +29 -13
  204. data/components/reporters/xml.rb +40 -23
  205. data/components/reporters/yaml.rb +1 -1
  206. data/config/write_paths.yml +4 -0
  207. data/lib/arachni/banner.rb +1 -1
  208. data/lib/arachni/browser/element_locator.rb +9 -5
  209. data/lib/arachni/browser/javascript/dom_monitor.rb +1 -1
  210. data/lib/arachni/browser/javascript/proxy/stub.rb +1 -1
  211. data/lib/arachni/browser/javascript/proxy.rb +1 -1
  212. data/lib/arachni/browser/javascript/scripts/dom_monitor.js +329 -72
  213. data/lib/arachni/browser/javascript/scripts/polyfills.js +0 -28
  214. data/lib/arachni/browser/javascript/scripts/taint_tracer.js +81 -25
  215. data/lib/arachni/browser/javascript/taint_tracer/frame/called_function.rb +1 -1
  216. data/lib/arachni/browser/javascript/taint_tracer/frame.rb +1 -1
  217. data/lib/arachni/browser/javascript/taint_tracer/sink/base.rb +1 -1
  218. data/lib/arachni/browser/javascript/taint_tracer/sink/data_flow.rb +1 -1
  219. data/lib/arachni/browser/javascript/taint_tracer/sink/execution_flow.rb +1 -1
  220. data/lib/arachni/browser/javascript/taint_tracer.rb +1 -1
  221. data/lib/arachni/browser/javascript.rb +111 -198
  222. data/lib/arachni/browser.rb +309 -382
  223. data/lib/arachni/browser_cluster/job/result.rb +1 -1
  224. data/lib/arachni/browser_cluster/job.rb +9 -2
  225. data/lib/arachni/browser_cluster/jobs/browser_provider.rb +8 -2
  226. data/lib/arachni/browser_cluster/jobs/dom_exploration/event_trigger/result.rb +1 -1
  227. data/lib/arachni/browser_cluster/jobs/dom_exploration/event_trigger.rb +1 -1
  228. data/lib/arachni/browser_cluster/jobs/dom_exploration/result.rb +1 -1
  229. data/lib/arachni/browser_cluster/jobs/dom_exploration.rb +13 -1
  230. data/lib/arachni/browser_cluster/jobs/taint_trace/event_trigger/result.rb +1 -1
  231. data/lib/arachni/browser_cluster/jobs/taint_trace/event_trigger.rb +1 -1
  232. data/lib/arachni/browser_cluster/jobs/taint_trace/result.rb +1 -1
  233. data/lib/arachni/browser_cluster/jobs/taint_trace.rb +1 -1
  234. data/lib/arachni/browser_cluster/worker.rb +97 -87
  235. data/lib/arachni/browser_cluster.rb +79 -62
  236. data/lib/arachni/check/auditor.rb +161 -155
  237. data/lib/arachni/check/base.rb +1 -1
  238. data/lib/arachni/check/manager.rb +1 -1
  239. data/lib/arachni/check.rb +1 -1
  240. data/lib/arachni/component/base.rb +3 -1
  241. data/lib/arachni/component/manager.rb +1 -1
  242. data/lib/arachni/component/options/address.rb +1 -1
  243. data/lib/arachni/component/options/base.rb +1 -1
  244. data/lib/arachni/component/options/bool.rb +1 -1
  245. data/lib/arachni/component/options/float.rb +1 -1
  246. data/lib/arachni/component/options/int.rb +1 -1
  247. data/lib/arachni/component/options/multiple_choice.rb +1 -1
  248. data/lib/arachni/component/options/object.rb +1 -1
  249. data/lib/arachni/component/options/path.rb +1 -1
  250. data/lib/arachni/component/options/port.rb +1 -1
  251. data/lib/arachni/component/options/string.rb +1 -1
  252. data/lib/arachni/component/options/url.rb +1 -1
  253. data/lib/arachni/component/options.rb +1 -1
  254. data/lib/arachni/component/output.rb +8 -2
  255. data/lib/arachni/component/utilities.rb +1 -1
  256. data/lib/arachni/component.rb +1 -1
  257. data/lib/arachni/data/framework/rpc.rb +2 -2
  258. data/lib/arachni/data/framework.rb +3 -2
  259. data/lib/arachni/data/issues.rb +1 -1
  260. data/lib/arachni/data/plugins.rb +1 -1
  261. data/lib/arachni/data/session.rb +1 -1
  262. data/lib/arachni/data.rb +1 -1
  263. data/lib/arachni/element/base.rb +1 -1
  264. data/lib/arachni/element/body.rb +1 -1
  265. data/lib/arachni/element/capabilities/analyzable/differential.rb +142 -175
  266. data/lib/arachni/element/capabilities/analyzable/signature.rb +40 -18
  267. data/lib/arachni/element/capabilities/analyzable/timeout.rb +1 -1
  268. data/lib/arachni/element/capabilities/analyzable.rb +1 -1
  269. data/lib/arachni/element/capabilities/auditable/buffered.rb +92 -0
  270. data/lib/arachni/element/capabilities/auditable/line_buffered.rb +103 -0
  271. data/lib/arachni/element/capabilities/auditable.rb +2 -8
  272. data/lib/arachni/element/capabilities/dom_only.rb +1 -1
  273. data/lib/arachni/element/capabilities/inputtable.rb +6 -2
  274. data/lib/arachni/element/capabilities/mutable.rb +1 -1
  275. data/lib/arachni/element/capabilities/refreshable.rb +1 -1
  276. data/lib/arachni/element/capabilities/submittable.rb +1 -1
  277. data/lib/arachni/element/capabilities/with_auditor/output.rb +4 -3
  278. data/lib/arachni/element/capabilities/with_auditor.rb +1 -1
  279. data/lib/arachni/element/capabilities/with_dom.rb +1 -1
  280. data/lib/arachni/element/capabilities/with_node.rb +3 -3
  281. data/lib/arachni/element/capabilities/with_scope/scope.rb +1 -1
  282. data/lib/arachni/element/capabilities/with_scope.rb +1 -1
  283. data/lib/arachni/element/capabilities/with_source.rb +2 -2
  284. data/lib/arachni/element/cookie/capabilities/inputtable.rb +1 -1
  285. data/lib/arachni/element/cookie/capabilities/mutable.rb +1 -1
  286. data/lib/arachni/element/cookie/capabilities/with_dom.rb +1 -1
  287. data/lib/arachni/element/cookie/dom.rb +1 -1
  288. data/lib/arachni/element/cookie.rb +49 -24
  289. data/lib/arachni/element/dom/capabilities/auditable.rb +44 -3
  290. data/lib/arachni/element/dom/capabilities/inputtable.rb +1 -1
  291. data/lib/arachni/element/dom/capabilities/locatable.rb +1 -1
  292. data/lib/arachni/element/dom/capabilities/mutable.rb +7 -3
  293. data/lib/arachni/element/dom/capabilities/submittable.rb +51 -22
  294. data/lib/arachni/element/dom.rb +1 -1
  295. data/lib/arachni/element/form/capabilities/auditable.rb +1 -1
  296. data/lib/arachni/element/form/capabilities/mutable.rb +16 -11
  297. data/lib/arachni/element/form/capabilities/submittable.rb +1 -1
  298. data/lib/arachni/element/form/capabilities/with_dom.rb +1 -1
  299. data/lib/arachni/element/form/dom.rb +1 -1
  300. data/lib/arachni/element/form.rb +21 -32
  301. data/lib/arachni/element/generic_dom.rb +1 -1
  302. data/lib/arachni/element/header/capabilities/inputtable.rb +1 -1
  303. data/lib/arachni/element/header/capabilities/mutable.rb +1 -1
  304. data/lib/arachni/element/header.rb +3 -1
  305. data/lib/arachni/element/json/capabilities/inputtable.rb +1 -1
  306. data/lib/arachni/element/json/capabilities/mutable.rb +1 -1
  307. data/lib/arachni/element/json.rb +4 -8
  308. data/lib/arachni/element/link/capabilities/auditable.rb +1 -1
  309. data/lib/arachni/element/link/capabilities/submittable.rb +1 -1
  310. data/lib/arachni/element/link/capabilities/with_dom.rb +1 -1
  311. data/lib/arachni/element/link/dom/capabilities/submittable.rb +1 -1
  312. data/lib/arachni/element/link/dom.rb +1 -1
  313. data/lib/arachni/element/link.rb +11 -30
  314. data/lib/arachni/element/link_template/capabilities/auditable.rb +1 -1
  315. data/lib/arachni/element/link_template/capabilities/inputtable.rb +1 -1
  316. data/lib/arachni/element/link_template/capabilities/with_dom.rb +1 -1
  317. data/lib/arachni/element/link_template/dom/capabilities/submittable.rb +1 -1
  318. data/lib/arachni/element/link_template/dom.rb +2 -2
  319. data/lib/arachni/element/link_template.rb +10 -19
  320. data/lib/arachni/element/nested_cookie/capabilities/submittable.rb +35 -0
  321. data/lib/arachni/element/nested_cookie.rb +370 -0
  322. data/lib/arachni/element/path.rb +1 -1
  323. data/lib/arachni/element/server.rb +11 -11
  324. data/lib/arachni/element/ui_form/dom.rb +1 -1
  325. data/lib/arachni/element/ui_form.rb +5 -6
  326. data/lib/arachni/element/ui_input/dom.rb +1 -1
  327. data/lib/arachni/element/ui_input.rb +4 -6
  328. data/lib/arachni/element/xml/capabilities/inputtable.rb +1 -1
  329. data/lib/arachni/element/xml/capabilities/mutable.rb +1 -1
  330. data/lib/arachni/element/xml.rb +3 -7
  331. data/lib/arachni/element_filter.rb +1 -1
  332. data/lib/arachni/error.rb +1 -1
  333. data/lib/arachni/ethon/easy.rb +1 -1
  334. data/lib/arachni/framework/parts/audit.rb +6 -1
  335. data/lib/arachni/framework/parts/browser.rb +14 -14
  336. data/lib/arachni/framework/parts/check.rb +1 -1
  337. data/lib/arachni/framework/parts/data.rb +1 -1
  338. data/lib/arachni/framework/parts/platform.rb +1 -1
  339. data/lib/arachni/framework/parts/plugin.rb +1 -1
  340. data/lib/arachni/framework/parts/report.rb +3 -3
  341. data/lib/arachni/framework/parts/scope.rb +1 -1
  342. data/lib/arachni/framework/parts/state.rb +1 -1
  343. data/lib/arachni/framework.rb +1 -1
  344. data/lib/arachni/http/client/dynamic_404_handler.rb +74 -16
  345. data/lib/arachni/http/client.rb +38 -11
  346. data/lib/arachni/http/cookie_jar.rb +13 -8
  347. data/lib/arachni/http/headers.rb +11 -5
  348. data/lib/arachni/http/message/scope.rb +1 -1
  349. data/lib/arachni/http/message.rb +10 -9
  350. data/lib/arachni/http/proxy_server/connection.rb +110 -82
  351. data/lib/arachni/http/proxy_server/ssl-interceptor-cacert.pem +18 -32
  352. data/lib/arachni/http/proxy_server/ssl-interceptor-cakey.pem +28 -49
  353. data/lib/arachni/http/proxy_server/ssl_interceptor.rb +8 -6
  354. data/lib/arachni/http/proxy_server/tunnel.rb +4 -4
  355. data/lib/arachni/http/proxy_server.rb +44 -11
  356. data/lib/arachni/http/request/scope.rb +1 -1
  357. data/lib/arachni/http/request.rb +239 -41
  358. data/lib/arachni/http/response/scope.rb +1 -1
  359. data/lib/arachni/http/response.rb +73 -10
  360. data/lib/arachni/http.rb +1 -1
  361. data/lib/arachni/issue/severity/base.rb +1 -1
  362. data/lib/arachni/issue/severity.rb +1 -1
  363. data/lib/arachni/issue.rb +42 -14
  364. data/lib/arachni/option_group.rb +1 -1
  365. data/lib/arachni/option_groups/audit.rb +11 -2
  366. data/lib/arachni/option_groups/browser_cluster.rb +32 -4
  367. data/lib/arachni/option_groups/datastore.rb +1 -1
  368. data/lib/arachni/option_groups/dispatcher.rb +1 -1
  369. data/lib/arachni/option_groups/http.rb +39 -10
  370. data/lib/arachni/option_groups/input.rb +1 -1
  371. data/lib/arachni/option_groups/output.rb +1 -1
  372. data/lib/arachni/option_groups/paths.rb +12 -1
  373. data/lib/arachni/option_groups/rpc.rb +1 -1
  374. data/lib/arachni/option_groups/scope.rb +58 -4
  375. data/lib/arachni/option_groups/session.rb +1 -1
  376. data/lib/arachni/option_groups/snapshot.rb +1 -1
  377. data/lib/arachni/option_groups.rb +1 -1
  378. data/lib/arachni/options.rb +23 -4
  379. data/lib/arachni/page/dom/transition.rb +5 -2
  380. data/lib/arachni/page/dom.rb +46 -54
  381. data/lib/arachni/page/scope.rb +1 -1
  382. data/lib/arachni/page.rb +10 -8
  383. data/lib/arachni/parser/document.rb +34 -0
  384. data/lib/arachni/parser/extractors/base.rb +48 -0
  385. data/lib/arachni/parser/nodes/base.rb +22 -0
  386. data/lib/arachni/parser/nodes/comment.rb +32 -0
  387. data/lib/arachni/parser/nodes/element/with_attributes/attributes.rb +31 -0
  388. data/lib/arachni/parser/nodes/element/with_attributes.rb +35 -0
  389. data/lib/arachni/parser/nodes/element.rb +48 -0
  390. data/lib/arachni/parser/nodes/text.rb +32 -0
  391. data/lib/arachni/parser/nodes/with_value.rb +29 -0
  392. data/lib/arachni/parser/sax.rb +76 -0
  393. data/lib/arachni/parser/with_children/search.rb +92 -0
  394. data/lib/arachni/parser/with_children.rb +35 -0
  395. data/lib/arachni/parser.rb +181 -78
  396. data/lib/arachni/platform/fingerprinter.rb +1 -1
  397. data/lib/arachni/platform/list.rb +1 -1
  398. data/lib/arachni/platform/manager.rb +2 -2
  399. data/lib/arachni/platform.rb +1 -1
  400. data/lib/arachni/plugin/base.rb +2 -2
  401. data/lib/arachni/plugin/formatter.rb +1 -1
  402. data/lib/arachni/plugin/manager.rb +8 -5
  403. data/lib/arachni/plugin.rb +1 -1
  404. data/lib/arachni/processes/dispatchers.rb +1 -1
  405. data/lib/arachni/processes/executables/base.rb +2 -1
  406. data/lib/arachni/processes/executables/browser.rb +0 -2
  407. data/lib/arachni/processes/helpers/dispatchers.rb +1 -1
  408. data/lib/arachni/processes/helpers/instances.rb +1 -1
  409. data/lib/arachni/processes/helpers/processes.rb +1 -1
  410. data/lib/arachni/processes/helpers.rb +1 -1
  411. data/lib/arachni/processes/instances.rb +1 -1
  412. data/lib/arachni/processes/manager.rb +18 -9
  413. data/lib/arachni/processes.rb +1 -1
  414. data/lib/arachni/report.rb +8 -1
  415. data/lib/arachni/reporter/base.rb +1 -1
  416. data/lib/arachni/reporter/formatter_manager.rb +1 -1
  417. data/lib/arachni/reporter/manager.rb +1 -1
  418. data/lib/arachni/reporter/options.rb +1 -10
  419. data/lib/arachni/reporter.rb +1 -1
  420. data/lib/arachni/rest/server/instance_helpers.rb +10 -1
  421. data/lib/arachni/rest/server.rb +13 -1
  422. data/lib/arachni/rpc/client/base.rb +1 -1
  423. data/lib/arachni/rpc/client/dispatcher.rb +1 -1
  424. data/lib/arachni/rpc/client/instance/framework.rb +1 -1
  425. data/lib/arachni/rpc/client/instance/service.rb +1 -1
  426. data/lib/arachni/rpc/client/instance.rb +1 -1
  427. data/lib/arachni/rpc/serializer.rb +1 -1
  428. data/lib/arachni/rpc/server/active_options.rb +1 -1
  429. data/lib/arachni/rpc/server/base.rb +1 -1
  430. data/lib/arachni/rpc/server/check/manager.rb +1 -1
  431. data/lib/arachni/rpc/server/dispatcher/node.rb +1 -1
  432. data/lib/arachni/rpc/server/dispatcher/service.rb +1 -1
  433. data/lib/arachni/rpc/server/dispatcher.rb +1 -1
  434. data/lib/arachni/rpc/server/framework/distributor.rb +1 -1
  435. data/lib/arachni/rpc/server/framework/master.rb +1 -1
  436. data/lib/arachni/rpc/server/framework/multi_instance.rb +1 -1
  437. data/lib/arachni/rpc/server/framework/slave.rb +1 -1
  438. data/lib/arachni/rpc/server/framework.rb +1 -1
  439. data/lib/arachni/rpc/server/instance.rb +1 -1
  440. data/lib/arachni/rpc/server/output.rb +1 -1
  441. data/lib/arachni/rpc/server/plugin/manager.rb +1 -1
  442. data/lib/arachni/ruby/array.rb +1 -1
  443. data/lib/arachni/ruby/hash.rb +1 -1
  444. data/lib/arachni/ruby/object.rb +1 -1
  445. data/lib/arachni/ruby/set.rb +1 -1
  446. data/lib/arachni/ruby/string.rb +9 -5
  447. data/lib/arachni/ruby/webrick/cookie.rb +1 -1
  448. data/lib/arachni/ruby/webrick/httprequest.rb +1 -1
  449. data/lib/arachni/ruby/webrick.rb +1 -1
  450. data/lib/arachni/ruby.rb +1 -1
  451. data/lib/arachni/scope.rb +1 -1
  452. data/lib/arachni/selenium/webdriver/element.rb +4 -4
  453. data/lib/arachni/selenium/webdriver/remote/typhoeus.rb +59 -0
  454. data/lib/arachni/session.rb +32 -13
  455. data/lib/arachni/snapshot.rb +2 -2
  456. data/lib/arachni/state/audit.rb +1 -1
  457. data/lib/arachni/state/element_filter.rb +1 -1
  458. data/lib/arachni/state/framework/rpc.rb +1 -1
  459. data/lib/arachni/state/framework.rb +1 -1
  460. data/lib/arachni/state/http.rb +2 -2
  461. data/lib/arachni/state/options.rb +1 -1
  462. data/lib/arachni/state/plugins.rb +1 -1
  463. data/lib/arachni/state.rb +1 -1
  464. data/lib/arachni/support/buffer/autoflush.rb +1 -1
  465. data/lib/arachni/support/buffer/base.rb +1 -1
  466. data/lib/arachni/support/buffer.rb +1 -1
  467. data/lib/arachni/support/cache/base.rb +1 -1
  468. data/lib/arachni/support/cache/least_cost_replacement.rb +1 -1
  469. data/lib/arachni/support/cache/least_recently_pushed.rb +1 -1
  470. data/lib/arachni/support/cache/least_recently_used.rb +1 -1
  471. data/lib/arachni/support/cache/preference.rb +1 -1
  472. data/lib/arachni/support/cache/random_replacement.rb +1 -1
  473. data/lib/arachni/support/cache.rb +1 -1
  474. data/lib/arachni/support/crypto/rsa_aes_cbc.rb +1 -1
  475. data/lib/arachni/support/crypto.rb +1 -1
  476. data/lib/arachni/support/database/base.rb +16 -10
  477. data/lib/arachni/support/database/hash.rb +1 -1
  478. data/lib/arachni/support/database/queue.rb +1 -1
  479. data/lib/arachni/support/database.rb +1 -1
  480. data/lib/arachni/support/glob.rb +1 -1
  481. data/lib/arachni/support/lookup/base.rb +1 -1
  482. data/lib/arachni/support/lookup/hash_set.rb +1 -1
  483. data/lib/arachni/support/lookup/moolb.rb +1 -1
  484. data/lib/arachni/support/lookup.rb +1 -1
  485. data/lib/arachni/support/mixins/observable.rb +1 -1
  486. data/lib/arachni/support/mixins/terminal.rb +1 -1
  487. data/lib/arachni/support/mixins.rb +1 -1
  488. data/lib/arachni/support/profiler.rb +52 -13
  489. data/lib/arachni/support/signature.rb +18 -6
  490. data/lib/arachni/support.rb +1 -1
  491. data/lib/arachni/trainer.rb +55 -39
  492. data/lib/arachni/ui/foo/output.rb +1 -1
  493. data/lib/arachni/uri/scope.rb +15 -13
  494. data/lib/arachni/uri.rb +129 -103
  495. data/lib/arachni/utilities.rb +10 -10
  496. data/lib/arachni/version.rb +1 -1
  497. data/lib/arachni.rb +1 -7
  498. data/lib/version +1 -1
  499. data/spec/arachni/browser/element_locator_spec.rb +42 -18
  500. data/spec/arachni/browser/javascript/dom_monitor_spec.rb +264 -109
  501. data/spec/arachni/browser/javascript/polyfills_spec.rb +0 -15
  502. data/spec/arachni/browser/javascript/proxy_spec.rb +0 -10
  503. data/spec/arachni/browser/javascript/taint_tracer_spec.rb +43 -118
  504. data/spec/arachni/browser/javascript_spec.rb +95 -60
  505. data/spec/arachni/browser_cluster/job_spec.rb +23 -8
  506. data/spec/arachni/browser_cluster/jobs/dom_exploration_spec.rb +6 -1
  507. data/spec/arachni/browser_cluster/worker_spec.rb +29 -87
  508. data/spec/arachni/browser_cluster_spec.rb +124 -43
  509. data/spec/arachni/browser_spec.rb +463 -421
  510. data/spec/arachni/check/auditor_spec.rb +162 -198
  511. data/spec/arachni/data/framework/rpc_spec.rb +1 -1
  512. data/spec/arachni/data/framework_spec.rb +1 -1
  513. data/spec/arachni/element/capabilities/analyzable/signature_spec.rb +46 -3
  514. data/spec/arachni/element/cookie/dom_spec.rb +1 -1
  515. data/spec/arachni/element/cookie_spec.rb +159 -64
  516. data/spec/arachni/element/form/dom_spec.rb +1 -1
  517. data/spec/arachni/element/form_spec.rb +101 -54
  518. data/spec/arachni/element/header_spec.rb +3 -1
  519. data/spec/arachni/element/json_spec.rb +2 -0
  520. data/spec/arachni/element/link/dom_spec.rb +2 -2
  521. data/spec/arachni/element/link_spec.rb +46 -15
  522. data/spec/arachni/element/link_template/dom_spec.rb +1 -1
  523. data/spec/arachni/element/link_template_spec.rb +36 -12
  524. data/spec/arachni/element/nested_cookie_spec.rb +687 -0
  525. data/spec/arachni/element/server_spec.rb +22 -5
  526. data/spec/arachni/element/ui_form/dom_spec.rb +1 -1
  527. data/spec/arachni/element/ui_form_spec.rb +2 -2
  528. data/spec/arachni/element/ui_input/dom_spec.rb +1 -1
  529. data/spec/arachni/element/ui_input_spec.rb +1 -1
  530. data/spec/arachni/element/xml_spec.rb +5 -3
  531. data/spec/arachni/framework/parts/audit_spec.rb +2 -14
  532. data/spec/arachni/framework/parts/data_spec.rb +0 -6
  533. data/spec/arachni/http/client/dynamic_404_handlers_spec.rb +126 -0
  534. data/spec/arachni/http/client_spec.rb +96 -36
  535. data/spec/arachni/http/cookie_jar_spec.rb +2 -2
  536. data/spec/arachni/http/headers_spec.rb +59 -12
  537. data/spec/arachni/http/proxy_server_spec.rb +58 -25
  538. data/spec/arachni/http/request_spec.rb +382 -35
  539. data/spec/arachni/http/response_spec.rb +135 -7
  540. data/spec/arachni/issue_spec.rb +21 -2
  541. data/spec/arachni/option_groups/browser_cluster_spec.rb +17 -0
  542. data/spec/arachni/option_groups/http_spec.rb +21 -6
  543. data/spec/arachni/option_groups/paths_spec.rb +23 -1
  544. data/spec/arachni/option_groups/scope_spec.rb +27 -7
  545. data/spec/arachni/options_spec.rb +8 -1
  546. data/spec/arachni/page/dom_spec.rb +20 -6
  547. data/spec/arachni/page_spec.rb +8 -7
  548. data/spec/arachni/parser/document_spec.rb +49 -0
  549. data/spec/arachni/parser/nodes/comment_spec.rb +24 -0
  550. data/spec/arachni/parser/nodes/element/with_attributes/attributes_spec.rb +40 -0
  551. data/spec/arachni/parser/nodes/element/with_attributes_spec.rb +50 -0
  552. data/spec/arachni/parser/nodes/element_spec.rb +18 -0
  553. data/spec/arachni/parser/nodes/text_spec.rb +24 -0
  554. data/spec/arachni/parser/sax_spec.rb +88 -0
  555. data/spec/arachni/parser/with_children/search_spec.rb +146 -0
  556. data/spec/arachni/parser/with_children_spec.rb +37 -0
  557. data/spec/arachni/parser_spec.rb +211 -27
  558. data/spec/arachni/platform/list_spec.rb +1 -2
  559. data/spec/arachni/report_spec.rb +9 -2
  560. data/spec/arachni/reporter/options_spec.rb +0 -14
  561. data/spec/arachni/rest/server_spec.rb +91 -8
  562. data/spec/arachni/rpc/server/active_options_spec.rb +1 -1
  563. data/spec/arachni/rpc/server/framework/distributor_spec.rb +6 -6
  564. data/spec/arachni/ruby/string_spec.rb +6 -0
  565. data/spec/arachni/session_spec.rb +69 -8
  566. data/spec/arachni/snapshot_spec.rb +1 -1
  567. data/spec/arachni/state/framework_spec.rb +2 -2
  568. data/spec/arachni/support/signature_spec.rb +58 -0
  569. data/spec/arachni/trainer_spec.rb +102 -21
  570. data/spec/arachni/uri_spec.rb +11 -8
  571. data/spec/arachni/utilities_spec.rb +3 -3
  572. data/spec/components/checks/active/code_injection_spec.rb +12 -7
  573. data/spec/components/checks/active/code_injection_timing_spec.rb +4 -3
  574. data/spec/components/checks/active/csrf_spec.rb +1 -21
  575. data/spec/components/checks/active/file_inclusion_spec.rb +15 -10
  576. data/spec/components/checks/active/ldap_injection_spec.rb +5 -4
  577. data/spec/components/checks/active/no_sql_injection_differential_spec.rb +1 -1
  578. data/spec/components/checks/active/no_sql_injection_spec.rb +5 -4
  579. data/spec/components/checks/active/os_cmd_injection_spec.rb +6 -4
  580. data/spec/components/checks/active/os_cmd_injection_timing_spec.rb +4 -3
  581. data/spec/components/checks/active/path_traversal_spec.rb +18 -15
  582. data/spec/components/checks/active/response_splitting_spec.rb +5 -4
  583. data/spec/components/checks/active/rfi_spec.rb +9 -8
  584. data/spec/components/checks/active/source_code_disclosure_spec.rb +33 -10
  585. data/spec/components/checks/active/sql_injection_differential_spec.rb +1 -1
  586. data/spec/components/checks/active/sql_injection_spec.rb +61 -35
  587. data/spec/components/checks/active/sql_injection_timing_spec.rb +11 -8
  588. data/spec/components/checks/active/unvalidated_redirect_spec.rb +9 -8
  589. data/spec/components/checks/active/xpath_injection_spec.rb +5 -4
  590. data/spec/components/checks/active/xss_dom_script_context_spec.rb +6 -10
  591. data/spec/components/checks/active/xss_dom_spec.rb +2 -2
  592. data/spec/components/checks/active/xss_event_spec.rb +11 -3
  593. data/spec/components/checks/active/xss_script_context_spec.rb +8 -7
  594. data/spec/components/checks/active/xss_spec.rb +7 -6
  595. data/spec/components/checks/active/xss_tag_spec.rb +11 -3
  596. data/spec/components/checks/passive/backup_directories_spec.rb +3 -1
  597. data/spec/components/checks/passive/backup_files_spec.rb +4 -1
  598. data/spec/components/checks/passive/grep/insecure_cookies_spec.rb +2 -2
  599. data/spec/components/checks/passive/grep/x_frame_options_spec.rb +6 -0
  600. data/spec/components/path_extractors/comments_spec.rb +3 -1
  601. data/spec/components/path_extractors/data_url_spec.rb +6 -2
  602. data/spec/components/path_extractors/links_spec.rb +1 -1
  603. data/spec/components/plugins/autologin_spec.rb +2 -2
  604. data/spec/components/plugins/webhook_notify_spec.rb +69 -0
  605. data/spec/spec_helper.rb +2 -1
  606. data/spec/support/factories/http/response.rb +1 -1
  607. data/spec/support/factories/issue.rb +1 -2
  608. data/spec/support/factories/page/dom.rb +6 -0
  609. data/spec/support/factories/scan_report.rb +1 -0
  610. data/spec/support/factories/vector.rb +7 -3
  611. data/spec/support/fixtures/check_with_invalid_platforms/with_invalid_platforms.rb +1 -1
  612. data/spec/support/fixtures/checks/test.rb +4 -4
  613. data/spec/support/fixtures/checks/test2.rb +1 -1
  614. data/spec/support/fixtures/checks/test3.rb +1 -1
  615. data/spec/support/fixtures/cookies.txt +2 -2
  616. data/spec/support/fixtures/executables/node.rb +2 -3
  617. data/spec/support/fixtures/fingerprinters/test.rb +1 -1
  618. data/spec/support/fixtures/nested_cookies.txt +11 -0
  619. data/spec/support/fixtures/plugins/bad.rb +1 -1
  620. data/spec/support/fixtures/plugins/defaults/default.rb +1 -1
  621. data/spec/support/fixtures/plugins/distributable.rb +1 -1
  622. data/spec/support/fixtures/plugins/loop.rb +1 -1
  623. data/spec/support/fixtures/plugins/suspendable.rb +1 -1
  624. data/spec/support/fixtures/plugins/wait.rb +1 -1
  625. data/spec/support/fixtures/plugins/with_options.rb +1 -1
  626. data/spec/support/fixtures/plugins_with_priorities/p0.rb +1 -1
  627. data/spec/support/fixtures/plugins_with_priorities/p00.rb +1 -1
  628. data/spec/support/fixtures/plugins_with_priorities/p1.rb +1 -1
  629. data/spec/support/fixtures/plugins_with_priorities/p2.rb +1 -1
  630. data/spec/support/fixtures/plugins_with_priorities/p22.rb +1 -1
  631. data/spec/support/fixtures/plugins_with_priorities/p222.rb +1 -1
  632. data/spec/support/fixtures/plugins_with_priorities/p_nil.rb +1 -1
  633. data/spec/support/fixtures/plugins_with_priorities/p_nil2.rb +1 -1
  634. data/spec/support/fixtures/report.afr +0 -0
  635. data/spec/support/fixtures/reporters/base_spec/plugin_formatters/with_formatters/foobar.rb +1 -1
  636. data/spec/support/fixtures/reporters/base_spec/with_formatters.rb +1 -1
  637. data/spec/support/fixtures/reporters/base_spec/with_outfile.rb +1 -1
  638. data/spec/support/fixtures/reporters/base_spec/without_outfile.rb +1 -1
  639. data/spec/support/fixtures/reporters/manager_spec/afr.rb +1 -1
  640. data/spec/support/fixtures/reporters/manager_spec/error.rb +1 -1
  641. data/spec/support/fixtures/reporters/manager_spec/foo.rb +1 -1
  642. data/spec/support/fixtures/run_check/body.rb +1 -1
  643. data/spec/support/fixtures/run_check/cookies.rb +1 -1
  644. data/spec/support/fixtures/run_check/empty.rb +1 -1
  645. data/spec/support/fixtures/run_check/flch.rb +1 -1
  646. data/spec/support/fixtures/run_check/forms.rb +1 -1
  647. data/spec/support/fixtures/run_check/headers.rb +1 -1
  648. data/spec/support/fixtures/run_check/links.rb +1 -1
  649. data/spec/support/fixtures/run_check/nil.rb +1 -1
  650. data/spec/support/fixtures/run_check/path.rb +1 -1
  651. data/spec/support/fixtures/run_check/server.rb +1 -1
  652. data/spec/support/fixtures/signature_check/signature.rb +1 -1
  653. data/spec/support/fixtures/wait_check/wait.rb +1 -1
  654. data/spec/support/helpers/browser_cluster/jobs/taint_tracer.rb +0 -3
  655. data/spec/support/helpers/framework.rb +1 -1
  656. data/spec/support/helpers/misc.rb +1 -1
  657. data/spec/support/helpers/paths.rb +1 -1
  658. data/spec/support/helpers/requires.rb +1 -1
  659. data/spec/support/helpers/resets.rb +1 -1
  660. data/spec/support/helpers/web_server.rb +1 -1
  661. data/spec/support/lib/factory.rb +1 -1
  662. data/spec/support/lib/web_server_client.rb +1 -1
  663. data/spec/support/lib/web_server_dispatcher.rb +1 -1
  664. data/spec/support/lib/web_server_manager.rb +4 -2
  665. data/spec/support/servers/arachni/browser/javascript/dom_monitor.rb +48 -0
  666. data/spec/support/servers/arachni/browser/javascript/taint_tracer.rb +15 -3
  667. data/spec/support/servers/arachni/browser.rb +275 -4
  668. data/spec/support/servers/arachni/check/auditor.rb +9 -0
  669. data/spec/support/servers/arachni/element/cookie.rb +34 -0
  670. data/spec/support/servers/arachni/element/form/form_dom.rb +1 -0
  671. data/spec/support/servers/arachni/element/form.rb +36 -2
  672. data/spec/support/servers/arachni/element/header.rb +36 -1
  673. data/spec/support/servers/arachni/element/json.rb +33 -0
  674. data/spec/support/servers/arachni/element/link.rb +33 -1
  675. data/spec/support/servers/arachni/element/link_template.rb +37 -5
  676. data/spec/support/servers/arachni/element/nested_cookie.rb +84 -0
  677. data/spec/support/servers/arachni/element/xml.rb +33 -0
  678. data/spec/support/servers/arachni/http/client/dynamic_404_handler.rb +36 -0
  679. data/spec/support/servers/arachni/http/client/dynamic_404_handler_redirect_1.rb +18 -0
  680. data/spec/support/servers/arachni/http/client/dynamic_404_handler_redirect_2.rb +11 -0
  681. data/spec/support/servers/arachni/http/client.rb +43 -4
  682. data/spec/support/servers/arachni/http/proxy_server.rb +12 -0
  683. data/spec/support/servers/arachni/parser.rb +6 -0
  684. data/spec/support/servers/arachni/session.rb +24 -1
  685. data/spec/support/servers/checks/active/code_injection.rb +18 -0
  686. data/spec/support/servers/checks/active/code_injection_timing.rb +18 -0
  687. data/spec/support/servers/checks/active/csrf.rb +0 -76
  688. data/spec/support/servers/checks/active/file_inclusion.rb +19 -1
  689. data/spec/support/servers/checks/active/ldap_injection.rb +18 -0
  690. data/spec/support/servers/checks/active/no_sql_injection.rb +27 -0
  691. data/spec/support/servers/checks/active/no_sql_injection_differential.rb +19 -0
  692. data/spec/support/servers/checks/active/os_cmd_injection.rb +29 -0
  693. data/spec/support/servers/checks/active/os_cmd_injection_timing.rb +18 -1
  694. data/spec/support/servers/checks/active/path_traversal.rb +30 -3
  695. data/spec/support/servers/checks/active/response_splitting.rb +30 -1
  696. data/spec/support/servers/checks/active/rfi.rb +30 -2
  697. data/spec/support/servers/checks/active/session_fixation.rb +1 -3
  698. data/spec/support/servers/checks/active/source_code_disclosure.rb +16 -0
  699. data/spec/support/servers/checks/active/sql_injection/java +2 -0
  700. data/spec/support/servers/checks/active/sql_injection.rb +27 -0
  701. data/spec/support/servers/checks/active/sql_injection_differential.rb +19 -0
  702. data/spec/support/servers/checks/active/sql_injection_timing.rb +19 -1
  703. data/spec/support/servers/checks/active/unvalidated_redirect.rb +121 -1
  704. data/spec/support/servers/checks/active/xpath_injection.rb +27 -0
  705. data/spec/support/servers/checks/active/xss.rb +40 -0
  706. data/spec/support/servers/checks/active/xss_event.rb +23 -2
  707. data/spec/support/servers/checks/active/xss_script_context.rb +18 -0
  708. data/spec/support/servers/checks/active/xss_tag.rb +40 -0
  709. data/spec/support/servers/checks/passive/backup_files.rb +20 -1
  710. data/spec/support/servers/checks/passive/grep/cookie_set_for_parent_domain.rb +3 -5
  711. data/spec/support/servers/checks/passive/grep/insecure_cookies_https.rb +9 -0
  712. data/spec/support/servers/checks/passive/grep/x_frame_options.rb +5 -0
  713. data/spec/support/servers/plugins/autologin.rb +17 -1
  714. data/spec/support/servers/plugins/webhook_notify.rb +9 -0
  715. data/spec/support/shared/check.rb +1 -0
  716. data/spec/support/shared/element/capabilities/auditable/buffered.rb +791 -0
  717. data/spec/support/shared/element/capabilities/auditable/line_buffered.rb +797 -0
  718. data/spec/support/shared/element/capabilities/auditable.rb +28 -34
  719. data/spec/support/shared/element/capabilities/inputtable.rb +26 -0
  720. data/spec/support/shared/element/capabilities/with_node.rb +2 -2
  721. data/spec/support/shared/element/dom/submittable.rb +10 -10
  722. data/spec/support/shared/path_extractor.rb +17 -5
  723. data/ui/cli/framework/option_parser.rb +78 -13
  724. data/ui/cli/framework.rb +29 -8
  725. data/ui/cli/option_parser.rb +1 -1
  726. data/ui/cli/output.rb +10 -3
  727. data/ui/cli/reporter/option_parser.rb +1 -1
  728. data/ui/cli/reporter.rb +1 -1
  729. data/ui/cli/reproduce/option_parser.rb +90 -0
  730. data/ui/cli/reproduce.rb +228 -0
  731. data/ui/cli/rest/server/option_parser.rb +1 -1
  732. data/ui/cli/rest/server.rb +1 -1
  733. data/ui/cli/restored_framework/option_parser.rb +1 -1
  734. data/ui/cli/restored_framework.rb +1 -1
  735. data/ui/cli/rpc/client/dispatcher_monitor/option_parser.rb +1 -1
  736. data/ui/cli/rpc/client/dispatcher_monitor.rb +9 -11
  737. data/ui/cli/rpc/client/instance.rb +7 -4
  738. data/ui/cli/rpc/client/local/option_parser.rb +1 -1
  739. data/ui/cli/rpc/client/local.rb +1 -1
  740. data/ui/cli/rpc/client/remote/option_parser.rb +1 -1
  741. data/ui/cli/rpc/client/remote.rb +1 -1
  742. data/ui/cli/rpc/server/dispatcher/option_parser.rb +1 -1
  743. data/ui/cli/rpc/server/dispatcher.rb +1 -1
  744. data/ui/cli/utilities.rb +1 -1
  745. metadata +178 -79
  746. data/ACKNOWLEDGMENTS.md +0 -21
  747. data/AUTHORS.md +0 -3
  748. data/CONTRIBUTORS.md +0 -22
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -48,11 +48,15 @@ class Job
48
48
  # Duration of the job, in seconds.
49
49
  attr_accessor :time
50
50
 
51
+ attr_accessor :args
52
+
51
53
  # @param [Hash] options
52
54
  def initialize( options = {} )
53
55
  @options = options.dup
54
56
  @options[:id] = @id = options.delete(:id) || increment_id
55
57
 
58
+ @args = @options[:args] || []
59
+
56
60
  options.each { |k, v| options[k] = send( "#{k}=", v ) }
57
61
  end
58
62
 
@@ -163,7 +167,9 @@ class Job
163
167
  # Forwarded request (preserving its {#id} and thus its callback as well),
164
168
  # configured with the given `options`.
165
169
  def forward_as( job_type, options = {} )
166
- job_type.new forward_options( options )
170
+ # Remove the ID because this will be a different class/job type and
171
+ # we thus need to keep track of it separately in the BrowserCluster.
172
+ job_type.new forward_options( options ).tap { |h| h.delete :id }
167
173
  end
168
174
 
169
175
  # @return [Integer]
@@ -190,6 +196,7 @@ class Job
190
196
 
191
197
  def forward_options( options )
192
198
  add_id( options ).merge(
199
+ args: args,
193
200
  never_ending: never_ending?,
194
201
  forwarder: self.clean_copy
195
202
  )
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -16,8 +16,14 @@ module Jobs
16
16
  # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
17
17
  class BrowserProvider < Job
18
18
 
19
+ def initialize( *args )
20
+ super()
21
+
22
+ @args = args
23
+ end
24
+
19
25
  def run
20
- browser.master.callback_for( self ).call browser
26
+ browser.master.callback_for( self ).call *[browser, @args].flatten.compact
21
27
  end
22
28
 
23
29
  def to_s
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -36,6 +36,18 @@ class DOMExploration < Job
36
36
  browser.trigger_events
37
37
  end
38
38
 
39
+ def resource=( r )
40
+ # Pages are heavy objects, better just keep the DOM since the browsers
41
+ # will only load them by it anyways.
42
+ if r.is_a? Page
43
+ @resource = r.dom
44
+ @resource.page = nil
45
+ return r
46
+ end
47
+
48
+ @resource = r
49
+ end
50
+
39
51
  def dup
40
52
  super.tap { |j| j.resource = resource }
41
53
  end
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -1,5 +1,5 @@
1
1
  =begin
2
- Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
2
+ Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
3
3
 
4
4
  This file is part of the Arachni Framework project and is subject to
5
5
  redistribution and commercial restrictions. Please see the Arachni Framework
@@ -20,6 +20,9 @@ class BrowserCluster
20
20
  class Worker < Arachni::Browser
21
21
  personalize_output
22
22
 
23
+ # How many times to retry timed-out jobs.
24
+ TRIES = 6
25
+
23
26
  # @return [BrowserCluster]
24
27
  attr_reader :master
25
28
 
@@ -27,9 +30,6 @@ class Worker < Arachni::Browser
27
30
  # Currently assigned job.
28
31
  attr_reader :job
29
32
 
30
- # @return [Integer]
31
- attr_accessor :job_timeout
32
-
33
33
  # @return [Integer]
34
34
  attr_accessor :max_time_to_live
35
35
 
@@ -44,15 +44,12 @@ class Worker < Arachni::Browser
44
44
  Options.browser_cluster.worker_time_to_live
45
45
  @time_to_live = @max_time_to_live
46
46
 
47
- @job_timeout = options.delete( :job_timeout ) ||
48
- Options.browser_cluster.job_timeout
47
+ @done_signal = Queue.new
49
48
 
50
49
  # Don't store pages if there's a master, we'll be sending them to him
51
50
  # as soon as they're logged.
52
51
  super options.merge( store_pages: false )
53
52
 
54
- @done_signal = Queue.new
55
-
56
53
  start_capture
57
54
 
58
55
  return if !@master
@@ -75,71 +72,62 @@ class Worker < Arachni::Browser
75
72
  # If we can't respawn, then bail out.
76
73
  return if browser_respawn_if_necessary.nil?
77
74
 
78
- # ap '=' * 250
79
- # ap '=' * 250
80
- # pre = $WATIR_REQ_COUNT
81
-
82
75
  time = Time.now
83
76
  begin
84
- with_timeout @job_timeout do
85
- exception_jail false do
86
- begin
87
- @job.configure_and_run( self )
88
- rescue Selenium::WebDriver::Error::WebDriverError,
89
- Watir::Exception::Error => e
90
-
91
- print_debug "Error while processing job: #{@job}"
92
- print_debug
93
- print_debug_exception e
94
-
95
- browser_respawn
96
- end
97
- end
77
+
78
+ Timeout.timeout Options.browser_cluster.job_timeout do
79
+ @job.configure_and_run( self )
98
80
  end
99
81
 
100
- job.time = Time.now - time
101
- rescue TimeoutError => e
102
- job.timed_out!( Time.now - time )
82
+ @job.time = Time.now - time
103
83
 
104
- print_bad "Job timed-out after #{@job_timeout} seconds: #{@job}"
84
+ rescue Selenium::WebDriver::Error::WebDriverError,
85
+ Watir::Exception::Error => e
105
86
 
106
- # Could have left us with a broken browser.
107
- browser_respawn
108
- end
87
+ print_debug "WebDriver error while processing job: #{@job}"
88
+ print_debug_exception e
109
89
 
110
- # ap $WATIR_REQ_COUNT - pre
90
+ # This can be thrown by a Selenium call somewhere down the line,
91
+ # catch it here and retry the entire job.
92
+ rescue Timeout::Error => e
111
93
 
112
- decrease_time_to_live
113
- browser_respawn_if_necessary
94
+ @job.timed_out!( Time.now - time )
114
95
 
115
- print_debug "Finished: #{@job}"
96
+ print_bad "Job timed-out: #{@job}"
97
+ print_debug_exception e
116
98
 
117
- true
118
- rescue Selenium::WebDriver::Error::WebDriverError
119
- browser_respawn
120
- nil
121
- ensure
122
- @javascript.taint = nil
99
+ master.increment_time_out_count
100
+ end
123
101
 
124
- clear_buffers
102
+ decrease_time_to_live
125
103
 
126
- # The jobs may have configured callbacks to capture pages etc.,
127
- # remove them.
128
- clear_observers
104
+ # Something went horribly wrong, cleanup.
105
+ rescue => e
106
+ print_error "Error while processing job: #{@job}"
107
+ print_exception e
129
108
 
109
+ browser_respawn
110
+ ensure
111
+ print_debug "Finished: #{@job}"
130
112
  @job = nil
113
+
114
+ reset
115
+ master.job_done job
131
116
  end
132
117
 
133
118
  # Direct the distribution to the master and let it take it from there.
134
119
  #
135
120
  # @see Jobs::EventTrigger
136
121
  # @see BrowserCluster#queue
137
- def distribute_event( page, element, event )
138
- master.queue @job.forward_as(
139
- @job.class::EventTrigger,
140
- resource: page,
141
- element: element,
142
- event: event
122
+ def distribute_event( resource, element, event )
123
+ master.queue(
124
+ @job.forward_as(
125
+ @job.class::EventTrigger,
126
+ resource: resource,
127
+ element: element,
128
+ event: event
129
+ ),
130
+ master.callback_for( @job )
143
131
  )
144
132
  true
145
133
  # Job may have been marked as done or the cluster may have been shut down.
@@ -148,6 +136,7 @@ class Worker < Arachni::Browser
148
136
  false
149
137
  end
150
138
 
139
+ alias :browser_shutdown :shutdown
151
140
  # @note If there is a running job it will wait for it to finish.
152
141
  #
153
142
  # Shuts down the worker.
@@ -155,19 +144,40 @@ class Worker < Arachni::Browser
155
144
  return if @shutdown
156
145
  @shutdown = true
157
146
 
147
+ print_debug "Shutting down (wait: #{wait}) ..."
148
+
158
149
  # Keep checking to see if any of the 'done' criteria are true.
159
150
  kill_check = Thread.new do
160
- sleep 0.1 while alive? && wait && @job
151
+ while wait && @job
152
+ print_debug_level_2 "Waiting for job to complete: #{job}"
153
+ sleep 0.1
154
+ end
155
+
156
+ print_debug_level_2 'Signaling done.'
161
157
  @done_signal << nil
162
158
  end
163
159
 
160
+ print_debug_level_2 'Waiting for done signal...'
164
161
  # If we've got a job running wait for it to finish before closing the
165
162
  # browser otherwise we'll get Selenium errors and zombie processes.
166
163
  @done_signal.pop
164
+ print_debug_level_2 '...done.'
165
+
166
+ print_debug_level_2 'Waiting for kill check...'
167
167
  kill_check.join
168
- @consumer.kill if @consumer
168
+ print_debug_level_2 '...done.'
169
+
170
+ if @consumer
171
+ print_debug_level_2 'Killing consumer thread...'
172
+ @consumer.kill
173
+ print_debug_level_2 '...done.'
174
+ end
169
175
 
170
- super()
176
+ print_debug_level_2 'Calling parent shutdown...'
177
+ browser_shutdown
178
+ print_debug_level_2 '...done.'
179
+
180
+ print_debug '...shutdown complete.'
171
181
  end
172
182
 
173
183
  def inspect
@@ -179,12 +189,18 @@ class Worker < Arachni::Browser
179
189
  s << '>'
180
190
  end
181
191
 
182
- def self.name
183
- "BrowserCluster Worker##{object_id}"
184
- end
185
-
186
192
  private
187
193
 
194
+ def reset
195
+ @javascript.taint = nil
196
+
197
+ clear_buffers
198
+
199
+ # The jobs may have configured callbacks to capture pages etc.,
200
+ # remove them.
201
+ clear_observers
202
+ end
203
+
188
204
  # @return [Support::LookUp::HashSet]
189
205
  # States that have been visited and should be skipped, for the given
190
206
  # {#job}.
@@ -210,46 +226,46 @@ class Worker < Arachni::Browser
210
226
  def start
211
227
  @consumer ||= Thread.new do
212
228
  while !@shutdown
213
- exception_jail false do
214
- j = master.pop
215
- exception_jail( false ) { run_job j }
216
- master.decrease_pending_job j
217
- end
229
+ run_job master.pop
218
230
  end
231
+
232
+ print_debug 'Got shutdown signal...'
219
233
  @done_signal << nil
234
+ print_debug '...and acknowledged it.'
220
235
  end
221
236
  end
222
237
 
223
238
  def browser_respawn_if_necessary
224
- return false if !time_to_die? && alive?
239
+ return false if !time_to_die?
225
240
  browser_respawn
226
241
  end
227
242
 
228
243
  def browser_respawn
244
+ print_debug "Re-spawning browser (TTD?: #{time_to_die?}) ..."
229
245
  @time_to_live = @max_time_to_live
230
246
 
231
- begin
232
- # If PhantomJS is already dead this will block for quite some time so
233
- # beware.
234
- @watir.close if @watir && alive?
235
- rescue Selenium::WebDriver::Error::WebDriverError,
236
- Watir::Exception::Error
237
- end
247
+ browser_shutdown
238
248
 
239
- kill_process
249
+ # Browser may fail to respawn but there's nothing we can do about that,
250
+ # just leave it dead and try again at the next job.
251
+ r = begin
252
+
253
+ start_webdriver
240
254
 
241
- # Browser may fail to respawn but there's nothing we can do about
242
- # that, just leave it dead and try again at the next job.
243
- begin
244
- @watir = ::Watir::Browser.new( selenium )
245
255
  true
246
256
  rescue Selenium::WebDriver::Error::WebDriverError,
247
257
  Browser::Error::Spawn => e
248
- print_error 'Could not respawn the browser, will try again at the ' <<
249
- "next job. (#{e})"
250
- print_error 'Please try increasing the maximum open files limit of your OS.'
258
+
259
+ print_error 'Could not respawn the browser, will try again at' <<
260
+ " the next job. (#{e})"
261
+ print_error 'Please try increasing the maximum open files limit' <<
262
+ ' of your OS.'
251
263
  nil
252
264
  end
265
+
266
+ print_debug "...re-spawned browser: #{r}"
267
+
268
+ r
253
269
  end
254
270
 
255
271
  def time_to_die?
@@ -260,12 +276,6 @@ class Worker < Arachni::Browser
260
276
  @time_to_live -= 1
261
277
  end
262
278
 
263
- def save_response( response )
264
- super( response )
265
- master.push_to_sitemap( response.url, response.code )
266
- response
267
- end
268
-
269
279
  end
270
280
  end
271
281
  end