arachni 0.2.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (262) hide show
  1. data/ACKNOWLEDGMENTS.md +14 -0
  2. data/AUTHORS.md +6 -0
  3. data/CHANGELOG.md +162 -0
  4. data/CONTRIBUTORS.md +10 -0
  5. data/EXPLOITATION.md +429 -0
  6. data/HACKING.md +101 -0
  7. data/LICENSE.md +341 -0
  8. data/README.md +350 -0
  9. data/Rakefile +86 -0
  10. data/bin/arachni +22 -0
  11. data/bin/arachni_web +77 -0
  12. data/bin/arachni_xmlrpc +21 -0
  13. data/bin/arachni_xmlrpcd +82 -0
  14. data/bin/arachni_xmlrpcd_monitor +74 -0
  15. data/conf/README.webui.yaml.txt +44 -0
  16. data/conf/webui.yaml +11 -0
  17. data/external/metasploit/LICENSE +24 -0
  18. data/external/metasploit/modules/exploits/unix/webapp/arachni_exec.rb +142 -0
  19. data/external/metasploit/modules/exploits/unix/webapp/arachni_path_traversal.rb +113 -0
  20. data/external/metasploit/modules/exploits/unix/webapp/arachni_php_eval.rb +150 -0
  21. data/external/metasploit/modules/exploits/unix/webapp/arachni_php_include.rb +141 -0
  22. data/external/metasploit/modules/exploits/unix/webapp/arachni_sqlmap.rb +92 -0
  23. data/external/metasploit/plugins/arachni.rb +536 -0
  24. data/getoptslong.rb +241 -0
  25. data/lib/anemone.rb +2 -0
  26. data/lib/anemone/cookie_store.rb +35 -0
  27. data/lib/anemone/core.rb +371 -0
  28. data/lib/anemone/exceptions.rb +5 -0
  29. data/lib/anemone/http.rb +144 -0
  30. data/lib/anemone/page.rb +337 -0
  31. data/lib/anemone/page_store.rb +160 -0
  32. data/lib/anemone/storage.rb +34 -0
  33. data/lib/anemone/storage/base.rb +75 -0
  34. data/lib/anemone/storage/exceptions.rb +15 -0
  35. data/lib/anemone/storage/mongodb.rb +89 -0
  36. data/lib/anemone/storage/pstore.rb +50 -0
  37. data/lib/anemone/storage/redis.rb +90 -0
  38. data/lib/anemone/storage/tokyo_cabinet.rb +57 -0
  39. data/lib/anemone/tentacle.rb +40 -0
  40. data/lib/arachni.rb +16 -0
  41. data/lib/audit_store.rb +346 -0
  42. data/lib/component_manager.rb +293 -0
  43. data/lib/component_options.rb +395 -0
  44. data/lib/exceptions.rb +76 -0
  45. data/lib/framework.rb +637 -0
  46. data/lib/http.rb +809 -0
  47. data/lib/issue.rb +302 -0
  48. data/lib/module.rb +4 -0
  49. data/lib/module/auditor.rb +455 -0
  50. data/lib/module/base.rb +188 -0
  51. data/lib/module/element_db.rb +158 -0
  52. data/lib/module/key_filler.rb +87 -0
  53. data/lib/module/manager.rb +87 -0
  54. data/lib/module/output.rb +68 -0
  55. data/lib/module/trainer.rb +240 -0
  56. data/lib/module/utilities.rb +110 -0
  57. data/lib/options.rb +547 -0
  58. data/lib/parser.rb +2 -0
  59. data/lib/parser/auditable.rb +522 -0
  60. data/lib/parser/elements.rb +296 -0
  61. data/lib/parser/page.rb +149 -0
  62. data/lib/parser/parser.rb +717 -0
  63. data/lib/plugin.rb +4 -0
  64. data/lib/plugin/base.rb +110 -0
  65. data/lib/plugin/manager.rb +162 -0
  66. data/lib/report.rb +4 -0
  67. data/lib/report/base.rb +119 -0
  68. data/lib/report/manager.rb +92 -0
  69. data/lib/rpc/xml/client/base.rb +71 -0
  70. data/lib/rpc/xml/client/dispatcher.rb +49 -0
  71. data/lib/rpc/xml/client/instance.rb +88 -0
  72. data/lib/rpc/xml/server/base.rb +90 -0
  73. data/lib/rpc/xml/server/dispatcher.rb +357 -0
  74. data/lib/rpc/xml/server/framework.rb +206 -0
  75. data/lib/rpc/xml/server/instance.rb +191 -0
  76. data/lib/rpc/xml/server/module/manager.rb +46 -0
  77. data/lib/rpc/xml/server/options.rb +124 -0
  78. data/lib/rpc/xml/server/output.rb +299 -0
  79. data/lib/rpc/xml/server/plugin/manager.rb +58 -0
  80. data/lib/ruby.rb +5 -0
  81. data/lib/ruby/object.rb +32 -0
  82. data/lib/ruby/string.rb +74 -0
  83. data/lib/ruby/xmlrpc/server.rb +27 -0
  84. data/lib/spider.rb +200 -0
  85. data/lib/typhoeus/request.rb +91 -0
  86. data/lib/typhoeus/response.rb +34 -0
  87. data/lib/ui/cli/cli.rb +744 -0
  88. data/lib/ui/cli/output.rb +279 -0
  89. data/lib/ui/web/log.rb +82 -0
  90. data/lib/ui/web/output_stream.rb +94 -0
  91. data/lib/ui/web/report_manager.rb +222 -0
  92. data/lib/ui/web/server.rb +903 -0
  93. data/lib/ui/web/server/db/placeholder +0 -0
  94. data/lib/ui/web/server/public/banner.png +0 -0
  95. data/lib/ui/web/server/public/bodybg-small.png +0 -0
  96. data/lib/ui/web/server/public/bodybg.png +0 -0
  97. data/lib/ui/web/server/public/css/smoothness/images/pbar-ani.gif +0 -0
  98. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_flat_0_aaaaaa_40x100.png +0 -0
  99. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png +0 -0
  100. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png +0 -0
  101. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_glass_65_ffffff_1x400.png +0 -0
  102. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_glass_75_dadada_1x400.png +0 -0
  103. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_glass_75_e6e6e6_1x400.png +0 -0
  104. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png +0 -0
  105. data/lib/ui/web/server/public/css/smoothness/images/ui-bg_highlight-soft_75_cccccc_1x100.png +0 -0
  106. data/lib/ui/web/server/public/css/smoothness/images/ui-icons_222222_256x240.png +0 -0
  107. data/lib/ui/web/server/public/css/smoothness/images/ui-icons_2e83ff_256x240.png +0 -0
  108. data/lib/ui/web/server/public/css/smoothness/images/ui-icons_454545_256x240.png +0 -0
  109. data/lib/ui/web/server/public/css/smoothness/images/ui-icons_888888_256x240.png +0 -0
  110. data/lib/ui/web/server/public/css/smoothness/images/ui-icons_cd0a0a_256x240.png +0 -0
  111. data/lib/ui/web/server/public/css/smoothness/jquery-ui-1.8.9.custom.css +573 -0
  112. data/lib/ui/web/server/public/favicon.ico +0 -0
  113. data/lib/ui/web/server/public/footer.jpg +0 -0
  114. data/lib/ui/web/server/public/icons/error.png +0 -0
  115. data/lib/ui/web/server/public/icons/info.png +0 -0
  116. data/lib/ui/web/server/public/icons/ok.png +0 -0
  117. data/lib/ui/web/server/public/icons/status.png +0 -0
  118. data/lib/ui/web/server/public/js/jquery-1.4.4.min.js +167 -0
  119. data/lib/ui/web/server/public/js/jquery-ui-1.8.9.custom.min.js +781 -0
  120. data/lib/ui/web/server/public/logo.png +0 -0
  121. data/lib/ui/web/server/public/nav-left.jpg +0 -0
  122. data/lib/ui/web/server/public/nav-right.jpg +0 -0
  123. data/lib/ui/web/server/public/nav-selected-left.jpg +0 -0
  124. data/lib/ui/web/server/public/nav-selected-right.jpg +0 -0
  125. data/lib/ui/web/server/public/reports/placeholder +1 -0
  126. data/lib/ui/web/server/public/sidebar-bottom.jpg +0 -0
  127. data/lib/ui/web/server/public/sidebar-h4.jpg +0 -0
  128. data/lib/ui/web/server/public/sidebar-top.jpg +0 -0
  129. data/lib/ui/web/server/public/spider.png +0 -0
  130. data/lib/ui/web/server/public/style.css +604 -0
  131. data/lib/ui/web/server/tmp/placeholder +0 -0
  132. data/lib/ui/web/server/views/dispatcher.erb +85 -0
  133. data/lib/ui/web/server/views/dispatcher_error.erb +14 -0
  134. data/lib/ui/web/server/views/error.erb +1 -0
  135. data/lib/ui/web/server/views/flash.erb +18 -0
  136. data/lib/ui/web/server/views/home.erb +14 -0
  137. data/lib/ui/web/server/views/instance.erb +213 -0
  138. data/lib/ui/web/server/views/layout.erb +95 -0
  139. data/lib/ui/web/server/views/log.erb +40 -0
  140. data/lib/ui/web/server/views/modules.erb +71 -0
  141. data/lib/ui/web/server/views/options.erb +23 -0
  142. data/lib/ui/web/server/views/output_results.erb +51 -0
  143. data/lib/ui/web/server/views/plugins.erb +42 -0
  144. data/lib/ui/web/server/views/report_formats.erb +30 -0
  145. data/lib/ui/web/server/views/reports.erb +55 -0
  146. data/lib/ui/web/server/views/settings.erb +120 -0
  147. data/lib/ui/web/server/views/welcome.erb +38 -0
  148. data/lib/ui/xmlrpc/dispatcher_monitor.rb +204 -0
  149. data/lib/ui/xmlrpc/xmlrpc.rb +843 -0
  150. data/logs/placeholder +0 -0
  151. data/metamodules/autothrottle.rb +74 -0
  152. data/metamodules/timeout_notice.rb +118 -0
  153. data/metamodules/uniformity.rb +98 -0
  154. data/modules/audit/code_injection.rb +136 -0
  155. data/modules/audit/code_injection_timing.rb +115 -0
  156. data/modules/audit/code_injection_timing/payloads.txt +4 -0
  157. data/modules/audit/csrf.rb +301 -0
  158. data/modules/audit/ldapi.rb +103 -0
  159. data/modules/audit/ldapi/errors.txt +26 -0
  160. data/modules/audit/os_cmd_injection.rb +103 -0
  161. data/modules/audit/os_cmd_injection/payloads.txt +2 -0
  162. data/modules/audit/os_cmd_injection_timing.rb +104 -0
  163. data/modules/audit/os_cmd_injection_timing/payloads.txt +3 -0
  164. data/modules/audit/path_traversal.rb +141 -0
  165. data/modules/audit/response_splitting.rb +105 -0
  166. data/modules/audit/rfi.rb +193 -0
  167. data/modules/audit/sqli.rb +120 -0
  168. data/modules/audit/sqli/regexp_ids.txt +90 -0
  169. data/modules/audit/sqli_blind_rdiff.rb +321 -0
  170. data/modules/audit/sqli_blind_timing.rb +103 -0
  171. data/modules/audit/sqli_blind_timing/payloads.txt +51 -0
  172. data/modules/audit/trainer.rb +89 -0
  173. data/modules/audit/unvalidated_redirect.rb +90 -0
  174. data/modules/audit/xpath.rb +104 -0
  175. data/modules/audit/xpath/errors.txt +26 -0
  176. data/modules/audit/xss.rb +99 -0
  177. data/modules/audit/xss_event.rb +134 -0
  178. data/modules/audit/xss_path.rb +125 -0
  179. data/modules/audit/xss_script_tag.rb +112 -0
  180. data/modules/audit/xss_tag.rb +112 -0
  181. data/modules/audit/xss_uri.rb +125 -0
  182. data/modules/recon/allowed_methods.rb +104 -0
  183. data/modules/recon/backdoors.rb +131 -0
  184. data/modules/recon/backdoors/filenames.txt +16 -0
  185. data/modules/recon/backup_files.rb +177 -0
  186. data/modules/recon/backup_files/extensions.txt +28 -0
  187. data/modules/recon/common_directories.rb +138 -0
  188. data/modules/recon/common_directories/directories.txt +265 -0
  189. data/modules/recon/common_files.rb +138 -0
  190. data/modules/recon/common_files/filenames.txt +17 -0
  191. data/modules/recon/directory_listing.rb +171 -0
  192. data/modules/recon/grep/captcha.rb +62 -0
  193. data/modules/recon/grep/credit_card.rb +85 -0
  194. data/modules/recon/grep/cvs_svn_users.rb +73 -0
  195. data/modules/recon/grep/emails.rb +59 -0
  196. data/modules/recon/grep/html_objects.rb +53 -0
  197. data/modules/recon/grep/private_ip.rb +54 -0
  198. data/modules/recon/grep/ssn.rb +53 -0
  199. data/modules/recon/htaccess_limit.rb +82 -0
  200. data/modules/recon/http_put.rb +95 -0
  201. data/modules/recon/interesting_responses.rb +118 -0
  202. data/modules/recon/unencrypted_password_forms.rb +119 -0
  203. data/modules/recon/webdav.rb +126 -0
  204. data/modules/recon/xst.rb +107 -0
  205. data/path_extractors/anchors.rb +35 -0
  206. data/path_extractors/forms.rb +35 -0
  207. data/path_extractors/frames.rb +38 -0
  208. data/path_extractors/generic.rb +39 -0
  209. data/path_extractors/links.rb +35 -0
  210. data/path_extractors/meta_refresh.rb +39 -0
  211. data/path_extractors/scripts.rb +37 -0
  212. data/path_extractors/sitemap.rb +31 -0
  213. data/plugins/autologin.rb +137 -0
  214. data/plugins/content_types.rb +90 -0
  215. data/plugins/cookie_collector.rb +99 -0
  216. data/plugins/form_dicattack.rb +185 -0
  217. data/plugins/healthmap.rb +94 -0
  218. data/plugins/http_dicattack.rb +133 -0
  219. data/plugins/metamodules.rb +118 -0
  220. data/plugins/proxy.rb +248 -0
  221. data/plugins/proxy/server.rb +66 -0
  222. data/plugins/waf_detector.rb +184 -0
  223. data/profiles/comprehensive.afp +74 -0
  224. data/profiles/full.afp +75 -0
  225. data/reports/afr.rb +59 -0
  226. data/reports/ap.rb +55 -0
  227. data/reports/html.rb +179 -0
  228. data/reports/html/default.erb +967 -0
  229. data/reports/metareport.rb +139 -0
  230. data/reports/metareport/arachni_metareport.rb +174 -0
  231. data/reports/plugin_formatters/html/content_types.rb +82 -0
  232. data/reports/plugin_formatters/html/cookie_collector.rb +66 -0
  233. data/reports/plugin_formatters/html/form_dicattack.rb +54 -0
  234. data/reports/plugin_formatters/html/healthmap.rb +76 -0
  235. data/reports/plugin_formatters/html/http_dicattack.rb +54 -0
  236. data/reports/plugin_formatters/html/metaformatters/timeout_notice.rb +65 -0
  237. data/reports/plugin_formatters/html/metaformatters/uniformity.rb +71 -0
  238. data/reports/plugin_formatters/html/metamodules.rb +93 -0
  239. data/reports/plugin_formatters/html/waf_detector.rb +54 -0
  240. data/reports/plugin_formatters/stdout/content_types.rb +73 -0
  241. data/reports/plugin_formatters/stdout/cookie_collector.rb +61 -0
  242. data/reports/plugin_formatters/stdout/form_dicattack.rb +52 -0
  243. data/reports/plugin_formatters/stdout/healthmap.rb +72 -0
  244. data/reports/plugin_formatters/stdout/http_dicattack.rb +53 -0
  245. data/reports/plugin_formatters/stdout/metaformatters/timeout_notice.rb +55 -0
  246. data/reports/plugin_formatters/stdout/metaformatters/uniformity.rb +68 -0
  247. data/reports/plugin_formatters/stdout/metamodules.rb +89 -0
  248. data/reports/plugin_formatters/stdout/waf_detector.rb +48 -0
  249. data/reports/plugin_formatters/xml/content_types.rb +91 -0
  250. data/reports/plugin_formatters/xml/cookie_collector.rb +70 -0
  251. data/reports/plugin_formatters/xml/form_dicattack.rb +57 -0
  252. data/reports/plugin_formatters/xml/healthmap.rb +82 -0
  253. data/reports/plugin_formatters/xml/http_dicattack.rb +57 -0
  254. data/reports/plugin_formatters/xml/metaformatters/timeout_notice.rb +67 -0
  255. data/reports/plugin_formatters/xml/metaformatters/uniformity.rb +82 -0
  256. data/reports/plugin_formatters/xml/metamodules.rb +91 -0
  257. data/reports/plugin_formatters/xml/waf_detector.rb +58 -0
  258. data/reports/stdout.rb +182 -0
  259. data/reports/txt.rb +77 -0
  260. data/reports/xml.rb +231 -0
  261. data/reports/xml/buffer.rb +98 -0
  262. metadata +516 -0
@@ -0,0 +1,241 @@
1
+ =begin
2
+ Arachni
3
+ Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
4
+
5
+ This is free software; you can copy and distribute and modify
6
+ this program under the term of the GPL v2.0 License
7
+ (See LICENSE file for details)
8
+
9
+ =end
10
+
11
+ require 'getoptlong'
12
+
13
+ # Construct getops struct
14
+ opts = GetoptLong.new(
15
+ [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
16
+ [ '--verbosity', '-v', GetoptLong::NO_ARGUMENT ],
17
+ [ '--only-positives', '-k', GetoptLong::NO_ARGUMENT ],
18
+ [ '--lsmod', GetoptLong::OPTIONAL_ARGUMENT ],
19
+ [ '--lsrep', GetoptLong::OPTIONAL_ARGUMENT ],
20
+ [ '--audit-links', '-g', GetoptLong::NO_ARGUMENT ],
21
+ [ '--audit-forms', '-p', GetoptLong::NO_ARGUMENT ],
22
+ [ '--audit-cookies', '-c', GetoptLong::NO_ARGUMENT ],
23
+ [ '--audit-cookie-jar', GetoptLong::NO_ARGUMENT ],
24
+ [ '--audit-headers', GetoptLong::NO_ARGUMENT ],
25
+ [ '--spider-first', GetoptLong::NO_ARGUMENT ],
26
+ [ '--obey-robots-txt', '-o', GetoptLong::NO_ARGUMENT ],
27
+ [ '--redundant', GetoptLong::REQUIRED_ARGUMENT ],
28
+ [ '--depth', '-d', GetoptLong::REQUIRED_ARGUMENT ],
29
+ [ '--redirect-limit', '-q', GetoptLong::REQUIRED_ARGUMENT ],
30
+ [ '--link-count', '-u', GetoptLong::REQUIRED_ARGUMENT ],
31
+ [ '--mods', '-m', GetoptLong::REQUIRED_ARGUMENT ],
32
+ [ '--report', GetoptLong::REQUIRED_ARGUMENT ],
33
+ [ '--repload', GetoptLong::REQUIRED_ARGUMENT ],
34
+ [ '--authed-by', GetoptLong::REQUIRED_ARGUMENT ],
35
+ [ '--load-profile', GetoptLong::REQUIRED_ARGUMENT ],
36
+ [ '--save-profile', GetoptLong::REQUIRED_ARGUMENT ],
37
+ [ '--show-profile', GetoptLong::NO_ARGUMENT ],
38
+ [ '--proxy', '-z', GetoptLong::REQUIRED_ARGUMENT ],
39
+ [ '--proxy-auth', '-x', GetoptLong::REQUIRED_ARGUMENT ],
40
+ [ '--proxy-type', '-y', GetoptLong::REQUIRED_ARGUMENT ],
41
+ [ '--cookie-jar', '-j', GetoptLong::REQUIRED_ARGUMENT ],
42
+ [ '--user-agent', '-b', GetoptLong::REQUIRED_ARGUMENT ],
43
+ [ '--exclude', '-e', GetoptLong::REQUIRED_ARGUMENT ],
44
+ [ '--include', '-i', GetoptLong::REQUIRED_ARGUMENT ],
45
+ [ '--exclude-cookie', GetoptLong::REQUIRED_ARGUMENT ],
46
+ [ '--http-req-limit', GetoptLong::REQUIRED_ARGUMENT ],
47
+ [ '--follow-subdomains', '-f', GetoptLong::NO_ARGUMENT ],
48
+ [ '--http-harvest-last', '-s', GetoptLong::NO_ARGUMENT ],
49
+ [ '--debug', '-w', GetoptLong::NO_ARGUMENT ],
50
+ [ '--server', GetoptLong::REQUIRED_ARGUMENT ],
51
+ [ '--plugin', GetoptLong::OPTIONAL_ARGUMENT ],
52
+ [ '--lsplug', GetoptLong::OPTIONAL_ARGUMENT ],
53
+ [ '--ssl', GetoptLong::NO_ARGUMENT ],
54
+ [ '--ssl-pkey', GetoptLong::REQUIRED_ARGUMENT ],
55
+ [ '--ssl-cert', GetoptLong::REQUIRED_ARGUMENT ],
56
+ [ '--ssl-ca', GetoptLong::REQUIRED_ARGUMENT ],
57
+ )
58
+
59
+ $:.unshift( File.expand_path( File.dirname( __FILE__ ) ) )
60
+
61
+ require 'lib/options'
62
+ options = Arachni::Options.instance
63
+
64
+ options.dir = Hash.new
65
+ options.dir['root'] = File.dirname( File.expand_path(__FILE__) ) + '/'
66
+ options.dir['modules'] = options.dir['root'] + 'modules/'
67
+ options.dir['reports'] = options.dir['root'] + 'reports/'
68
+ options.dir['plugins'] = options.dir['root'] + 'plugins/'
69
+ options.dir['lib'] = options.dir['root'] + 'lib/'
70
+
71
+ opts.quiet = true
72
+
73
+ begin
74
+ opts.each {
75
+ |opt, arg|
76
+
77
+ case opt
78
+
79
+ when '--help'
80
+ options.help = true
81
+
82
+ when '--only-positives'
83
+ options.only_positives = true
84
+
85
+ when '--verbosity'
86
+ options.arachni_verbose = true
87
+
88
+ when '--debug'
89
+ options.debug = true
90
+
91
+ when '--spider-first'
92
+ options.spider_first = true
93
+
94
+ when '--plugin'
95
+ plugin, opt_str = arg.split( ':', 2 )
96
+
97
+ opts = {}
98
+ if( opt_str )
99
+ opt_arr = opt_str.split( ',' )
100
+ opt_arr.each {
101
+ |opt|
102
+ name, val = opt.split( '=', 2 )
103
+ opts[name] = val
104
+ }
105
+ end
106
+
107
+ options.plugins[plugin] = opts
108
+
109
+ when '--redundant'
110
+ options.redundant << {
111
+ 'regexp' => Regexp.new( arg.to_s.split( /:/ )[0] ),
112
+ 'count' => Integer( arg.to_s.split( /:/ )[1] ),
113
+ }
114
+
115
+ when '--obey_robots_txt'
116
+ options.obey_robots_txt = true
117
+
118
+ when '--depth'
119
+ options.depth_limit = arg.to_i
120
+
121
+ when '--link-count'
122
+ options.link_count_limit = arg.to_i
123
+
124
+ when '--redirect-limit'
125
+ options.redirect_limit = arg.to_i
126
+
127
+ when '--lsmod'
128
+ options.lsmod << Regexp.new( arg.to_s )
129
+
130
+ when '--lsplug'
131
+ options.lsplug << Regexp.new( arg.to_s )
132
+
133
+ when '--lsrep'
134
+ options.lsrep << Regexp.new( arg.to_s )
135
+
136
+ when '--http-req-limit'
137
+ options.http_req_limit = arg.to_i
138
+
139
+ when '--audit-links'
140
+ options.audit_links = true
141
+
142
+ when '--audit-forms'
143
+ options.audit_forms = true
144
+
145
+ when '--audit-cookies'
146
+ options.audit_cookies = true
147
+
148
+ when '--audit-cookie-jar'
149
+ options.audit_cookie_jar = true
150
+
151
+ when '--audit-headers'
152
+ options.audit_headers = true
153
+
154
+ when '--mods'
155
+ options.mods = arg.to_s.split( /,/ )
156
+
157
+ when '--report'
158
+ report, opt_str = arg.split( ':' )
159
+
160
+ opts = {}
161
+ if( opt_str )
162
+ opt_arr = opt_str.split( ',' )
163
+ opt_arr.each {
164
+ |opt|
165
+ name, val = opt.split( '=' )
166
+ opts[name] = val
167
+ }
168
+ end
169
+
170
+ options.reports[report] = opts
171
+
172
+ when '--repload'
173
+ options.repload = arg
174
+
175
+ when '--save-profile'
176
+ options.save_profile = arg
177
+
178
+ when '--load-profile'
179
+ options.load_profile << arg
180
+
181
+ when '--show-profile'
182
+ options.show_profile = true
183
+
184
+ when '--authed-by'
185
+ options.authed_by = arg
186
+
187
+ when '--proxy'
188
+ options.proxy_addr, options.proxy_port =
189
+ arg.to_s.split( /:/ )
190
+
191
+ when '--proxy-auth'
192
+ options.proxy_user, options.proxy_pass =
193
+ arg.to_s.split( /:/ )
194
+
195
+ when '--proxy-type'
196
+ options.proxy_type = arg.to_s
197
+
198
+ when '--cookie-jar'
199
+ options.cookie_jar = arg.to_s
200
+
201
+ when '--user-agent'
202
+ options.user_agent = arg.to_s
203
+
204
+ when '--exclude'
205
+ options.exclude << Regexp.new( arg )
206
+
207
+ when '--include'
208
+ options.include << Regexp.new( arg )
209
+
210
+ when '--exclude-cookie'
211
+ options.exclude_cookies << arg
212
+
213
+ when '--follow-subdomains'
214
+ options.follow_subdomains = true
215
+
216
+ when '--http-harvest-last'
217
+ options.http_harvest_last = true
218
+
219
+ when '--ssl'
220
+ options.ssl = true
221
+
222
+ when '--ssl-pkey'
223
+ options.ssl_pkey = arg.to_s
224
+
225
+ when '--ssl-cert'
226
+ options.ssl_cert = arg.to_s
227
+
228
+ when '--ssl-ca'
229
+ options.ssl_ca = arg.to_s
230
+
231
+ when '--server'
232
+ options.server = arg.to_s
233
+
234
+ end
235
+ }
236
+ rescue Exception => e
237
+ puts e.inspect
238
+ exit
239
+ end
240
+
241
+ options.url = ARGV.shift
@@ -0,0 +1,2 @@
1
+ opts = Arachni::Options.instance
2
+ require opts.dir['lib'] + 'anemone/core'
@@ -0,0 +1,35 @@
1
+ require 'delegate'
2
+ require 'webrick/cookie'
3
+
4
+ class WEBrick::Cookie
5
+ def expired?
6
+ !!expires && expires < Time.now
7
+ end
8
+ end
9
+
10
+ module Anemone
11
+ class CookieStore < DelegateClass(Hash)
12
+
13
+ def initialize(cookies = nil)
14
+ @cookies = {}
15
+ cookies.each { |name, value| @cookies[name] = WEBrick::Cookie.new(name, value) } if cookies
16
+ super(@cookies)
17
+ end
18
+
19
+ def merge!(set_cookie_str)
20
+ begin
21
+ cookie_hash = WEBrick::Cookie.parse_set_cookies(set_cookie_str).inject({}) do |hash, cookie|
22
+ hash[cookie.name] = cookie if !!cookie
23
+ hash
24
+ end
25
+ @cookies.merge! cookie_hash
26
+ rescue
27
+ end
28
+ end
29
+
30
+ def to_s
31
+ @cookies.values.reject { |cookie| cookie.expired? }.map { |cookie| "#{cookie.name}=#{cookie.value}" }.join(';')
32
+ end
33
+
34
+ end
35
+ end
@@ -0,0 +1,371 @@
1
+ =begin
2
+ Arachni
3
+ Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
4
+
5
+ This is free software; you can copy and distribute and modify
6
+ this program under the term of the GPL v2.0 License
7
+ (See LICENSE file for details)
8
+
9
+ =end
10
+
11
+ require 'thread'
12
+ require 'robots'
13
+
14
+ opts = Arachni::Options.instance
15
+ require opts.dir['lib'] + 'anemone/tentacle'
16
+ require opts.dir['lib'] + 'anemone/page'
17
+ require opts.dir['lib'] + 'anemone/exceptions'
18
+ require opts.dir['lib'] + 'anemone/page_store'
19
+ require opts.dir['lib'] + 'anemone/storage'
20
+ require opts.dir['lib'] + 'anemone/storage/base'
21
+
22
+ module Anemone
23
+
24
+ VERSION = '0.5.0';
25
+
26
+ #
27
+ # Convenience method to start a crawl
28
+ #
29
+ def Anemone.crawl(urls, options = {}, &block)
30
+ Core.crawl(urls, options, &block)
31
+ end
32
+
33
+
34
+ #
35
+ # Overides Anemone's Core class method skip_link?( link )
36
+ # to support regexp matching to the whole url and enforce redundancy checks.
37
+ # <br/>
38
+ # Messages were also added to inform the user in case of redundant URLs.
39
+ #
40
+ # @author: Tasos "Zapotek" Laskos
41
+ # <tasos.laskos@gmail.com>
42
+ # <zapotek@segfault.gr>
43
+ # @version: 0.1
44
+ #
45
+ class Core
46
+
47
+ include Arachni::UI::Output
48
+
49
+ # PageStore storing all Page objects encountered during the crawl
50
+ attr_reader :pages
51
+ # Hash of options for the crawl
52
+ attr_reader :opts
53
+
54
+ DEFAULT_OPTS = {
55
+ # run 4 Tentacle threads to fetch pages
56
+ :threads => 4,
57
+ # disable verbose output
58
+ :verbose => false,
59
+ # don't throw away the page response body after scanning it for links
60
+ :discard_page_bodies => false,
61
+ # identify self as Anemone/VERSION
62
+ :user_agent => "Anemone/#{Anemone::VERSION}",
63
+ # no delay between requests
64
+ :delay => 0,
65
+ # don't obey the robots exclusion protocol
66
+ :obey_robots_txt => false,
67
+ # by default, don't limit the depth of the crawl
68
+ :depth_limit => false,
69
+ # number of times HTTP redirects will be followed
70
+ :redirect_limit => 5,
71
+ # storage engine defaults to Hash in +process_options+ if none specified
72
+ :storage => nil,
73
+ # Hash of cookie name => value to send with HTTP requests
74
+ :cookies => nil,
75
+ # accept cookies from the server and send them back?
76
+ :accept_cookies => false,
77
+ # skip any link with a query string? e.g. http://foo.com/?u=user
78
+ :skip_query_strings => false
79
+ }
80
+
81
+ # Create setter methods for all options to be called from the crawl block
82
+ DEFAULT_OPTS.keys.each do |key|
83
+ define_method "#{key}=" do |value|
84
+ @opts[key.to_sym] = value
85
+ end
86
+ end
87
+
88
+ #
89
+ # Initialize the crawl with starting *urls* (single URL or Array of URLs)
90
+ # and optional *block*
91
+ #
92
+ def initialize(urls, opts = {})
93
+ @urls = [urls].flatten.map{ |url| url.is_a?(URI) ? url : URI(url) }
94
+ @urls.each{ |url| url.path = '/' if url.path.empty? }
95
+
96
+ @tentacles = []
97
+ @on_every_page_blocks = []
98
+ @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
99
+ @skip_link_patterns = []
100
+ @after_crawl_blocks = []
101
+ @opts = opts
102
+
103
+ yield self if block_given?
104
+ end
105
+
106
+ #
107
+ # Convenience method to start a new crawl
108
+ #
109
+ def self.crawl(urls, opts = {})
110
+ self.new(urls, opts) do |core|
111
+ yield core if block_given?
112
+ core.run
113
+ end
114
+ end
115
+
116
+ #
117
+ # Add a block to be executed on the PageStore after the crawl
118
+ # is finished
119
+ #
120
+ def after_crawl(&block)
121
+ @after_crawl_blocks << block
122
+ self
123
+ end
124
+
125
+ #
126
+ # Add one ore more Regex patterns for URLs which should not be
127
+ # followed
128
+ #
129
+ def skip_links_like(*patterns)
130
+ @skip_link_patterns.concat [patterns].flatten.compact
131
+ self
132
+ end
133
+
134
+ #
135
+ # Add a block to be executed on every Page as they are encountered
136
+ # during the crawl
137
+ #
138
+ def on_every_page(&block)
139
+ @on_every_page_blocks << block
140
+ self
141
+ end
142
+
143
+ #
144
+ # Add a block to be executed on Page objects with a URL matching
145
+ # one or more patterns
146
+ #
147
+ def on_pages_like(*patterns, &block)
148
+ if patterns
149
+ patterns.each do |pattern|
150
+ @on_pages_like_blocks[pattern] << block
151
+ end
152
+ end
153
+ self
154
+ end
155
+
156
+ #
157
+ # Specify a block which will select which links to follow on each page.
158
+ # The block should return an Array of URI objects.
159
+ #
160
+ def focus_crawl(&block)
161
+ @focus_crawl_block = block
162
+ self
163
+ end
164
+
165
+ #
166
+ # Perform the crawl
167
+ #
168
+ def run
169
+ process_options
170
+
171
+ @urls.delete_if { |url| !visit_link?(url) }
172
+ return if @urls.empty?
173
+
174
+ link_queue = Queue.new
175
+ page_queue = Queue.new
176
+
177
+ @opts[:threads].times do
178
+ @tentacles << Thread.new { Tentacle.new(link_queue, page_queue, @opts).run }
179
+ end
180
+
181
+ @urls.each{ |url| link_queue.enq(url) }
182
+
183
+ loop do
184
+ page = page_queue.deq
185
+ @pages.touch_key page.url
186
+ puts "#{page.url} Queue: #{link_queue.size}" if @opts[:verbose]
187
+ do_page_blocks page
188
+ page.discard_doc! if @opts[:discard_page_bodies]
189
+
190
+ links = links_to_follow page
191
+ links.each do |link|
192
+ link_queue << [link, page.url.dup, page.depth + 1]
193
+ end
194
+ @pages.touch_keys links
195
+
196
+ @pages[page.url] = page
197
+
198
+ # if we are done with the crawl, tell the threads to end
199
+ if link_queue.empty? and page_queue.empty?
200
+ until link_queue.num_waiting == @tentacles.size
201
+ Thread.pass
202
+ end
203
+ if page_queue.empty?
204
+ @tentacles.size.times { link_queue << :END }
205
+ break
206
+ end
207
+ end
208
+ end
209
+
210
+ @tentacles.each { |thread| thread.join }
211
+ do_after_crawl_blocks
212
+ self
213
+ end
214
+
215
+ private
216
+
217
+ def process_options
218
+ @opts = DEFAULT_OPTS.merge @opts
219
+ @opts[:threads] = 1 if @opts[:delay] > 0
220
+ storage = Anemone::Storage::Base.new(@opts[:storage] || Anemone::Storage.Hash)
221
+ @pages = PageStore.new(storage)
222
+ @robots = Robots.new(@opts[:user_agent]) if @opts[:obey_robots_txt]
223
+
224
+ freeze_options
225
+ end
226
+
227
+ #
228
+ # Freeze the opts Hash so that no options can be modified
229
+ # once the crawl begins
230
+ #
231
+ def freeze_options
232
+ @opts.freeze
233
+ @opts.each_key { |key| @opts[key].freeze }
234
+ @opts[:cookies].each_key { |key| @opts[:cookies][key].freeze } rescue nil
235
+ end
236
+
237
+ #
238
+ # Execute the after_crawl blocks
239
+ #
240
+ def do_after_crawl_blocks
241
+ @after_crawl_blocks.each { |block| block.call(@pages) }
242
+ end
243
+
244
+ #
245
+ # Execute the on_every_page blocks for *page*
246
+ #
247
+ # Modified it to fix a bug in Anemone when given more than one<br/>
248
+ # regular expression for "@on_pages_like_blocks".
249
+ #
250
+ def do_page_blocks(page)
251
+ @on_every_page_blocks.each do |block|
252
+ block.call(page)
253
+ end
254
+
255
+ @on_pages_like_blocks.each do |patterns, blocks|
256
+ if matches_pattern?( page.url.to_s, patterns )
257
+ blocks.each { |block| block.call(page) }
258
+ end
259
+ end
260
+ end
261
+
262
+ #
263
+ # Return an Array of links to follow from the given page.
264
+ # Based on whether or not the link has already been crawled,
265
+ # and the block given to focus_crawl()
266
+ #
267
+ def links_to_follow(page)
268
+ links = @focus_crawl_block ? @focus_crawl_block.call(page) : page.links
269
+ links.select { |link| visit_link?(link, page) }.map { |link| link.dup }
270
+ end
271
+
272
+ #
273
+ # Returns +true+ if *link* has not been visited already,
274
+ # and is not excluded by a skip_link pattern...
275
+ # and is not excluded by robots.txt...
276
+ # and is not deeper than the depth limit
277
+ # Returns +false+ otherwise.
278
+ #
279
+ def visit_link?(link, from_page = nil)
280
+ !@pages.has_page?(link) &&
281
+ !skip_link?(link) &&
282
+ !skip_query_string?(link) &&
283
+ allowed(link) &&
284
+ !too_deep?(from_page)
285
+ end
286
+
287
+ #
288
+ # Returns +true+ if we are obeying robots.txt and the link
289
+ # is granted access in it. Always returns +true+ when we are
290
+ # not obeying robots.txt.
291
+ #
292
+ def allowed(link)
293
+ @opts[:obey_robots_txt] ? @robots.allowed?(link) : true
294
+ end
295
+
296
+ #
297
+ # Returns +true+ if we are over the page depth limit.
298
+ # This only works when coming from a page and with the +depth_limit+ option set.
299
+ # When neither is the case, will always return +false+.
300
+ def too_deep?(from_page)
301
+ if from_page && @opts[:depth_limit]
302
+ from_page.depth >= @opts[:depth_limit]
303
+ else
304
+ false
305
+ end
306
+ end
307
+
308
+ #
309
+ # Returns +true+ if *link* should not be visited because
310
+ # it has a query string and +skip_query_strings+ is true.
311
+ #
312
+ def skip_query_string?(link)
313
+ @opts[:skip_query_strings] && link.query
314
+ end
315
+
316
+ #
317
+ # Returns +true+ if *link* should not be visited because
318
+ # its URL matches a skip_link pattern or the reundancy countdown has reached
319
+ # zero.
320
+ #
321
+ def skip_link?( link )
322
+
323
+ url = link.to_s
324
+ skip = false
325
+ @opts['redundant'].each_with_index {
326
+ |redundant, i|
327
+
328
+ if( url =~ redundant['regexp'] )
329
+
330
+ if( @opts['redundant'][i]['count'] == 0 )
331
+ print_verbose( 'Discarding redundant page: \'' + url + '\'' )
332
+ return true
333
+ end
334
+
335
+ print_info( 'Matched redundancy rule: ' +
336
+ redundant['regexp'].to_s + ' for page \'' +
337
+ url + '\'' )
338
+
339
+ print_info( 'Count-down: ' +
340
+ @opts['redundant'][i]['count'].to_s )
341
+
342
+ @opts['redundant'][i]['count'] -= 1
343
+ end
344
+ }
345
+
346
+ @skip_link_patterns.any? { |pattern| url =~ pattern }
347
+
348
+ end
349
+
350
+ #
351
+ # Decides whether or not a url matches any of the regular expressions
352
+ # in "patterns".
353
+ #
354
+ # @param [String] url
355
+ # @param [Array] patterns array of regular expressions
356
+ #
357
+ # @return [Bool]
358
+ #
359
+ def matches_pattern?( url, patterns )
360
+
361
+ patterns.each {
362
+ |pattern|
363
+ return true if url =~ pattern
364
+ }
365
+
366
+ return false
367
+ end
368
+
369
+ end
370
+
371
+ end