ronin-web 0.3.0.rc1 → 1.0.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +11 -0
- data/.github/workflows/ruby.yml +31 -0
- data/.gitignore +13 -0
- data/.mailmap +1 -0
- data/.ruby-version +1 -0
- data/COPYING.txt +3 -3
- data/ChangeLog.md +115 -70
- data/Gemfile +42 -37
- data/README.md +159 -145
- data/Rakefile +12 -3
- data/bin/ronin-web +9 -17
- data/data/new/nokogiri.rb.erb +12 -0
- data/data/new/server.rb.erb +22 -0
- data/data/new/spider.rb.erb +26 -0
- data/data/new/webapp/.gitignore +15 -0
- data/data/new/webapp/.ruby-version.erb +1 -0
- data/data/new/webapp/Dockerfile.erb +11 -0
- data/data/new/webapp/Gemfile +6 -0
- data/data/new/webapp/app.rb.erb +15 -0
- data/data/new/webapp/config.ru +4 -0
- data/data/new/webapp/docker-compose.yml.erb +9 -0
- data/gemspec.yml +32 -14
- data/lib/ronin/web/cli/command.rb +36 -0
- data/lib/ronin/web/cli/commands/diff.rb +106 -0
- data/lib/ronin/web/cli/commands/html.rb +174 -0
- data/lib/ronin/web/cli/commands/irb.rb +56 -0
- data/lib/ronin/web/cli/commands/new/nokogiri.rb +85 -0
- data/lib/ronin/web/cli/commands/new/server.rb +96 -0
- data/lib/ronin/web/cli/commands/new/spider.rb +315 -0
- data/lib/ronin/web/cli/commands/new/webapp.rb +123 -0
- data/lib/ronin/web/cli/commands/new.rb +64 -0
- data/lib/ronin/web/cli/commands/reverse_proxy.rb +215 -0
- data/lib/ronin/web/cli/commands/server.rb +155 -0
- data/lib/ronin/web/cli/commands/spider.rb +822 -0
- data/lib/ronin/web/cli/ruby_shell.rb +50 -0
- data/lib/ronin/web/cli.rb +44 -0
- data/lib/ronin/web/html.rb +85 -0
- data/lib/ronin/web/mechanize.rb +34 -36
- data/lib/ronin/web/root.rb +27 -0
- data/lib/ronin/web/version.rb +7 -10
- data/lib/ronin/web/xml.rb +85 -0
- data/lib/ronin/web.rb +372 -13
- data/man/ronin-web-diff.1 +41 -0
- data/man/ronin-web-diff.1.md +30 -0
- data/man/ronin-web-html.1 +89 -0
- data/man/ronin-web-html.1.md +66 -0
- data/man/ronin-web-irb.1 +31 -0
- data/man/ronin-web-irb.1.md +22 -0
- data/man/ronin-web-new-nokogiri.1 +41 -0
- data/man/ronin-web-new-nokogiri.1.md +30 -0
- data/man/ronin-web-new-server.1 +45 -0
- data/man/ronin-web-new-server.1.md +33 -0
- data/man/ronin-web-new-spider.1 +173 -0
- data/man/ronin-web-new-spider.1.md +129 -0
- data/man/ronin-web-new-webapp.1 +53 -0
- data/man/ronin-web-new-webapp.1.md +39 -0
- data/man/ronin-web-new.1 +59 -0
- data/man/ronin-web-new.1.md +44 -0
- data/man/ronin-web-reverse-proxy.1 +63 -0
- data/man/ronin-web-reverse-proxy.1.md +47 -0
- data/man/ronin-web-server.1 +59 -0
- data/man/ronin-web-server.1.md +43 -0
- data/man/ronin-web-spider.1 +225 -0
- data/man/ronin-web-spider.1.md +168 -0
- data/man/ronin-web.1 +41 -0
- data/man/ronin-web.1.md +30 -0
- data/ronin-web.gemspec +39 -109
- data/spec/cli/ruby_shell_spec.rb +14 -0
- data/spec/html_spec.rb +43 -0
- data/spec/mechanize_spec.rb +72 -0
- data/spec/spec_helper.rb +5 -3
- data/spec/web_spec.rb +97 -0
- data/spec/xml_spec.rb +42 -0
- metadata +236 -224
- data/.gemtest +0 -0
- data/data/ronin/web/user_agents.yml +0 -247
- data/lib/ronin/network/mixins/web.rb +0 -258
- data/lib/ronin/web/config.rb +0 -34
- data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +0 -47
- data/lib/ronin/web/extensions/nokogiri/xml/document.rb +0 -48
- data/lib/ronin/web/extensions/nokogiri/xml/element.rb +0 -57
- data/lib/ronin/web/extensions/nokogiri/xml/node.rb +0 -86
- data/lib/ronin/web/extensions/nokogiri/xml/text.rb +0 -47
- data/lib/ronin/web/extensions/nokogiri/xml.rb +0 -27
- data/lib/ronin/web/extensions/nokogiri.rb +0 -23
- data/lib/ronin/web/extensions.rb +0 -23
- data/lib/ronin/web/middleware/base.rb +0 -144
- data/lib/ronin/web/middleware/directories.rb +0 -179
- data/lib/ronin/web/middleware/files.rb +0 -144
- data/lib/ronin/web/middleware/filters/campaign_filter.rb +0 -77
- data/lib/ronin/web/middleware/filters/ip_filter.rb +0 -73
- data/lib/ronin/web/middleware/filters/path_filter.rb +0 -73
- data/lib/ronin/web/middleware/filters/referer_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters/user_agent_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters/vhost_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters.rb +0 -28
- data/lib/ronin/web/middleware/helpers.rb +0 -145
- data/lib/ronin/web/middleware/proxy.rb +0 -265
- data/lib/ronin/web/middleware/proxy_request.rb +0 -262
- data/lib/ronin/web/middleware/request.rb +0 -79
- data/lib/ronin/web/middleware/response.rb +0 -33
- data/lib/ronin/web/middleware/router.rb +0 -167
- data/lib/ronin/web/middleware/rule.rb +0 -103
- data/lib/ronin/web/middleware.rb +0 -27
- data/lib/ronin/web/proxy/app.rb +0 -32
- data/lib/ronin/web/proxy/base.rb +0 -46
- data/lib/ronin/web/proxy/web.rb +0 -46
- data/lib/ronin/web/proxy.rb +0 -25
- data/lib/ronin/web/server/app.rb +0 -32
- data/lib/ronin/web/server/base.rb +0 -461
- data/lib/ronin/web/server/web.rb +0 -66
- data/lib/ronin/web/server.rb +0 -25
- data/lib/ronin/web/spider.rb +0 -120
- data/lib/ronin/web/user_agents.rb +0 -196
- data/lib/ronin/web/web.rb +0 -560
- data/spec/helpers/output.rb +0 -3
- data/spec/web/extensions/nokogiri_spec.rb +0 -38
- data/spec/web/helpers/rack_app.rb +0 -24
- data/spec/web/helpers/root/test1/index.html +0 -1
- data/spec/web/helpers/root/test1/test1.txt +0 -1
- data/spec/web/helpers/root/test1.txt +0 -1
- data/spec/web/helpers/root/test2/test2.txt +0 -1
- data/spec/web/helpers/root/test2.txt +0 -1
- data/spec/web/helpers/root/test3/test3.txt +0 -1
- data/spec/web/helpers/root/test3.txt +0 -1
- data/spec/web/helpers/root.rb +0 -15
- data/spec/web/mechanize_spec.rb +0 -62
- data/spec/web/middleware/directories_spec.rb +0 -86
- data/spec/web/middleware/files_spec.rb +0 -57
- data/spec/web/middleware/filters/campaign_filter_spec.rb +0 -30
- data/spec/web/middleware/filters/ip_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/path_filter_spec.rb +0 -29
- data/spec/web/middleware/filters/referer_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/user_agent_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/vhost_filter_spec.rb +0 -23
- data/spec/web/middleware/proxy_spec.rb +0 -67
- data/spec/web/middleware/response_spec.rb +0 -20
- data/spec/web/middleware/router_spec.rb +0 -65
- data/spec/web/middleware/rule_spec.rb +0 -37
- data/spec/web/proxy/base_spec.rb +0 -8
- data/spec/web/server/base_spec.rb +0 -77
- data/spec/web/server/classes/public1/static1.txt +0 -1
- data/spec/web/server/classes/public2/static2.txt +0 -1
- data/spec/web/server/classes/sub_app.rb +0 -13
- data/spec/web/server/classes/test_app.rb +0 -20
- data/spec/web/user_agents_spec.rb +0 -56
- data/spec/web/web_spec.rb +0 -101
@@ -1,247 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Sources:
|
3
|
-
#
|
4
|
-
# * http://www.zytrax.com/tech/web/browser_ids.htm
|
5
|
-
#
|
6
|
-
---
|
7
|
-
:googlebot:
|
8
|
-
- "Googlebot/2.1 ( http://www.googlebot.com/bot.html)"
|
9
|
-
- "Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)"
|
10
|
-
- "Mediapartners-Google/2.1"
|
11
|
-
- "Google-Sitemaps/1.0"
|
12
|
-
:askjeeves:
|
13
|
-
- "Mozilla/2.0 (compatible; Ask Jeeves)"
|
14
|
-
- "msnbot-Products/1.0 (+http://search.msn.com/msnbot.htm)"
|
15
|
-
:aptget:
|
16
|
-
- "Ubuntu APT-HTTP/1.3 (0.7.23.1ubuntu2)"
|
17
|
-
- "Ubuntu APT-HTTP/1.3"
|
18
|
-
:chrome:
|
19
|
-
- "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.60 Safari/534.24"
|
20
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.205 Safari/534.16"
|
21
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16"
|
22
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16"
|
23
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16"
|
24
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.127 Safari/534.16"
|
25
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.98 Safari/534.13"
|
26
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10"
|
27
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3"
|
28
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.127 Safari/533.4"
|
29
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4"
|
30
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4"
|
31
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4"
|
32
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Chrome/5.0.322.2 Safari/533.1"
|
33
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.307.9 Safari/532.9"
|
34
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.307.11 Safari/532.9"
|
35
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.78 Safari/532.5"
|
36
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.4 (KHTML, like Gecko) Chrome/4.0.233.0 Safari/532.4"
|
37
|
-
- "Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198.0 Safari/532.0"
|
38
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; Valve Steam GameOverlay; ) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/3.0.195.24 Safari/532.1"
|
39
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13"
|
40
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.33 Safari/532.0"
|
41
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19"
|
42
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-US) AppleWebKit/531.0 (KHTML, like Gecko) Chrome/3.0.183 Safari/531.0"
|
43
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19"
|
44
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.36 Safari/525.19"
|
45
|
-
- "Mozilla/5.0 (Linux; U; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13"
|
46
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13"
|
47
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13"
|
48
|
-
:curl:
|
49
|
-
- "curl/7.19.5 (i586-pc-mingw32msvc) libcurl/7.19.5 OpenSSL/0.9.8l zlib/1.2.3"
|
50
|
-
- "curl/7.7.2 (powerpc-apple-darwin6.0) libcurl 7.7.2 (OpenSSL 0.9.6b)"
|
51
|
-
:firefox:
|
52
|
-
- "Mozilla/5.0 (Windows NT 5.1; rv:2.0) Gecko/20100101 Firefox/4.0:"
|
53
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.4) Gecko/2008102920 Firefox/3.0.4 (Splashtop-v1.0.5.0)"
|
54
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20051111 Firefox/1.5 BAVM/1.0.0"
|
55
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1a2pre) Gecko/2008073000 Shredder/3.0a2pre ThunderBrowse/3.2.1.8"
|
56
|
-
- "Mozilla/5.0 (X11; U; Linux armv61; en-US; rv:1.9.1b2pre) Gecko/20081015 Fennec/1.0a1"
|
57
|
-
- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b7pre) Gecko/20100925 Firefox/4.0b7pre"
|
58
|
-
- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"
|
59
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.16) Gecko/20110319 Firefox/3.6.16"
|
60
|
-
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.8) Gecko/20100215 Solaris/10.1 (GNU) Superswan/3.5.8 (Byte/me)"
|
61
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3"
|
62
|
-
- "Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.8.1.14) Gecko/20080821 Firefox/2.0.0.14"
|
63
|
-
- "Mozilla/5.0 (X11; U; Darwin Power Macintosh; en-US; rv:1.8.0.12) Gecko/20070803 Firefox/1.5.0.12 Fink Community Edition"
|
64
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.13) Gecko/20060410 Firefox/1.0.8"
|
65
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.3) Gecko/20041002 Firefox/0.10.1"
|
66
|
-
- "Mozilla/5.0 (X11; U; SunOS sun4m; en-US; rv:1.4b) Gecko/20030517 Mozilla Firebird/0.6"
|
67
|
-
- "Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.3a) Gecko/20021207 Phoenix/0.5"
|
68
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.2b) Gecko/20020923 Phoenix/0.1"
|
69
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Google Wireless Transcoder;)"
|
70
|
-
:android:
|
71
|
-
- "Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10+ (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2"
|
72
|
-
- "Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522+ (KHTML, like Gecko) Safari/419.3"
|
73
|
-
:links:
|
74
|
-
- "Links (2.1pre31; Linux 2.6.21-omap1 armv6l; x)"
|
75
|
-
- "Links (2.1pre18; Linux 2.6.17-dyne i686; x)"
|
76
|
-
- "Links (2.1pre15; Linux 2.4.26-vc4 i586; x)"
|
77
|
-
- "Links (2.1pre14; OS/2 1 i386; 80x33)"
|
78
|
-
- "Links (0.99; OS/2 1 i386; 80x33)"
|
79
|
-
- "Links (0.98; Linux 2.6.7-rc2 i686; 132x43)"
|
80
|
-
- "Links (0.98; Unix; 80x25)"
|
81
|
-
- "Links (0.95; Unix)"
|
82
|
-
:lynx:
|
83
|
-
- "Lynx/2.8.5dev.16 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6b"
|
84
|
-
- "Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.0.16"
|
85
|
-
- "Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/0.8.12"
|
86
|
-
- "Lynx/2.8.3rel.1 libwww-FM/2.14FM"
|
87
|
-
- "Lynx/2.8.4dev.11 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6"
|
88
|
-
- "Lynx/2.6 libwww-FM/2.14"
|
89
|
-
:seamonkey:
|
90
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.9) Gecko/20100508 SeaMonkey/2.0.4"
|
91
|
-
- "Mozilla/5.0 (X11; U; Linux i686; nb-NO; rv:1.9.1.8) Gecko/20100205 SeaMonkey/2.0.3"
|
92
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1b4pre) Gecko/20090405 SeaMonkey/2.0b1pre"
|
93
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1b1pre) Gecko/20080915000512 SeaMonkey/2.0a1pre"
|
94
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9pre) Gecko/2008060901 SeaMonkey/2.0a1pre"
|
95
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.13) Gecko/20080313 SeaMonkey/1.1.9 (Ubuntu-1.1.9+nobinonly-0ubuntu1)"
|
96
|
-
- "Mozilla/5.0 (X11; U; SunOS sun4u; en-US; rv:1.7) Gecko/20070606"
|
97
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.8) Gecko/20050927 Debian/1.7.8-1sarge3"
|
98
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.1.2) Gecko/20070220 Firefox/2.0.0.8"
|
99
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.1.8) Gecko/20071009 SeaMonkey/1.1.5"
|
100
|
-
- "Mozilla/5.0 (Windows; U; WinNT3.51; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4"
|
101
|
-
- "Mozilla/5.0 (BeOS; U; BeOS BePC; en-US; rv:1.8.1.8pre) Gecko/20070926 SeaMonkey/1.1.5pre"
|
102
|
-
- "Mozilla/5.0 (X11; U; Darwin Power Macintosh; en-US; rv:1.8.1.5) Gecko/20070803 SeaMonkey/1.1.3"
|
103
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1b2) Gecko/20060823 SeaMonkey/1.1a"
|
104
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060417"
|
105
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.1) Gecko/20060130 SeaMonkey/1.0"
|
106
|
-
- "Mozilla/5.0 (OS/2; U; Warp 4.5; en-US; rv:1.9a1) Gecko/20051119 MultiZilla/1.8.1.0s SeaMonkey/1.5a"
|
107
|
-
- "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.12) Gecko/20051105"
|
108
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12) Gecko/20050920"
|
109
|
-
- "Mozilla/5.0 (X11; U; Linux ppc; en-US; rv:1.7.12) Gecko/20051009 Debian/1.7.12-1"
|
110
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8b4) Gecko/20050910 SeaMonkey/1.0a"
|
111
|
-
- "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.8b3) Gecko/20050713 SeaMonkey/1.0a"
|
112
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8a2) Gecko/20040704"
|
113
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.7.11) Gecko/20050728"
|
114
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.8) Gecko/20050511"
|
115
|
-
- "Mozilla/5.0+(X11;+U;+Linux+i686;+en-US;+rv:1.7.3)+Gecko/20040922"
|
116
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Debian/1.6-5"
|
117
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040616 MultiZilla/1.6.3.1d"
|
118
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:1.0.2) Gecko/20020924 AOL/7.0"
|
119
|
-
- "Mozilla/5.0 (Windows; U; Win 9x 4.90) Gecko/20020502 CS 2000 7.0/7.0"
|
120
|
-
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5b; MultiZilla v1.5.0.2g) Gecko/20030827"
|
121
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4b) Gecko/20030504 Mozilla Firebird/0.5+ StumbleUpon/1.63"
|
122
|
-
- "Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.2) Gecko/20021126"
|
123
|
-
- "Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.2) Gecko/20021126"
|
124
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.2a) Gecko/20020910"
|
125
|
-
- "Mozilla/5.0 (X11; U; Linux 2.4.3-20mdk i586; en-US; rv:0.9.1) Gecko/20010611"
|
126
|
-
:ie:
|
127
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; InfoPath.1)"
|
128
|
-
- "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"
|
129
|
-
- "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"
|
130
|
-
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Creative AutoUpdate v1.40.02)"
|
131
|
-
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Creative AutoUpdate v1.40.02)"
|
132
|
-
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; GTB6.5; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; InfoPath.1; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)"
|
133
|
-
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0; .NET CLR 1.1.4322; .NET CLR 3.0.04506; eMusic DLM/4; OfficeLiveConnector.1.3; OfficeLiveConnector.1.4; OfficeLivePatch.0.0; OfficeLivePatch.1.3; SLCC1; .NET4.0C; .NET4.0E; InfoPath.3; Zune 4.7; MS-RTC LM 8; yie8)"
|
134
|
-
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; FunWebProducts; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Windows-Media-Player/10.00.00.3990)"
|
135
|
-
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; GTB6.4; .NET CLR 1.1.4322; FDM; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)"
|
136
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Rogers Hi·Speed Internet; (R1 1.3))"
|
137
|
-
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98)"
|
138
|
-
- "Mozilla/4.0 (compatible; MSIE 5.0; SunOS 5.10 sun4u; X11)"
|
139
|
-
- "Mozilla/4.0 (compatible; MSIE 5.22; Mac_PowerPC)"
|
140
|
-
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows NT 5.0)"
|
141
|
-
- "Mozilla/2.0 (compatible; MSIE 3.02; Windows CE; 240x320)"
|
142
|
-
- "Mozilla/1.22 (compatible; MSIE 2.0; Windows 95)"
|
143
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 7.11)"
|
144
|
-
:opera:
|
145
|
-
- "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.10"
|
146
|
-
- "Opera/9.80 (Windows Mobile; WCE; Opera Mobi/WMD-50433; U; en) Presto/2.4.13 Version/10.00"
|
147
|
-
- "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.62 Version/11.00"
|
148
|
-
- "Opera/9.80 (X11; Linux i686; U; en-GB) Presto/2.6.30 Version/10.62"
|
149
|
-
- "Opera/9.80 (Macintosh; Intel Mac OS X; U; en) Presto/2.6.30 Version/10.61"
|
150
|
-
- "Opera/9.80 (J2ME/MIDP; Opera Mini/5.1.21214/19.916; U; en) Presto/2.5.25"
|
151
|
-
- "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.5.24 Version/10.54"
|
152
|
-
- "Opera/9.80 (S60; SymbOS; Opera Mobi/499; U; ru) Presto/2.4.18 Version/10.00"
|
153
|
-
- "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.5.22 Version/10.50"
|
154
|
-
- "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.5.22 Version/10.50"
|
155
|
-
- "Opera/9.80 (X11; Linux x86_64; U; Linux Mint; en) Presto/2.2.15 Version/10.10"
|
156
|
-
- "Opera/9.80 (J2ME/MIDP; Opera Mini/5.0.16823/1428; U; en) Presto/2.2.0"
|
157
|
-
- "Opera/9.80 (J2ME/MIDP; Opera Mini/4.2.14912/1186; U; en) Presto/2.2.0"
|
158
|
-
- "Opera/9.80 (Windows NT 5.2; U; en) Presto/2.2.15 Version/10.10"
|
159
|
-
- "Opera/9.80 (X11; Linux i686; U; nl) Presto/2.2.15 Version/10.00"
|
160
|
-
- "Opera/9.60 (J2ME/MIDP; Opera Mini/4.2.13337/458; U; en) Presto/2.2.0"
|
161
|
-
- "Opera/9.60 (J2ME/MIDP; Opera Mini/4.1.11320/608; U; en) Presto/2.2.0"
|
162
|
-
- "Opera/10.00 (X11; Linux i686 ; U; en) Presto/2.2.0"
|
163
|
-
- "Opera/9.62 (Windows NT 5.1; U; en) Presto/2.1.1"
|
164
|
-
- "Opera/9.60 (X11; Linux i686; U; en) Presto/2.1.1"
|
165
|
-
- "Opera/9.52 (Windows NT 5.1; U; en)"
|
166
|
-
- "Opera/9.25 (Windows NT 6.0; U; en)"
|
167
|
-
- "Opera/9.20 (Macintosh; Intel Mac OS X; U; en)"
|
168
|
-
- "Opera/9.02 (Windows NT 5.0; U; en)"
|
169
|
-
- "Opera/9.00 (Windows NT 4.0; U; en)"
|
170
|
-
- "Opera/9.00 (X11; Linux i686; U; en)"
|
171
|
-
- "Opera/9.00 (Windows NT 5.1; U; en)"
|
172
|
-
- "Opera/9.0 (Windows NT 5.1; U; en)"
|
173
|
-
- "Opera/9.0 (Macintosh; PPC Mac OS X; U; en)"
|
174
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.0"
|
175
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; PPC; 480x640) Opera 8.60 [en]"
|
176
|
-
- "Opera/8.5 (Macintosh; PPC Mac OS X; U; en)"
|
177
|
-
- "Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.5"
|
178
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.5"
|
179
|
-
- "Opera/8.0 (Macintosh; PPC Mac OS X; U; en)"
|
180
|
-
- "Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0"
|
181
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.0"
|
182
|
-
- "Opera/8.01 (Windows NT 5.1)"
|
183
|
-
- "Mozilla/5.0 (Windows NT 5.1; U; en) Opera 8.01"
|
184
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
|
185
|
-
- "Mozilla/5.0 (Windows NT 5.1; U; en) Opera 8.00"
|
186
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.00"
|
187
|
-
- "Opera/8.00 (Windows NT 5.1; U; en)"
|
188
|
-
- "Mozilla/5.0 (X11; Linux i386; U) Opera 7.60 [en-GB]"
|
189
|
-
- "Opera/7.60 (Windows NT 5.2; U) [en] (IBM EVV/3.0/EAK01AG9/LE)"
|
190
|
-
- "Opera/7.54 (Windows NT 5.1; U) [pl]"
|
191
|
-
- "Opera/7.50 (X11; Linux i686; U) [en]"
|
192
|
-
- "Mozilla/5.0 (X11; Linux i686; U) Opera 7.50 [en]"
|
193
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686) Opera 7.20 [en]"
|
194
|
-
- "Opera/7.11 (Windows NT 5.1; U) [en]"
|
195
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows ME) Opera 7.11 [en]"
|
196
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) Opera 7.02 Bork-edition [en]"
|
197
|
-
- "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 4.0) Opera 7.0 [en]"
|
198
|
-
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.0 [en]"
|
199
|
-
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) Opera 6.01 [en]"
|
200
|
-
- "Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Opera 5.0 [en]"
|
201
|
-
:safari:
|
202
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"
|
203
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4"
|
204
|
-
- "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7"
|
205
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-us) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8"
|
206
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10"
|
207
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-us) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10"
|
208
|
-
- "Mozilla/5.0 (iPod; U; CPU iPhone OS 2_2_1 like Mac OS X; en-us) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5H11a Safari/525.20"
|
209
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-us) AppleWebKit/525.28.3 (KHTML, like Gecko) Version/3.2.3 Safari/525.28.3"
|
210
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Version/3.1.2 Safari/525.21"
|
211
|
-
- "Mozilla/5.0 (iPhone; U; CPU iPhone OS 2_1 like Mac OS X; en-us) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5F136 Safari/525.20"
|
212
|
-
- "Mozilla/5.0 (iPhone; U; CPU iPhone OS 2_0 like Mac OS X; en-us) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5A347 Safari/525.20"
|
213
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Version/3.1.2 Safari/525.21"
|
214
|
-
- "Mozilla/5.0 (iPod; U; CPU iPhone OS 2_0 like Mac OS X; de-de) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5A347 Safari/525.20"
|
215
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Version/3.1.2 Safari/525.21"
|
216
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.17"
|
217
|
-
- "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3"
|
218
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_2; en-us) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13"
|
219
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_2; en-us) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13"
|
220
|
-
- "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/4A93 Safari/419.3"
|
221
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-gb) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6"
|
222
|
-
- "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A100a Safari/419.3"
|
223
|
-
- "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3"
|
224
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en) AppleWebKit/522.11.1 (KHTML, like Gecko) Version/3.0.3 Safari/522.12.1"
|
225
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; bg) AppleWebKit/522.13.1 (KHTML, like Gecko) Version/3.0.2 Safari/522.13.1"
|
226
|
-
- "Mozilla/4.0 (compatible; Mozilla/4.0; Mozilla/5.0; Mozilla/6.0; Safari/431.7; Macintosh; U; PPC Mac OS X 10.6 Leopard; AppleWebKit/421.9 (KHTML, like Gecko) )"
|
227
|
-
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru) AppleWebKit/522.11.3 (KHTML, like Gecko) Version/3.0 Safari/522.11.3"
|
228
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en) AppleWebKit/419.3 (KHTML, like Gecko) Safari/419.3"
|
229
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418.8 (KHTML, like Gecko) Safari/419.3"
|
230
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/417.9 (KHTML, like Gecko) Safari/417.8"
|
231
|
-
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en) AppleWebKit/417.3 (KHTML, like Gecko) Safari/417.2"
|
232
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/412 (KHTML, like Gecko) Safari/412"
|
233
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.9 (KHTML, like Gecko) Safari/312.6"
|
234
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; fr-fr) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
|
235
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; fr-fr) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3"
|
236
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/124 (KHTML, like Gecko) Safari/125.1"
|
237
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/106.2 (KHTML, like Gecko) Safari/100.1"
|
238
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; es) AppleWebKit/85 (KHTML, like Gecko) Safari/85"
|
239
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/74 (KHTML, like Gecko) Safari/74"
|
240
|
-
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/51 (like Gecko) Safari/51"
|
241
|
-
:wget:
|
242
|
-
- "Wget/1.8.1"
|
243
|
-
- "Wget/1.6"
|
244
|
-
:wdg_validator:
|
245
|
-
- "WDG_Validator/1.6.1"
|
246
|
-
:w3c_validator:
|
247
|
-
- "W3C_Validator/1.183 libwww-perl/5.64"
|
@@ -1,258 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
-
# scraping and spidering functionality.
|
4
|
-
#
|
5
|
-
# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
-
#
|
7
|
-
# This file is part of Ronin Web.
|
8
|
-
#
|
9
|
-
# Ronin is free software: you can redistribute it and/or modify
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# Ronin is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with Ronin. If not, see <http://www.gnu.org/licenses/>.
|
21
|
-
#
|
22
|
-
|
23
|
-
require 'ronin/ui/output/helpers'
|
24
|
-
require 'ronin/network/http/proxy'
|
25
|
-
require 'ronin/web/web'
|
26
|
-
require 'ronin/mixin'
|
27
|
-
|
28
|
-
require 'parameters'
|
29
|
-
|
30
|
-
module Ronin
|
31
|
-
module Network
|
32
|
-
module Mixins
|
33
|
-
module Web
|
34
|
-
include Mixin
|
35
|
-
|
36
|
-
mixin UI::Output::Helpers,
|
37
|
-
Parameters
|
38
|
-
|
39
|
-
mixin do
|
40
|
-
# The Web Proxy host to connect to
|
41
|
-
parameter :web_proxy_host, :type => String,
|
42
|
-
:description => 'Web Proxy host'
|
43
|
-
|
44
|
-
# The Web Proxy port to connect to
|
45
|
-
parameter :web_proxy_port, :type => Integer,
|
46
|
-
:description => 'Web Proxy port'
|
47
|
-
|
48
|
-
# The Web Proxy user to authenticate with
|
49
|
-
parameter :web_proxy_user, :type => String,
|
50
|
-
:description => 'Web Proxy authentication user'
|
51
|
-
|
52
|
-
# The Web Proxy password to authenticate with
|
53
|
-
parameter :web_proxy_password, :type => String,
|
54
|
-
:description => 'Web Proxy authentication password'
|
55
|
-
end
|
56
|
-
|
57
|
-
protected
|
58
|
-
|
59
|
-
#
|
60
|
-
# Combines the proxy information set by the `web_proxy_host`,
|
61
|
-
# `web_proxy_port`, `web_proxy_user` and `web_proxy_password`
|
62
|
-
# parameters.
|
63
|
-
#
|
64
|
-
# @return [Network::HTTP::Proxy]
|
65
|
-
# The current proxy information.
|
66
|
-
#
|
67
|
-
# @api semipublic
|
68
|
-
#
|
69
|
-
def web_proxy
|
70
|
-
HTTP::Proxy.new(
|
71
|
-
:host => @web_proxy_host,
|
72
|
-
:port => @web_proxy_port,
|
73
|
-
:user => @web_proxy_user,
|
74
|
-
:password => @web_proxy_password
|
75
|
-
)
|
76
|
-
end
|
77
|
-
|
78
|
-
#
|
79
|
-
# Provides a persistant Mechanize agent.
|
80
|
-
#
|
81
|
-
# @param [Hash] options
|
82
|
-
# Additional options for initializing the agent.
|
83
|
-
#
|
84
|
-
# @option options [Hash] :proxy (web_proxy)
|
85
|
-
# Proxy information.
|
86
|
-
#
|
87
|
-
# @option options [String] :user_agent (web_user_agent)
|
88
|
-
# User-Agent string to use.
|
89
|
-
#
|
90
|
-
# @return [Mechanize]
|
91
|
-
# The agent.
|
92
|
-
#
|
93
|
-
# @api semipublic
|
94
|
-
#
|
95
|
-
def web_agent(options={},&block)
|
96
|
-
unless @web_agent
|
97
|
-
options[:proxy] ||= web_proxy
|
98
|
-
options[:user_agent] ||= @web_user_agent
|
99
|
-
|
100
|
-
@web_agent = Ronin::Web.agent(options,&block)
|
101
|
-
end
|
102
|
-
|
103
|
-
return @web_agent
|
104
|
-
end
|
105
|
-
|
106
|
-
#
|
107
|
-
# Creates a Mechanize Page for the contents at a given URL.
|
108
|
-
#
|
109
|
-
# @param [URI::Generic, String] url
|
110
|
-
# The URL to request.
|
111
|
-
#
|
112
|
-
# @param [Hash] options
|
113
|
-
# Additional options to initialize the agent with.
|
114
|
-
#
|
115
|
-
# @option options [Hash] :proxy (web_proxy)
|
116
|
-
# Proxy information.
|
117
|
-
#
|
118
|
-
# @option options [String] :user_agent (web_user_agent)
|
119
|
-
# User-Agent string to use.
|
120
|
-
#
|
121
|
-
# @yield [page]
|
122
|
-
# If a block is given, it will be passed the page for the
|
123
|
-
# requested URL.
|
124
|
-
#
|
125
|
-
# @yieldparam [Mechanize::Page] page
|
126
|
-
# The requested page.
|
127
|
-
#
|
128
|
-
# @return [Mechanize::Page]
|
129
|
-
# The requested page.
|
130
|
-
#
|
131
|
-
# @api semipublic
|
132
|
-
#
|
133
|
-
def web_get(url,options={})
|
134
|
-
print_info "Requesting #{url}"
|
135
|
-
page = web_agent(options).get(url)
|
136
|
-
|
137
|
-
yield page if block_given?
|
138
|
-
return page
|
139
|
-
end
|
140
|
-
|
141
|
-
#
|
142
|
-
# Requests the body of the Mechanize Page created from the response
|
143
|
-
# of the given URL.
|
144
|
-
#
|
145
|
-
# @param [URI::Generic, String] url
|
146
|
-
# The URL to request.
|
147
|
-
#
|
148
|
-
# @param [Hash] options
|
149
|
-
# Additional options to initialize the agent with.
|
150
|
-
#
|
151
|
-
# @option options [Hash] :proxy (web_proxy)
|
152
|
-
# Proxy information.
|
153
|
-
#
|
154
|
-
# @option options [String] :user_agent (web_user_agent)
|
155
|
-
# User-Agent string to use.
|
156
|
-
#
|
157
|
-
# @yield [body]
|
158
|
-
# If a block is given, it will be passed the body of the page.
|
159
|
-
#
|
160
|
-
# @yieldparam [String] body
|
161
|
-
# The requested body of the page.
|
162
|
-
#
|
163
|
-
# @return [String]
|
164
|
-
# The requested body of the page.
|
165
|
-
#
|
166
|
-
# @api semipublic
|
167
|
-
#
|
168
|
-
def web_get_body(url,options={})
|
169
|
-
page = web_get(url,options)
|
170
|
-
body = page.body
|
171
|
-
|
172
|
-
yield body if block_given?
|
173
|
-
return body
|
174
|
-
end
|
175
|
-
|
176
|
-
#
|
177
|
-
# Posts to a given URL and creates a Mechanize Page from the
|
178
|
-
# response.
|
179
|
-
#
|
180
|
-
# @param [URI::Generic, String] url
|
181
|
-
# The URL to post to.
|
182
|
-
#
|
183
|
-
# @param [Hash] options
|
184
|
-
# Additional options to initialize the agent with.
|
185
|
-
#
|
186
|
-
# @option options [Hash] :query
|
187
|
-
# Additional query parameters to post with.
|
188
|
-
#
|
189
|
-
# @option options [Hash] :proxy (web_proxy)
|
190
|
-
# Proxy information.
|
191
|
-
#
|
192
|
-
# @option options [String] :user_agent (web_user_agent)
|
193
|
-
# User-Agent string to use.
|
194
|
-
#
|
195
|
-
# @yield [page]
|
196
|
-
# If a block is given, it will be passed the page for the
|
197
|
-
# requested URL.
|
198
|
-
#
|
199
|
-
# @yieldparam [Mechanize::Page] page
|
200
|
-
# The requested page.
|
201
|
-
#
|
202
|
-
# @return [Mechanize::Page]
|
203
|
-
# The requested page.
|
204
|
-
#
|
205
|
-
# @api semipublic
|
206
|
-
#
|
207
|
-
def web_post(url,options={})
|
208
|
-
query = {}
|
209
|
-
query.merge!(options[:query]) if options[:query]
|
210
|
-
|
211
|
-
print_info "Posting #{url}"
|
212
|
-
page = web_agent(options).post(url)
|
213
|
-
|
214
|
-
yield page if block_given?
|
215
|
-
return page
|
216
|
-
end
|
217
|
-
|
218
|
-
#
|
219
|
-
# Posts to a given URL and returns the body of the Mechanize Page
|
220
|
-
# created from the response.
|
221
|
-
#
|
222
|
-
# @param [URI::Generic, String] url
|
223
|
-
# The URL to post to.
|
224
|
-
#
|
225
|
-
# @param [Hash] options
|
226
|
-
# Additional options to initialize the agent with.
|
227
|
-
#
|
228
|
-
# @option options [Hash] :query
|
229
|
-
# Additional query parameters to post with.
|
230
|
-
#
|
231
|
-
# @option options [Hash] :proxy (web_proxy)
|
232
|
-
# Proxy information.
|
233
|
-
#
|
234
|
-
# @option options [String] :user_agent (web_user_agent)
|
235
|
-
# User-Agent string to use.
|
236
|
-
#
|
237
|
-
# @yield [body]
|
238
|
-
# If a block is given, it will be passed the body of the page.
|
239
|
-
#
|
240
|
-
# @yieldparam [Mechanize::Page] page
|
241
|
-
# The body of the requested page.
|
242
|
-
#
|
243
|
-
# @return [String]
|
244
|
-
# The requested body of the page.
|
245
|
-
#
|
246
|
-
# @api semipublic
|
247
|
-
#
|
248
|
-
def web_post_body(url,options={})
|
249
|
-
page = web_post(url,options)
|
250
|
-
body = page.body
|
251
|
-
|
252
|
-
yield body if block_given?
|
253
|
-
return body
|
254
|
-
end
|
255
|
-
end
|
256
|
-
end
|
257
|
-
end
|
258
|
-
end
|
data/lib/ronin/web/config.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
-
# scraping and spidering functionality.
|
4
|
-
#
|
5
|
-
# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
-
#
|
7
|
-
# This file is part of Ronin Web.
|
8
|
-
#
|
9
|
-
# Ronin is free software: you can redistribute it and/or modify
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# Ronin is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with Ronin. If not, see <http://www.gnu.org/licenses/>.
|
21
|
-
#
|
22
|
-
|
23
|
-
require 'data_paths'
|
24
|
-
|
25
|
-
module Ronin
|
26
|
-
module Web
|
27
|
-
module Config
|
28
|
-
include DataPaths
|
29
|
-
extend DataPaths::Finders
|
30
|
-
|
31
|
-
register_data_path File.join(File.dirname(__FILE__),'..','..','..','data')
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
@@ -1,47 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
-
# scraping and spidering functionality.
|
4
|
-
#
|
5
|
-
# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
-
#
|
7
|
-
# This file is part of Ronin Web.
|
8
|
-
#
|
9
|
-
# Ronin is free software: you can redistribute it and/or modify
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# Ronin is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with Ronin. If not, see <http://www.gnu.org/licenses/>.
|
21
|
-
#
|
22
|
-
|
23
|
-
require 'nokogiri'
|
24
|
-
|
25
|
-
module Nokogiri
|
26
|
-
module XML
|
27
|
-
class Attr < Node
|
28
|
-
|
29
|
-
#
|
30
|
-
# Determines if the attribute is similar to another attribute.
|
31
|
-
#
|
32
|
-
# @param [Nokogiri::XML::Attr] other
|
33
|
-
# The other attribute.
|
34
|
-
#
|
35
|
-
# @return [Boolean]
|
36
|
-
# Specifies if the attribute is similar, in indentity or value,
|
37
|
-
# to the other attribute.
|
38
|
-
#
|
39
|
-
# @api public
|
40
|
-
#
|
41
|
-
def ==(other)
|
42
|
-
super(other) && (self.value == other.value)
|
43
|
-
end
|
44
|
-
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
@@ -1,48 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
-
# scraping and spidering functionality.
|
4
|
-
#
|
5
|
-
# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
-
#
|
7
|
-
# This file is part of Ronin Web.
|
8
|
-
#
|
9
|
-
# Ronin is free software: you can redistribute it and/or modify
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# Ronin is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with Ronin. If not, see <http://www.gnu.org/licenses/>.
|
21
|
-
#
|
22
|
-
|
23
|
-
require 'nokogiri'
|
24
|
-
|
25
|
-
module Nokogiri
|
26
|
-
module XML
|
27
|
-
class Document < Node
|
28
|
-
|
29
|
-
#
|
30
|
-
# Calculates the sum of all sub-children of the document.
|
31
|
-
#
|
32
|
-
# @return [Integer]
|
33
|
-
# The total number of children and sub-children of the document.
|
34
|
-
# Returns `0` if the document has no root element.
|
35
|
-
#
|
36
|
-
# @api public
|
37
|
-
#
|
38
|
-
def total_children
|
39
|
-
if root
|
40
|
-
1 + root.total_children
|
41
|
-
else
|
42
|
-
0
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|