ronin-web 0.1.3 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data.tar.gz.sig +0 -0
- data/History.txt +25 -0
- data/Manifest.txt +36 -4
- data/README.txt +67 -64
- data/Rakefile +12 -3
- data/bin/ronin-web +1 -1
- data/lib/ronin/network/helpers/web.rb +221 -0
- data/lib/ronin/web.rb +1 -2
- data/lib/ronin/web/extensions.rb +0 -2
- data/lib/ronin/web/extensions/nokogiri.rb +0 -23
- data/lib/ronin/web/proxy.rb +3 -103
- data/lib/ronin/web/proxy/app.rb +31 -0
- data/lib/ronin/web/proxy/base.rb +41 -0
- data/lib/ronin/web/proxy/web.rb +42 -0
- data/lib/ronin/web/server.rb +3 -530
- data/lib/ronin/web/server/app.rb +31 -0
- data/lib/ronin/web/server/base.rb +334 -0
- data/lib/ronin/web/server/files.rb +92 -0
- data/lib/ronin/web/server/helpers.rb +25 -0
- data/lib/ronin/web/server/helpers/files.rb +126 -0
- data/lib/ronin/web/server/helpers/hosts.rb +72 -0
- data/lib/ronin/web/server/helpers/proxy.rb +153 -0
- data/lib/ronin/web/server/helpers/rendering.rb +36 -0
- data/lib/ronin/web/server/hosts.rb +86 -0
- data/lib/ronin/web/server/proxy.rb +116 -0
- data/lib/ronin/web/server/web.rb +62 -0
- data/lib/ronin/web/spider.rb +53 -26
- data/lib/ronin/web/version.rb +1 -3
- data/lib/ronin/web/web.rb +253 -95
- data/spec/spec_helper.rb +1 -1
- data/spec/web/proxy/base_spec.rb +9 -0
- data/spec/web/server/base_spec.rb +86 -0
- data/spec/web/server/classes/files/dir/file.txt +1 -0
- data/spec/web/server/classes/files/dir/index.html +1 -0
- data/spec/web/server/classes/files/dir2/file2.txt +1 -0
- data/spec/web/server/classes/files/dir3/page.xml +4 -0
- data/spec/web/server/classes/files/file.txt +1 -0
- data/spec/web/server/classes/files_app.rb +27 -0
- data/spec/web/server/classes/hosts_app.rb +40 -0
- data/spec/web/server/classes/proxy_app.rb +45 -0
- data/spec/web/server/classes/public1/static1.txt +1 -0
- data/spec/web/server/classes/public2/static2.txt +1 -0
- data/spec/web/server/classes/sub_app.rb +13 -0
- data/spec/web/server/classes/test_app.rb +20 -0
- data/spec/web/server/files_spec.rb +74 -0
- data/spec/web/server/helpers/server.rb +42 -0
- data/spec/web/server/hosts_spec.rb +55 -0
- data/spec/web/server/proxy_spec.rb +49 -0
- data/tasks/spec.rb +1 -0
- data/tasks/yard.rb +13 -0
- metadata +76 -17
- metadata.gz.sig +0 -0
- data/TODO.txt +0 -7
- data/lib/ronin/sessions/web.rb +0 -80
- data/lib/ronin/web/fingerprint.rb +0 -76
- data/spec/web/server_spec.rb +0 -142
@@ -0,0 +1,116 @@
|
|
1
|
+
#
|
2
|
+
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
+
# scraping and spidering functionality.
|
4
|
+
#
|
5
|
+
# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
+
#
|
7
|
+
# This program is free software; you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU General Public License as published by
|
9
|
+
# the Free Software Foundation; either version 2 of the License, or
|
10
|
+
# (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU General Public License
|
18
|
+
# along with this program; if not, write to the Free Software
|
19
|
+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
20
|
+
#
|
21
|
+
|
22
|
+
require 'ronin/web/server/helpers/proxy'
|
23
|
+
|
24
|
+
module Ronin
|
25
|
+
module Web
|
26
|
+
module Server
|
27
|
+
module Proxy
|
28
|
+
def self.included(base)
|
29
|
+
base.module_eval do
|
30
|
+
#
|
31
|
+
# Proxies requests to a given path.
|
32
|
+
#
|
33
|
+
# @param [String] path
|
34
|
+
# The path to proxy requests for.
|
35
|
+
#
|
36
|
+
# @param [Hash] options
|
37
|
+
# Additional options.
|
38
|
+
#
|
39
|
+
# @yield [(response), body]
|
40
|
+
# If a block is given, it will be passed the optional
|
41
|
+
# response of the proxied request and the body received
|
42
|
+
# from the proxied request.
|
43
|
+
#
|
44
|
+
# @yieldparam [Net::HTTP::Response] response
|
45
|
+
# The response.
|
46
|
+
#
|
47
|
+
# @yieldparam [String] body
|
48
|
+
# The body from the response.
|
49
|
+
#
|
50
|
+
# @example
|
51
|
+
# proxy '/login.php' do |body|
|
52
|
+
# body.gsub(/https/,'http')
|
53
|
+
# end
|
54
|
+
#
|
55
|
+
# @example
|
56
|
+
# proxy '/login*' do |response,body|
|
57
|
+
# end
|
58
|
+
#
|
59
|
+
# @since 0.2.0
|
60
|
+
#
|
61
|
+
def self.proxy(path,options={},&block)
|
62
|
+
any(path) do
|
63
|
+
proxy(options,&block)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
#
|
68
|
+
# Proxies requests to a given path.
|
69
|
+
#
|
70
|
+
# @param [String] path
|
71
|
+
# The path to proxy requests for.
|
72
|
+
#
|
73
|
+
# @param [Hash] options
|
74
|
+
# Additional options.
|
75
|
+
#
|
76
|
+
# @yield [(response), page]
|
77
|
+
# If a block is given, it will be passed the optional
|
78
|
+
# response of the proxied request and the page from the
|
79
|
+
# proxied request.
|
80
|
+
#
|
81
|
+
# @yieldparam [Net::HTTP::Response] response
|
82
|
+
# The response.
|
83
|
+
#
|
84
|
+
# @yieldparam [Nokogiri::HTML, Nokogiri::XML] page
|
85
|
+
# The page from the response.
|
86
|
+
#
|
87
|
+
# @example
|
88
|
+
# proxy '/login.php' do |page|
|
89
|
+
# body.search('@action').each do |action|
|
90
|
+
# action.inner_text = action.inner_text.gsub(
|
91
|
+
# /https/, 'http'
|
92
|
+
# )
|
93
|
+
# end
|
94
|
+
# end
|
95
|
+
#
|
96
|
+
# @example
|
97
|
+
# proxy '/login*' do |response,body|
|
98
|
+
# end
|
99
|
+
#
|
100
|
+
# @since 0.2.0
|
101
|
+
#
|
102
|
+
def self.proxy_page(path,options={},&block)
|
103
|
+
any(path) do
|
104
|
+
proxy_page(options,&block)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
protected
|
109
|
+
|
110
|
+
helpers Ronin::Web::Server::Helpers::Proxy
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
#
|
2
|
+
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
+
# scraping and spidering functionality.
|
4
|
+
#
|
5
|
+
# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
+
#
|
7
|
+
# This program is free software; you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU General Public License as published by
|
9
|
+
# the Free Software Foundation; either version 2 of the License, or
|
10
|
+
# (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU General Public License
|
18
|
+
# along with this program; if not, write to the Free Software
|
19
|
+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
20
|
+
#
|
21
|
+
|
22
|
+
require 'ronin/web/server/app'
|
23
|
+
|
24
|
+
module Ronin
|
25
|
+
module Web
|
26
|
+
#
|
27
|
+
# Returns the Ronin Web Server.
|
28
|
+
#
|
29
|
+
# @param [Hash] options
|
30
|
+
# Additional options.
|
31
|
+
#
|
32
|
+
# @yield [server]
|
33
|
+
# If a block is given, it will be passed the current web server.
|
34
|
+
#
|
35
|
+
# @yieldparam [Server::App]
|
36
|
+
# The current web server class.
|
37
|
+
#
|
38
|
+
# @return [Server::App]
|
39
|
+
# The current web server class.
|
40
|
+
#
|
41
|
+
# @example
|
42
|
+
# Web.server do
|
43
|
+
# get '/hello' do
|
44
|
+
# 'world'
|
45
|
+
# end
|
46
|
+
# end
|
47
|
+
#
|
48
|
+
# @see Server::Base.run!
|
49
|
+
# @since 0.2.0
|
50
|
+
#
|
51
|
+
def Web.server(options={},&block)
|
52
|
+
unless class_variable_defined?('@@ronin_web_server')
|
53
|
+
@@ronin_web_server = Server::App
|
54
|
+
@@ronin_web_server.run!(options.merge(:background => true))
|
55
|
+
end
|
56
|
+
|
57
|
+
@@ronin_web_server.class_eval(&block)
|
58
|
+
|
59
|
+
return @@ronin_web_server
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/ronin/web/spider.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
#
|
2
|
-
#--
|
3
2
|
# Ronin Web - A Ruby library for Ronin that provides support for web
|
4
3
|
# scraping and spidering functionality.
|
5
4
|
#
|
@@ -18,11 +17,10 @@
|
|
18
17
|
# You should have received a copy of the GNU General Public License
|
19
18
|
# along with this program; if not, write to the Free Software
|
20
19
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
21
|
-
#++
|
22
20
|
#
|
23
21
|
|
24
22
|
require 'ronin/web/web'
|
25
|
-
require 'ronin/ui/
|
23
|
+
require 'ronin/ui/output/helpers'
|
26
24
|
|
27
25
|
require 'spidr/agent'
|
28
26
|
|
@@ -30,31 +28,60 @@ module Ronin
|
|
30
28
|
module Web
|
31
29
|
class Spider < Spidr::Agent
|
32
30
|
|
33
|
-
include UI::
|
31
|
+
include UI::Output::Helpers
|
34
32
|
|
35
33
|
#
|
36
|
-
# Creates a new Spider object
|
37
|
-
#
|
38
|
-
#
|
39
|
-
#
|
40
|
-
#
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
#
|
45
|
-
#
|
46
|
-
#
|
47
|
-
#
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
51
|
-
#
|
52
|
-
#
|
53
|
-
#
|
54
|
-
#
|
55
|
-
#
|
56
|
-
#
|
57
|
-
#
|
34
|
+
# Creates a new Spider object.
|
35
|
+
#
|
36
|
+
# @param [Hash] options
|
37
|
+
# Additional options.
|
38
|
+
#
|
39
|
+
# @option options [Hash] :proxy (Web.proxy)
|
40
|
+
# The proxy to use while spidering.
|
41
|
+
#
|
42
|
+
# @option options [String] :user_agent
|
43
|
+
# The User-Agent string to send.
|
44
|
+
#
|
45
|
+
# @option options [String] :referer
|
46
|
+
# The referer URL to send.
|
47
|
+
#
|
48
|
+
# @option options [Integer] :delay (0)
|
49
|
+
# Duration in seconds to pause between spidering each link.
|
50
|
+
#
|
51
|
+
# @option options [String, Regexp] :host
|
52
|
+
# The host-name to visit.
|
53
|
+
#
|
54
|
+
# @option options [Array] :hosts
|
55
|
+
# Patterns of host-names to visit.
|
56
|
+
#
|
57
|
+
# @option options [Array] :ignore_hosts
|
58
|
+
# Patterns of host-names not to visit.
|
59
|
+
#
|
60
|
+
# @option options [Array] :ports
|
61
|
+
# Ports to visit.
|
62
|
+
#
|
63
|
+
# @option options [Array] :ignore_ports
|
64
|
+
# Ports not to visit.
|
65
|
+
#
|
66
|
+
# @option options [Array] :links
|
67
|
+
# Patterns of links to visit.
|
68
|
+
#
|
69
|
+
# @option options [Array] :ignore_links
|
70
|
+
# Patterns of links not to visit.
|
71
|
+
#
|
72
|
+
# @option options [Array] :ext
|
73
|
+
# Patterns of file extensions to accept.
|
74
|
+
#
|
75
|
+
# @option options [Array] :ignore_exts
|
76
|
+
# Patterns of file extensions to ignore.
|
77
|
+
#
|
78
|
+
# @yield [spider]
|
79
|
+
# If a block is given, it will be passed the newly created spider.
|
80
|
+
#
|
81
|
+
# @yieldparam [Spider] spider
|
82
|
+
# The newly created spider.
|
83
|
+
#
|
84
|
+
# @see http://spidr.rubyforge.org/docs/classes/Spidr/Agent.html
|
58
85
|
#
|
59
86
|
def initialize(options={},&block)
|
60
87
|
options = {
|
data/lib/ronin/web/version.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
#
|
2
|
-
#--
|
3
2
|
# Ronin Web - A Ruby library for Ronin that provides support for web
|
4
3
|
# scraping and spidering functionality.
|
5
4
|
#
|
@@ -18,12 +17,11 @@
|
|
18
17
|
# You should have received a copy of the GNU General Public License
|
19
18
|
# along with this program; if not, write to the Free Software
|
20
19
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
21
|
-
#++
|
22
20
|
#
|
23
21
|
|
24
22
|
module Ronin
|
25
23
|
module Web
|
26
24
|
# Ronin Web Version
|
27
|
-
VERSION = '0.
|
25
|
+
VERSION = '0.2.0'
|
28
26
|
end
|
29
27
|
end
|
data/lib/ronin/web/web.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
#
|
2
|
-
#--
|
3
2
|
# Ronin Web - A Ruby library for Ronin that provides support for web
|
4
3
|
# scraping and spidering functionality.
|
5
4
|
#
|
@@ -18,7 +17,6 @@
|
|
18
17
|
# You should have received a copy of the GNU General Public License
|
19
18
|
# along with this program; if not, write to the Free Software
|
20
19
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
21
|
-
#++
|
22
20
|
#
|
23
21
|
|
24
22
|
require 'ronin/network/http'
|
@@ -31,8 +29,19 @@ require 'open-uri'
|
|
31
29
|
module Ronin
|
32
30
|
module Web
|
33
31
|
#
|
34
|
-
#
|
35
|
-
#
|
32
|
+
# Parses the body of a document into a HTML document object.
|
33
|
+
#
|
34
|
+
# @param [String, IO] body
|
35
|
+
# The body of the document to parse.
|
36
|
+
#
|
37
|
+
# @yield [doc]
|
38
|
+
# If a block is given, it will be passed the newly created document
|
39
|
+
# object.
|
40
|
+
#
|
41
|
+
# @yieldparam [Nokogiri::HTML::Document] doc
|
42
|
+
# The new HTML document object.
|
43
|
+
#
|
44
|
+
# @return [Nokogiri::HTML::Document] The new HTML document object.
|
36
45
|
#
|
37
46
|
def Web.html(body,&block)
|
38
47
|
doc = Nokogiri::HTML(body)
|
@@ -42,8 +51,15 @@ module Ronin
|
|
42
51
|
end
|
43
52
|
|
44
53
|
#
|
45
|
-
# Creates a new Nokogiri::HTML::Builder
|
54
|
+
# Creates a new Nokogiri::HTML::Builder.
|
55
|
+
#
|
56
|
+
# @yield []
|
57
|
+
# The block that will be used to construct the HTML document.
|
46
58
|
#
|
59
|
+
# @return [Nokogiri::HTML::Builder]
|
60
|
+
# The new HTML builder object.
|
61
|
+
#
|
62
|
+
# @example
|
47
63
|
# Web.build_html do
|
48
64
|
# body {
|
49
65
|
# div(:style => 'display:none;') {
|
@@ -57,8 +73,19 @@ module Ronin
|
|
57
73
|
end
|
58
74
|
|
59
75
|
#
|
60
|
-
#
|
61
|
-
#
|
76
|
+
# Parses the body of a document into a XML document object.
|
77
|
+
#
|
78
|
+
# @param [String, IO] body
|
79
|
+
# The body of the document to parse.
|
80
|
+
#
|
81
|
+
# @yield [doc]
|
82
|
+
# If a block is given, it will be passed the newly created document
|
83
|
+
# object.
|
84
|
+
#
|
85
|
+
# @yieldparam [Nokogiri::XML::Document] doc
|
86
|
+
# The new XML document object.
|
87
|
+
#
|
88
|
+
# @return [Nokogiri::XML::Document] The new XML document object.
|
62
89
|
#
|
63
90
|
def Web.xml(body,&block)
|
64
91
|
doc = Nokogiri::XML(body)
|
@@ -68,8 +95,15 @@ module Ronin
|
|
68
95
|
end
|
69
96
|
|
70
97
|
#
|
71
|
-
# Creates a new Nokogiri::XML::Builder
|
98
|
+
# Creates a new Nokogiri::XML::Builder.
|
72
99
|
#
|
100
|
+
# @yield []
|
101
|
+
# The block that will be used to construct the XML document.
|
102
|
+
#
|
103
|
+
# @return [Nokogiri::XML::Builder]
|
104
|
+
# The new XML builder object.
|
105
|
+
#
|
106
|
+
# @example
|
73
107
|
# Web.build_xml do
|
74
108
|
# post(:id => 2) {
|
75
109
|
# title { text("some example) }
|
@@ -82,104 +116,113 @@ module Ronin
|
|
82
116
|
end
|
83
117
|
|
84
118
|
#
|
85
|
-
#
|
119
|
+
# Proxy information for Ronin::Web to use.
|
86
120
|
#
|
87
|
-
|
88
|
-
|
89
|
-
end
|
90
|
-
|
121
|
+
# @return [Network::HTTP::Proxy]
|
122
|
+
# The Ronin Web proxy information.
|
91
123
|
#
|
92
|
-
#
|
93
|
-
#
|
94
|
-
def Web.default_proxy_port=(port)
|
95
|
-
Network::HTTP.default_proxy_port = port
|
96
|
-
end
|
97
|
-
|
98
|
-
#
|
99
|
-
# Returns the +Hash+ of the Ronin Web proxy information.
|
124
|
+
# @see Ronin::Network::HTTP.proxy
|
100
125
|
#
|
101
126
|
def Web.proxy
|
102
127
|
Network::HTTP.proxy
|
103
128
|
end
|
104
129
|
|
105
130
|
#
|
106
|
-
#
|
131
|
+
# Creates a HTTP URI based on a Hash of proxy information.
|
107
132
|
#
|
108
|
-
|
109
|
-
|
110
|
-
end
|
111
|
-
|
133
|
+
# @param [Network::HTTP::Proxy, Hash] proxy_info
|
134
|
+
# The proxy information.
|
112
135
|
#
|
113
|
-
#
|
114
|
-
#
|
136
|
+
# @return [URI::HTTP, nil]
|
137
|
+
# The HTTP URI that represents the proxy. If the proxy is diabled,
|
138
|
+
# +nil+ will be returned.
|
115
139
|
#
|
116
140
|
def Web.proxy_url(proxy_info=Web.proxy)
|
117
|
-
|
118
|
-
userinfo = nil
|
119
|
-
|
120
|
-
if (Web.proxy[:user] || Web.proxy[:password])
|
121
|
-
userinfo = "#{Web.proxy[:user]}:#{Web.proxy[:password]}"
|
122
|
-
end
|
123
|
-
|
124
|
-
return URI::HTTP.build(
|
125
|
-
:host => Web.proxy[:host],
|
126
|
-
:port => Web.proxy[:port],
|
127
|
-
:userinfo => userinfo,
|
128
|
-
:path => '/'
|
129
|
-
)
|
130
|
-
end
|
141
|
+
Network::HTTP::Proxy.new(proxy_info).url
|
131
142
|
end
|
132
143
|
|
133
144
|
#
|
134
|
-
#
|
145
|
+
# @return [Array]
|
146
|
+
# The supported Web User-Agent Aliases.
|
135
147
|
#
|
136
148
|
def Web.user_agent_aliases
|
137
149
|
WWW::Mechanize::AGENT_ALIASES
|
138
150
|
end
|
139
151
|
|
140
152
|
#
|
141
|
-
#
|
153
|
+
# @return [String, nil]
|
154
|
+
# The Ronin Web User-Agent
|
142
155
|
#
|
143
156
|
def Web.user_agent
|
144
157
|
Network::HTTP.user_agent
|
145
158
|
end
|
146
159
|
|
147
160
|
#
|
148
|
-
# Sets the Ronin Web User-Agent
|
161
|
+
# Sets the Ronin Web User-Agent.
|
162
|
+
#
|
163
|
+
# @param [String] new_agent
|
164
|
+
# The User-Agent string to use.
|
165
|
+
#
|
166
|
+
# @return [String]
|
167
|
+
# The new User-Agent string.
|
149
168
|
#
|
150
169
|
def Web.user_agent=(new_agent)
|
151
170
|
Network::HTTP.user_agent = new_agent
|
152
171
|
end
|
153
172
|
|
154
173
|
#
|
155
|
-
# Sets the Ronin Web User-Agent
|
156
|
-
#
|
174
|
+
# Sets the Ronin Web User-Agent.
|
175
|
+
#
|
176
|
+
# @param [String] name
|
177
|
+
# The User-Agent alias to use.
|
178
|
+
#
|
179
|
+
# @return [String]
|
180
|
+
# The new User-Agent string.
|
157
181
|
#
|
158
182
|
def Web.user_agent_alias=(name)
|
159
183
|
Network::HTTP.user_agent = Web.user_agent_aliases[name.to_s]
|
160
184
|
end
|
161
185
|
|
162
186
|
#
|
163
|
-
# Opens
|
164
|
-
#
|
187
|
+
# Opens a URL as a temporary file.
|
188
|
+
#
|
189
|
+
# @param [Hash] options
|
190
|
+
# Additional options.
|
191
|
+
#
|
192
|
+
# @option options [String] :user_agent_alias
|
193
|
+
# The User-Agent Alias to use.
|
194
|
+
#
|
195
|
+
# @option options [String] :user_agent
|
196
|
+
# The User-Agent string to use.
|
165
197
|
#
|
166
|
-
#
|
167
|
-
#
|
168
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
169
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
170
|
-
# <tt>:user</tt>:: The HTTP Basic Authentication user name.
|
171
|
-
# <tt>:password</tt>:: The HTTP Basic Authentication password.
|
172
|
-
# <tt>:content_length_proc</tt>:: A callback which will be passed the
|
173
|
-
# content-length of the HTTP response.
|
174
|
-
# <tt>:progress_proc</tt>:: A callback which will be passed the size
|
175
|
-
# of each fragment, once received from the
|
176
|
-
# server.
|
198
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
199
|
+
# Proxy information.
|
177
200
|
#
|
201
|
+
# @option options [String] :user
|
202
|
+
# The HTTP Basic Authentication user name.
|
203
|
+
#
|
204
|
+
# @option options [String] :password
|
205
|
+
# The HTTP Basic Authentication password.
|
206
|
+
#
|
207
|
+
# @option options [Proc] :content_length_proc
|
208
|
+
# A callback which will be passed the content-length of the HTTP
|
209
|
+
# response.
|
210
|
+
#
|
211
|
+
# @option options [Proc] :progress_proc
|
212
|
+
# A callback which will be passed the size of each fragment, once
|
213
|
+
# received from the server.
|
214
|
+
#
|
215
|
+
# @return [File]
|
216
|
+
# The contents of the URL.
|
217
|
+
#
|
218
|
+
# @example Open a given URL.
|
178
219
|
# Web.open('http://www.hackety.org/')
|
179
220
|
#
|
221
|
+
# @example Open a given URL, using a custom User-Agent alias.
|
180
222
|
# Web.open('http://tenderlovemaking.com/',
|
181
223
|
# :user_agent_alias => 'Linux Mozilla')
|
182
224
|
#
|
225
|
+
# @example Open a given URL, using a custom User-Agent string.
|
183
226
|
# Web.open('http://www.wired.com/', :user_agent => 'the future')
|
184
227
|
#
|
185
228
|
def Web.open(url,options={})
|
@@ -216,17 +259,41 @@ module Ronin
|
|
216
259
|
end
|
217
260
|
|
218
261
|
#
|
219
|
-
# Creates a new Mechanize
|
262
|
+
# Creates a new Mechanize Agent.
|
263
|
+
#
|
264
|
+
# @param [Hash] options
|
265
|
+
# Additional options.
|
266
|
+
#
|
267
|
+
# @option options [String] :user_agent_alias
|
268
|
+
# The User-Agent Alias to use.
|
269
|
+
#
|
270
|
+
# @option options [String] :user_agent
|
271
|
+
# The User-Agent string to use.
|
272
|
+
#
|
273
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
274
|
+
# Proxy information.
|
220
275
|
#
|
221
|
-
#
|
222
|
-
#
|
223
|
-
#
|
224
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
276
|
+
# @yield [agent]
|
277
|
+
# If a block is given, it will be passed the newly created Mechanize
|
278
|
+
# agent.
|
225
279
|
#
|
280
|
+
# @yieldparam [WWW::Mechanize] agent
|
281
|
+
# The new Mechanize agent.
|
282
|
+
#
|
283
|
+
# @return [WWW::Mechanize]
|
284
|
+
# The new Mechanize agent.
|
285
|
+
#
|
286
|
+
# @example Create a new agent.
|
226
287
|
# Web.agent
|
288
|
+
#
|
289
|
+
# @example Create a new agent, with a custom User-Agent alias.
|
227
290
|
# Web.agent(:user_agent_alias => 'Linux Mozilla')
|
291
|
+
#
|
292
|
+
# @example Create a new agent, with a custom User-Agent string.
|
228
293
|
# Web.agent(:user_agent => 'wooden pants')
|
229
294
|
#
|
295
|
+
# @see http://mechanize.rubyforge.org/mechanize/WWW/Mechanize.html
|
296
|
+
#
|
230
297
|
def Web.agent(options={},&block)
|
231
298
|
agent = WWW::Mechanize.new
|
232
299
|
|
@@ -248,22 +315,46 @@ module Ronin
|
|
248
315
|
end
|
249
316
|
|
250
317
|
#
|
251
|
-
#
|
252
|
-
#
|
318
|
+
# Creates a Mechanize Page for the contents at a given URL.
|
319
|
+
#
|
320
|
+
# @param [URI::Generic, String] url
|
321
|
+
# The URL to request.
|
322
|
+
#
|
323
|
+
# @param [Hash] options
|
324
|
+
# Additional options.
|
325
|
+
#
|
326
|
+
# @option options [String] :user_agent_alias
|
327
|
+
# The User-Agent Alias to use.
|
328
|
+
#
|
329
|
+
# @option options [String] :user_agent
|
330
|
+
# The User-Agent string to use.
|
253
331
|
#
|
254
|
-
#
|
255
|
-
#
|
256
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
257
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
332
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
333
|
+
# Proxy information.
|
258
334
|
#
|
259
|
-
#
|
335
|
+
# @yield [page]
|
336
|
+
# If a block is given, it will be passed the page for the requested
|
337
|
+
# URL.
|
260
338
|
#
|
339
|
+
# @yieldparam [WWW::Mechanize::Page] page
|
340
|
+
# The requested page.
|
341
|
+
#
|
342
|
+
# @return [WWW::Mechanize::Page]
|
343
|
+
# The requested page.
|
344
|
+
#
|
345
|
+
# @example
|
346
|
+
# Web.get('http://www.0x000000.com')
|
347
|
+
# # => WWW::Mechanize::Page
|
348
|
+
#
|
349
|
+
# @example
|
261
350
|
# Web.get('http://www.rubyinside.com') do |page|
|
262
351
|
# page.search('div.post/h2/a').each do |title|
|
263
352
|
# puts title.inner_text
|
264
353
|
# end
|
265
354
|
# end
|
266
355
|
#
|
356
|
+
# @see http://mechanize.rubyforge.org/mechanize/WWW/Mechanize/Page.html
|
357
|
+
#
|
267
358
|
def Web.get(url,options={},&block)
|
268
359
|
page = Web.agent(options).get(url)
|
269
360
|
|
@@ -272,17 +363,37 @@ module Ronin
|
|
272
363
|
end
|
273
364
|
|
274
365
|
#
|
275
|
-
#
|
276
|
-
# of the
|
277
|
-
#
|
366
|
+
# Requests the body of the Mechanize Page created from the response
|
367
|
+
# of the given URL.
|
368
|
+
#
|
369
|
+
# @param [URI::Generic, String] url
|
370
|
+
# The URL to request.
|
371
|
+
#
|
372
|
+
# @param [Hash] options
|
373
|
+
# Additional options.
|
374
|
+
#
|
375
|
+
# @option options [String] :user_agent_alias
|
376
|
+
# The User-Agent Alias to use.
|
377
|
+
#
|
378
|
+
# @option options [String] :user_agent
|
379
|
+
# The User-Agent string to use.
|
380
|
+
#
|
381
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
382
|
+
# Proxy information.
|
278
383
|
#
|
279
|
-
#
|
280
|
-
#
|
281
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
282
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
384
|
+
# @yield [body]
|
385
|
+
# If a block is given, it will be passed the body of the page.
|
283
386
|
#
|
387
|
+
# @yieldparam [String] body
|
388
|
+
# The requested body of the page.
|
389
|
+
#
|
390
|
+
# @return [String]
|
391
|
+
# The requested body of the page.
|
392
|
+
#
|
393
|
+
# @example
|
284
394
|
# Web.get_body('http://www.rubyinside.com') # => String
|
285
395
|
#
|
396
|
+
# @example
|
286
397
|
# Web.get_body('http://www.rubyinside.com') do |body|
|
287
398
|
# puts body
|
288
399
|
# end
|
@@ -295,16 +406,39 @@ module Ronin
|
|
295
406
|
end
|
296
407
|
|
297
408
|
#
|
298
|
-
# Posts
|
299
|
-
#
|
409
|
+
# Posts to a given URL and creates a Mechanize Page from the response.
|
410
|
+
#
|
411
|
+
# @param [URI::Generic, String] url
|
412
|
+
# The URL to request.
|
413
|
+
#
|
414
|
+
# @param [Hash] options
|
415
|
+
# Additional options.
|
416
|
+
#
|
417
|
+
# @option options [Hash] :query
|
418
|
+
# Additional query parameters to post with.
|
419
|
+
#
|
420
|
+
# @option options [String] :user_agent_alias
|
421
|
+
# The User-Agent Alia to use.
|
422
|
+
#
|
423
|
+
# @option options [String] :user_agent
|
424
|
+
# The User-Agent string to use.
|
425
|
+
#
|
426
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
427
|
+
# Proxy information.
|
428
|
+
#
|
429
|
+
# @yield [page]
|
430
|
+
# If a block is given, it will be passed the page for the requested
|
431
|
+
# URL.
|
300
432
|
#
|
301
|
-
#
|
302
|
-
#
|
303
|
-
# <tt>:user_agent_alias</tt>:: The User-Agent Alias to use.
|
304
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
305
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
433
|
+
# @yieldparam [WWW::Mechanize::Page] page
|
434
|
+
# The requested page.
|
306
435
|
#
|
307
|
-
#
|
436
|
+
# @return [WWW::Mechanize::Page]
|
437
|
+
# The requested page.
|
438
|
+
#
|
439
|
+
# @example
|
440
|
+
# Web.post('http://www.rubyinside.com')
|
441
|
+
# # => WWW::Mechanize::Page
|
308
442
|
#
|
309
443
|
def Web.post(url,options={},&block)
|
310
444
|
query = {}
|
@@ -317,17 +451,41 @@ module Ronin
|
|
317
451
|
end
|
318
452
|
|
319
453
|
#
|
320
|
-
#
|
321
|
-
#
|
322
|
-
#
|
454
|
+
# Posts to a given URL and returns the body of the Mechanize Page
|
455
|
+
# created from the response.
|
456
|
+
#
|
457
|
+
# @param [URI::Generic, String] url
|
458
|
+
# The URL to request.
|
459
|
+
#
|
460
|
+
# @param [Hash] options
|
461
|
+
# Additional options.
|
462
|
+
#
|
463
|
+
# @option options [Hash] :query
|
464
|
+
# Additional query parameters to post with.
|
465
|
+
#
|
466
|
+
# @option options [String] :user_agent_alias
|
467
|
+
# The User-Agent Alias to use.
|
468
|
+
#
|
469
|
+
# @option options [String] :user_agent
|
470
|
+
# The User-Agent string to use.
|
471
|
+
#
|
472
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
473
|
+
# Proxy information.
|
474
|
+
#
|
475
|
+
# @yield [body]
|
476
|
+
# If a block is given, it will be passed the body of the page.
|
477
|
+
#
|
478
|
+
# @yieldparam [WWW::Mechanize::Page] page
|
479
|
+
# The body of the requested page.
|
323
480
|
#
|
324
|
-
#
|
325
|
-
#
|
326
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
327
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
481
|
+
# @return [WWW::Mechanize::Page]
|
482
|
+
# The body of the requested page.
|
328
483
|
#
|
329
|
-
#
|
484
|
+
# @example
|
485
|
+
# Web.post_body('http://www.rubyinside.com')
|
486
|
+
# # => String
|
330
487
|
#
|
488
|
+
# @example
|
331
489
|
# Web.post_body('http://www.rubyinside.com') do |body|
|
332
490
|
# puts body
|
333
491
|
# end
|