tiny_grabber 0.0.7 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +7 -2
- data/lib/tiny_grabber/version.rb +1 -1
- data/lib/tiny_grabber.rb +101 -33
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9125f8132b04b1a08e64a47abf7ccb5085688b55
|
4
|
+
data.tar.gz: 050aad4742cdeeba67cfc64dad8c6d9c0a02ec97
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1ff3deb449bc1aedf86b2886156043decf28cf327ec25e30cd4725f07f5dc82faa01c803aa161bf573e3799f747a15673919fb470db75546810a932ee513093
|
7
|
+
data.tar.gz: c977e50867454e3af09e00e6b9f69f5b8efc7b22aaa0127b3c54c8aee206692359322a0c4418b81ab23bbc4a54a562f197f9fc98088225d97a70d1d1699d83f2
|
data/README.md
CHANGED
@@ -44,8 +44,11 @@ auth = { username: '', password: '' }
|
|
44
44
|
# Request HTTP type by default is GET. You can send POST request with setting post params. Also you cat send empty POST request.
|
45
45
|
post = { some_data: '' }
|
46
46
|
|
47
|
-
#
|
48
|
-
response = TinyGrabber.get url, headers: headers, proxy: proxy, auth: auth
|
47
|
+
# HTTP GET request
|
48
|
+
response = TinyGrabber.get url, headers: headers, proxy: proxy, auth: auth
|
49
|
+
|
50
|
+
# HTTP GET request
|
51
|
+
response = TinyGrabber.post url, post, headers: headers, proxy: proxy, auth: auth
|
49
52
|
|
50
53
|
# HTTP answer code
|
51
54
|
p response.code
|
@@ -62,6 +65,8 @@ p response.cookies
|
|
62
65
|
|
63
66
|
## Changelog
|
64
67
|
|
68
|
+
* *v 0.1.0*
|
69
|
+
* Add TinyGrabber.post method for HTTP POST request
|
65
70
|
* *v 0.0.7*
|
66
71
|
* Add POST request
|
67
72
|
* Add Basic Authentication
|
data/lib/tiny_grabber/version.rb
CHANGED
data/lib/tiny_grabber.rb
CHANGED
@@ -8,10 +8,15 @@ require 'tiny_grabber/http'
|
|
8
8
|
# Main class for TinyGrabber
|
9
9
|
class TinyGrabber
|
10
10
|
|
11
|
+
@uri
|
12
|
+
@params
|
13
|
+
@http
|
14
|
+
@request
|
15
|
+
|
11
16
|
##
|
12
17
|
# Get Net::HTTP object for content from remote single page
|
13
|
-
# [url (string)] Link
|
14
|
-
# [params (hash)]
|
18
|
+
# [url (string)] Link to resource
|
19
|
+
# [params (hash)] Request params
|
15
20
|
# * [proxy (hash)] Configuration of remote proxy server
|
16
21
|
# * *ip* address of remote proxy server
|
17
22
|
# * *port* of remote proxy server
|
@@ -22,58 +27,121 @@ class TinyGrabber
|
|
22
27
|
# * [auth (hash)] Basic Authentication
|
23
28
|
# * *username* Authenticate username
|
24
29
|
# * *password* Authenticate password
|
25
|
-
# * [post (hash)] POST data
|
26
30
|
#
|
27
31
|
# = Example
|
28
32
|
# TinyGrabber.get url, headers: { 'Content-Type' => 'application/json' }, proxy: { ip: 'xx.xx.xx.xx', port: xx, proxy_type: :socks }
|
29
33
|
#
|
30
|
-
# @param url Link
|
31
|
-
# @param params
|
34
|
+
# @param url Link to resource
|
35
|
+
# @param params Request params
|
32
36
|
#
|
33
37
|
def self.get url, params = {}
|
34
|
-
uri =
|
38
|
+
@uri = set_url url
|
39
|
+
@params = convert_params_to_sym params
|
40
|
+
@http = set_http_connect
|
41
|
+
@request = Net::HTTP::Get.new(@uri.request_uri)
|
42
|
+
set_basic_auth
|
43
|
+
set_headers
|
44
|
+
get_request
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
##
|
49
|
+
# Get Net::HTTP object for content from remote single page
|
50
|
+
# [url (string)] Link to resource
|
51
|
+
# [data (hash)] Post data
|
52
|
+
# [params (hash)] Request params
|
53
|
+
# * [proxy (hash)] Configuration of remote proxy server
|
54
|
+
# * *ip* address of remote proxy server
|
55
|
+
# * *port* of remote proxy server
|
56
|
+
# * *proxy_type* of remote proxy server
|
57
|
+
# * *http* for http(s) proxy servers (by default)
|
58
|
+
# * *socks* for socks4/5 proxy servers
|
59
|
+
# * [headers (hash)] Headers
|
60
|
+
# * [auth (hash)] Basic Authentication
|
61
|
+
# * *username* Authenticate username
|
62
|
+
# * *password* Authenticate password
|
63
|
+
#
|
64
|
+
# = Example
|
65
|
+
# TinyGrabber.post url, data, headers: { 'Content-Type' => 'application/json' }, proxy: { ip: 'xx.xx.xx.xx', port: xx, proxy_type: :socks }
|
66
|
+
#
|
67
|
+
# @param url Link to resource
|
68
|
+
# @param params Request params
|
69
|
+
#
|
70
|
+
def self.post url, data, params = {}
|
71
|
+
@uri = set_url url
|
72
|
+
data = convert_params_to_sym data
|
73
|
+
@params = convert_params_to_sym params
|
74
|
+
@http = set_http_connect
|
75
|
+
@request = Net::HTTP::Post.new(@uri.request_uri)
|
76
|
+
@request.set_form_data(data)
|
77
|
+
set_basic_auth
|
78
|
+
set_headers
|
79
|
+
get_request
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
35
83
|
|
36
|
-
|
84
|
+
# Convert all params key to symbols
|
85
|
+
#
|
86
|
+
# @param params Request params
|
87
|
+
#
|
88
|
+
def self.convert_params_to_sym params
|
89
|
+
params.is_a?(Hash) ? params.inject({}) { |h, (k, v)| h[k.to_sym] = convert_params_to_sym v; h } : params
|
90
|
+
end
|
91
|
+
|
92
|
+
# Convert URL to URI
|
93
|
+
#
|
94
|
+
# @param url Link to resource
|
95
|
+
#
|
96
|
+
def self.set_url url
|
97
|
+
URI(URI.escape(url))
|
98
|
+
end
|
37
99
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
100
|
+
|
101
|
+
# Use proxy for request
|
102
|
+
#
|
103
|
+
def self.set_http_connect
|
104
|
+
if @params[:proxy]
|
105
|
+
if ['socks', :socks].include?(@params[:proxy][:proxy_type])
|
106
|
+
Net::HTTP.SOCKSProxy(@params[:proxy][:ip], @params[:proxy][:port])
|
42
107
|
else
|
43
|
-
|
108
|
+
Net::HTTP::Proxy(@params[:proxy][:ip], @params[:proxy][:port])
|
44
109
|
end
|
45
110
|
else
|
46
|
-
|
111
|
+
Net::HTTP
|
47
112
|
end
|
113
|
+
end
|
48
114
|
|
49
|
-
# Set HTTP request type
|
50
|
-
if params[:post] or params.key?(:post)
|
51
|
-
request = Net::HTTP::Post.new(uri.request_uri)
|
52
|
-
request.set_form_data(params[:post])
|
53
|
-
else
|
54
|
-
request = Net::HTTP::Get.new(uri.request_uri)
|
55
|
-
end
|
56
115
|
|
57
|
-
|
58
|
-
|
59
|
-
|
116
|
+
# Set Basic Auth
|
117
|
+
#
|
118
|
+
# @param request Request object
|
119
|
+
# @param params Request params
|
120
|
+
#
|
121
|
+
def self.set_basic_auth
|
122
|
+
if @params[:auth]
|
123
|
+
@request.basic_auth @params[:auth][:username], @params[:auth][:password]
|
60
124
|
end
|
125
|
+
end
|
61
126
|
|
62
|
-
# Set headers
|
63
|
-
params[:headers].each { |k, v| request.add_field(String(k), v) } if params[:headers]
|
64
127
|
|
65
|
-
|
66
|
-
|
128
|
+
# Headers
|
129
|
+
#
|
130
|
+
# @param request Request object
|
131
|
+
# @param params Request params
|
132
|
+
#
|
133
|
+
def self.set_headers
|
134
|
+
@params[:headers].each { |k, v| @request.add_field(String(k), v) } if @params[:headers]
|
67
135
|
end
|
68
136
|
|
69
|
-
private
|
70
137
|
|
71
|
-
#
|
138
|
+
# Make request to remote resource
|
72
139
|
#
|
73
|
-
# @param
|
140
|
+
# @param uri URI object
|
141
|
+
# @param http
|
74
142
|
#
|
75
|
-
def self.
|
76
|
-
|
77
|
-
params
|
143
|
+
def self.get_request
|
144
|
+
@http.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https') { |http| http.request(@request) }
|
78
145
|
end
|
146
|
+
|
79
147
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tiny_grabber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aleksandr Chernyshev
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: socksify
|