tiny_grabber 0.0.7 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -2
- data/lib/tiny_grabber/version.rb +1 -1
- data/lib/tiny_grabber.rb +101 -33
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9125f8132b04b1a08e64a47abf7ccb5085688b55
|
4
|
+
data.tar.gz: 050aad4742cdeeba67cfc64dad8c6d9c0a02ec97
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1ff3deb449bc1aedf86b2886156043decf28cf327ec25e30cd4725f07f5dc82faa01c803aa161bf573e3799f747a15673919fb470db75546810a932ee513093
|
7
|
+
data.tar.gz: c977e50867454e3af09e00e6b9f69f5b8efc7b22aaa0127b3c54c8aee206692359322a0c4418b81ab23bbc4a54a562f197f9fc98088225d97a70d1d1699d83f2
|
data/README.md
CHANGED
@@ -44,8 +44,11 @@ auth = { username: '', password: '' }
|
|
44
44
|
# Request HTTP type by default is GET. You can send POST request with setting post params. Also you cat send empty POST request.
|
45
45
|
post = { some_data: '' }
|
46
46
|
|
47
|
-
#
|
48
|
-
response = TinyGrabber.get url, headers: headers, proxy: proxy, auth: auth
|
47
|
+
# HTTP GET request
|
48
|
+
response = TinyGrabber.get url, headers: headers, proxy: proxy, auth: auth
|
49
|
+
|
50
|
+
# HTTP GET request
|
51
|
+
response = TinyGrabber.post url, post, headers: headers, proxy: proxy, auth: auth
|
49
52
|
|
50
53
|
# HTTP answer code
|
51
54
|
p response.code
|
@@ -62,6 +65,8 @@ p response.cookies
|
|
62
65
|
|
63
66
|
## Changelog
|
64
67
|
|
68
|
+
* *v 0.1.0*
|
69
|
+
* Add TinyGrabber.post method for HTTP POST request
|
65
70
|
* *v 0.0.7*
|
66
71
|
* Add POST request
|
67
72
|
* Add Basic Authentication
|
data/lib/tiny_grabber/version.rb
CHANGED
data/lib/tiny_grabber.rb
CHANGED
@@ -8,10 +8,15 @@ require 'tiny_grabber/http'
|
|
8
8
|
# Main class for TinyGrabber
|
9
9
|
class TinyGrabber
|
10
10
|
|
11
|
+
@uri
|
12
|
+
@params
|
13
|
+
@http
|
14
|
+
@request
|
15
|
+
|
11
16
|
##
|
12
17
|
# Get Net::HTTP object for content from remote single page
|
13
|
-
# [url (string)] Link
|
14
|
-
# [params (hash)]
|
18
|
+
# [url (string)] Link to resource
|
19
|
+
# [params (hash)] Request params
|
15
20
|
# * [proxy (hash)] Configuration of remote proxy server
|
16
21
|
# * *ip* address of remote proxy server
|
17
22
|
# * *port* of remote proxy server
|
@@ -22,58 +27,121 @@ class TinyGrabber
|
|
22
27
|
# * [auth (hash)] Basic Authentication
|
23
28
|
# * *username* Authenticate username
|
24
29
|
# * *password* Authenticate password
|
25
|
-
# * [post (hash)] POST data
|
26
30
|
#
|
27
31
|
# = Example
|
28
32
|
# TinyGrabber.get url, headers: { 'Content-Type' => 'application/json' }, proxy: { ip: 'xx.xx.xx.xx', port: xx, proxy_type: :socks }
|
29
33
|
#
|
30
|
-
# @param url Link
|
31
|
-
# @param params
|
34
|
+
# @param url Link to resource
|
35
|
+
# @param params Request params
|
32
36
|
#
|
33
37
|
def self.get url, params = {}
|
34
|
-
uri =
|
38
|
+
@uri = set_url url
|
39
|
+
@params = convert_params_to_sym params
|
40
|
+
@http = set_http_connect
|
41
|
+
@request = Net::HTTP::Get.new(@uri.request_uri)
|
42
|
+
set_basic_auth
|
43
|
+
set_headers
|
44
|
+
get_request
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
##
|
49
|
+
# Get Net::HTTP object for content from remote single page
|
50
|
+
# [url (string)] Link to resource
|
51
|
+
# [data (hash)] Post data
|
52
|
+
# [params (hash)] Request params
|
53
|
+
# * [proxy (hash)] Configuration of remote proxy server
|
54
|
+
# * *ip* address of remote proxy server
|
55
|
+
# * *port* of remote proxy server
|
56
|
+
# * *proxy_type* of remote proxy server
|
57
|
+
# * *http* for http(s) proxy servers (by default)
|
58
|
+
# * *socks* for socks4/5 proxy servers
|
59
|
+
# * [headers (hash)] Headers
|
60
|
+
# * [auth (hash)] Basic Authentication
|
61
|
+
# * *username* Authenticate username
|
62
|
+
# * *password* Authenticate password
|
63
|
+
#
|
64
|
+
# = Example
|
65
|
+
# TinyGrabber.post url, data, headers: { 'Content-Type' => 'application/json' }, proxy: { ip: 'xx.xx.xx.xx', port: xx, proxy_type: :socks }
|
66
|
+
#
|
67
|
+
# @param url Link to resource
|
68
|
+
# @param params Request params
|
69
|
+
#
|
70
|
+
def self.post url, data, params = {}
|
71
|
+
@uri = set_url url
|
72
|
+
data = convert_params_to_sym data
|
73
|
+
@params = convert_params_to_sym params
|
74
|
+
@http = set_http_connect
|
75
|
+
@request = Net::HTTP::Post.new(@uri.request_uri)
|
76
|
+
@request.set_form_data(data)
|
77
|
+
set_basic_auth
|
78
|
+
set_headers
|
79
|
+
get_request
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
35
83
|
|
36
|
-
|
84
|
+
# Convert all params key to symbols
|
85
|
+
#
|
86
|
+
# @param params Request params
|
87
|
+
#
|
88
|
+
def self.convert_params_to_sym params
|
89
|
+
params.is_a?(Hash) ? params.inject({}) { |h, (k, v)| h[k.to_sym] = convert_params_to_sym v; h } : params
|
90
|
+
end
|
91
|
+
|
92
|
+
# Convert URL to URI
|
93
|
+
#
|
94
|
+
# @param url Link to resource
|
95
|
+
#
|
96
|
+
def self.set_url url
|
97
|
+
URI(URI.escape(url))
|
98
|
+
end
|
37
99
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
100
|
+
|
101
|
+
# Use proxy for request
|
102
|
+
#
|
103
|
+
def self.set_http_connect
|
104
|
+
if @params[:proxy]
|
105
|
+
if ['socks', :socks].include?(@params[:proxy][:proxy_type])
|
106
|
+
Net::HTTP.SOCKSProxy(@params[:proxy][:ip], @params[:proxy][:port])
|
42
107
|
else
|
43
|
-
|
108
|
+
Net::HTTP::Proxy(@params[:proxy][:ip], @params[:proxy][:port])
|
44
109
|
end
|
45
110
|
else
|
46
|
-
|
111
|
+
Net::HTTP
|
47
112
|
end
|
113
|
+
end
|
48
114
|
|
49
|
-
# Set HTTP request type
|
50
|
-
if params[:post] or params.key?(:post)
|
51
|
-
request = Net::HTTP::Post.new(uri.request_uri)
|
52
|
-
request.set_form_data(params[:post])
|
53
|
-
else
|
54
|
-
request = Net::HTTP::Get.new(uri.request_uri)
|
55
|
-
end
|
56
115
|
|
57
|
-
|
58
|
-
|
59
|
-
|
116
|
+
# Set Basic Auth
|
117
|
+
#
|
118
|
+
# @param request Request object
|
119
|
+
# @param params Request params
|
120
|
+
#
|
121
|
+
def self.set_basic_auth
|
122
|
+
if @params[:auth]
|
123
|
+
@request.basic_auth @params[:auth][:username], @params[:auth][:password]
|
60
124
|
end
|
125
|
+
end
|
61
126
|
|
62
|
-
# Set headers
|
63
|
-
params[:headers].each { |k, v| request.add_field(String(k), v) } if params[:headers]
|
64
127
|
|
65
|
-
|
66
|
-
|
128
|
+
# Headers
|
129
|
+
#
|
130
|
+
# @param request Request object
|
131
|
+
# @param params Request params
|
132
|
+
#
|
133
|
+
def self.set_headers
|
134
|
+
@params[:headers].each { |k, v| @request.add_field(String(k), v) } if @params[:headers]
|
67
135
|
end
|
68
136
|
|
69
|
-
private
|
70
137
|
|
71
|
-
#
|
138
|
+
# Make request to remote resource
|
72
139
|
#
|
73
|
-
# @param
|
140
|
+
# @param uri URI object
|
141
|
+
# @param http
|
74
142
|
#
|
75
|
-
def self.
|
76
|
-
|
77
|
-
params
|
143
|
+
def self.get_request
|
144
|
+
@http.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https') { |http| http.request(@request) }
|
78
145
|
end
|
146
|
+
|
79
147
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tiny_grabber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aleksandr Chernyshev
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: socksify
|