rhack 1.0.0.rc4 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +5 -2
- data/lib/rhack/curl/easy.rb +12 -3
- data/lib/rhack/scout.rb +6 -13
- data/lib/rhack/services.rb +7 -1
- data/lib/rhack/services/base.rb +6 -15
- data/lib/rhack/services/compatibility.rb +4 -0
- data/lib/rhack/version.rb +1 -1
- metadata +6 -5
data/README.md
CHANGED
@@ -11,7 +11,7 @@ RHACK is Ruby Http ACcess Kit: curl-based web-client framework created for devel
|
|
11
11
|
* ::Scout (Curl::Easy wrapper with transparent cookies and extendable anonimization processing, detailed request/response info, callbacks and retry configuration)
|
12
12
|
* ::Frame (Scout array wrapper with smart request interpretor, load balancing and extendable response processor)
|
13
13
|
* Support of javascript processing on loaded html pages is included (johnson gem)
|
14
|
-
* Web-service abstraction implementing some examples of how to use this library
|
14
|
+
* Web-service-client abstraction implementing some examples of how to use this library
|
15
15
|
|
16
16
|
---
|
17
17
|
|
@@ -19,7 +19,7 @@ It's still randomly documented since it's just my working tool.
|
|
19
19
|
|
20
20
|
#### Main goals for 1.x
|
21
21
|
|
22
|
-
*
|
22
|
+
* Client subclass for OAuth2 with a full set of abstract authorizaztion and API methods. Main idea is a per-user key-value token storage with a handling of tokens expiration.
|
23
23
|
* Redis-based cache storage for scrapers data.
|
24
24
|
* More agile response postprocessing configuration. Instead of using :json, :hash etc as a flag, define some "before filters" in the Page and chain them.
|
25
25
|
* Route :xhr option to the Scout; add some transparent control on user-agents: desktop, mobile, randomly predefined...
|
@@ -46,6 +46,9 @@ It's still randomly documented since it's just my working tool.
|
|
46
46
|
|
47
47
|
* ::ScoutSquad
|
48
48
|
* Automatically Curl.execute on #next and #rand if Carier Thread is exited without an exception
|
49
|
+
|
50
|
+
* ::Service
|
51
|
+
* Is renamed to Client what is more sensible. RHACK::Service is still usable as alias
|
49
52
|
|
50
53
|
* Structural changes
|
51
54
|
* Updated and documented rhack.yml.template that now lies in <gemdir>/config
|
data/lib/rhack/curl/easy.rb
CHANGED
@@ -5,15 +5,24 @@ module Curl
|
|
5
5
|
__init__
|
6
6
|
attr_accessor :base
|
7
7
|
|
8
|
+
def outdate!
|
9
|
+
@outdated = true
|
10
|
+
end
|
11
|
+
|
8
12
|
def res
|
9
|
-
|
13
|
+
if @res && !@outdated
|
14
|
+
@res
|
15
|
+
else
|
16
|
+
@outdated = false
|
17
|
+
@res = Response(self)
|
18
|
+
end
|
10
19
|
end
|
11
|
-
alias response res
|
20
|
+
alias :response :res
|
12
21
|
|
13
22
|
def req
|
14
23
|
res.req
|
15
24
|
end
|
16
|
-
alias request req
|
25
|
+
alias :request :req
|
17
26
|
|
18
27
|
def host
|
19
28
|
url.parse(:uri).root
|
data/lib/rhack/scout.rb
CHANGED
@@ -92,20 +92,13 @@ module RHACK
|
|
92
92
|
end
|
93
93
|
alias :inspect :to_s
|
94
94
|
|
95
|
-
def update_res
|
96
|
-
@outdated = false
|
97
|
-
@res = @http.res
|
98
|
-
@headers = nil
|
99
|
-
@res
|
100
|
-
end
|
101
|
-
|
102
95
|
def res
|
103
|
-
|
104
|
-
@res
|
105
|
-
else update_res end
|
96
|
+
@http.res
|
106
97
|
end
|
107
98
|
|
108
|
-
def req
|
99
|
+
def req
|
100
|
+
res.req
|
101
|
+
end
|
109
102
|
|
110
103
|
def dump
|
111
104
|
str = "IP: #{@proxystr}\nRequest: "
|
@@ -222,7 +215,7 @@ module RHACK
|
|
222
215
|
|
223
216
|
@http.on_complete {|c|
|
224
217
|
@error = nil
|
225
|
-
|
218
|
+
c.outdate!
|
226
219
|
ProcCookies c.res if @cookieProc
|
227
220
|
# We cannot just cancel on_complete in on_redirect block
|
228
221
|
# because loadGet will immediately reset on_complete back
|
@@ -239,7 +232,7 @@ module RHACK
|
|
239
232
|
L.log << "Got Curl::Err::CurlOK, response was: #{c.res}"
|
240
233
|
else
|
241
234
|
@http.on_complete &Proc::NULL
|
242
|
-
|
235
|
+
c.outdate!
|
243
236
|
if retry? e
|
244
237
|
L.debug "#{e[0]} -> reloading scout"
|
245
238
|
#load uri, headers, not_redir, relvl, &callback
|
data/lib/rhack/services.rb
CHANGED
data/lib/rhack/services/base.rb
CHANGED
@@ -1,19 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
def #{name}(key=nil)
|
7
|
-
key ? self.class::#{name}[key] : self.class::#{name}
|
8
|
-
end}
|
9
|
-
end unless defined? alias_constant
|
10
|
-
end
|
11
|
-
|
12
|
-
# Вызовы сервисов всегда ждут и возвращают обработанный ответ, если вызвваны без блока.
|
13
|
-
# В противном случае используется событийная модель и обработанный ответ передаётся в блок.
|
3
|
+
# TODO 1.0+: опция для клиента, чтобы это описание имело смысл, т.к. сейчас это ложь:
|
4
|
+
# Вызовам клиентов всегда следует ждут и возвращают обработанный ответ, если вызвваны без блока.
|
5
|
+
# В противном случае используется событийная модель и обработанный ответ передаётся в блок.
|
14
6
|
module RHACK
|
15
7
|
|
16
|
-
class
|
8
|
+
class Client
|
17
9
|
attr_accessor :f
|
18
10
|
alias_constant :URI
|
19
11
|
|
@@ -54,6 +46,5 @@ module RHACK
|
|
54
46
|
|
55
47
|
end
|
56
48
|
|
57
|
-
class
|
58
|
-
|
59
|
-
end
|
49
|
+
class ClientError < Exception; end
|
50
|
+
end
|
data/lib/rhack/version.rb
CHANGED
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rhack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0
|
5
|
-
prerelease:
|
4
|
+
version: 1.0.0
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Sergey Baev
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-05-
|
12
|
+
date: 2013-05-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -180,6 +180,7 @@ files:
|
|
180
180
|
- lib/rhack/scout_squad.rb
|
181
181
|
- lib/rhack/services.rb
|
182
182
|
- lib/rhack/services/base.rb
|
183
|
+
- lib/rhack/services/compatibility.rb
|
183
184
|
- lib/rhack/services/examples.rb
|
184
185
|
- lib/rhack/version.rb
|
185
186
|
- lib/rhack_in.rb
|
@@ -203,9 +204,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
203
204
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
204
205
|
none: false
|
205
206
|
requirements:
|
206
|
-
- - ! '
|
207
|
+
- - ! '>='
|
207
208
|
- !ruby/object:Gem::Version
|
208
|
-
version:
|
209
|
+
version: '0'
|
209
210
|
requirements: []
|
210
211
|
rubyforge_project:
|
211
212
|
rubygems_version: 1.8.24
|