rhack 1.0.0.rc4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +5 -2
- data/lib/rhack/curl/easy.rb +12 -3
- data/lib/rhack/scout.rb +6 -13
- data/lib/rhack/services.rb +7 -1
- data/lib/rhack/services/base.rb +6 -15
- data/lib/rhack/services/compatibility.rb +4 -0
- data/lib/rhack/version.rb +1 -1
- metadata +6 -5
data/README.md
CHANGED
@@ -11,7 +11,7 @@ RHACK is Ruby Http ACcess Kit: curl-based web-client framework created for devel
|
|
11
11
|
* ::Scout (Curl::Easy wrapper with transparent cookies and extendable anonimization processing, detailed request/response info, callbacks and retry configuration)
|
12
12
|
* ::Frame (Scout array wrapper with smart request interpretor, load balancing and extendable response processor)
|
13
13
|
* Support of javascript processing on loaded html pages is included (johnson gem)
|
14
|
-
* Web-service abstraction implementing some examples of how to use this library
|
14
|
+
* Web-service-client abstraction implementing some examples of how to use this library
|
15
15
|
|
16
16
|
---
|
17
17
|
|
@@ -19,7 +19,7 @@ It's still randomly documented since it's just my working tool.
|
|
19
19
|
|
20
20
|
#### Main goals for 1.x
|
21
21
|
|
22
|
-
*
|
22
|
+
* Client subclass for OAuth2 with a full set of abstract authorizaztion and API methods. Main idea is a per-user key-value token storage with a handling of tokens expiration.
|
23
23
|
* Redis-based cache storage for scrapers data.
|
24
24
|
* More agile response postprocessing configuration. Instead of using :json, :hash etc as a flag, define some "before filters" in the Page and chain them.
|
25
25
|
* Route :xhr option to the Scout; add some transparent control on user-agents: desktop, mobile, randomly predefined...
|
@@ -46,6 +46,9 @@ It's still randomly documented since it's just my working tool.
|
|
46
46
|
|
47
47
|
* ::ScoutSquad
|
48
48
|
* Automatically Curl.execute on #next and #rand if Carier Thread is exited without an exception
|
49
|
+
|
50
|
+
* ::Service
|
51
|
+
* Is renamed to Client what is more sensible. RHACK::Service is still usable as alias
|
49
52
|
|
50
53
|
* Structural changes
|
51
54
|
* Updated and documented rhack.yml.template that now lies in <gemdir>/config
|
data/lib/rhack/curl/easy.rb
CHANGED
@@ -5,15 +5,24 @@ module Curl
|
|
5
5
|
__init__
|
6
6
|
attr_accessor :base
|
7
7
|
|
8
|
+
def outdate!
|
9
|
+
@outdated = true
|
10
|
+
end
|
11
|
+
|
8
12
|
def res
|
9
|
-
|
13
|
+
if @res && !@outdated
|
14
|
+
@res
|
15
|
+
else
|
16
|
+
@outdated = false
|
17
|
+
@res = Response(self)
|
18
|
+
end
|
10
19
|
end
|
11
|
-
alias response res
|
20
|
+
alias :response :res
|
12
21
|
|
13
22
|
def req
|
14
23
|
res.req
|
15
24
|
end
|
16
|
-
alias request req
|
25
|
+
alias :request :req
|
17
26
|
|
18
27
|
def host
|
19
28
|
url.parse(:uri).root
|
data/lib/rhack/scout.rb
CHANGED
@@ -92,20 +92,13 @@ module RHACK
|
|
92
92
|
end
|
93
93
|
alias :inspect :to_s
|
94
94
|
|
95
|
-
def update_res
|
96
|
-
@outdated = false
|
97
|
-
@res = @http.res
|
98
|
-
@headers = nil
|
99
|
-
@res
|
100
|
-
end
|
101
|
-
|
102
95
|
def res
|
103
|
-
|
104
|
-
@res
|
105
|
-
else update_res end
|
96
|
+
@http.res
|
106
97
|
end
|
107
98
|
|
108
|
-
def req
|
99
|
+
def req
|
100
|
+
res.req
|
101
|
+
end
|
109
102
|
|
110
103
|
def dump
|
111
104
|
str = "IP: #{@proxystr}\nRequest: "
|
@@ -222,7 +215,7 @@ module RHACK
|
|
222
215
|
|
223
216
|
@http.on_complete {|c|
|
224
217
|
@error = nil
|
225
|
-
|
218
|
+
c.outdate!
|
226
219
|
ProcCookies c.res if @cookieProc
|
227
220
|
# We cannot just cancel on_complete in on_redirect block
|
228
221
|
# because loadGet will immediately reset on_complete back
|
@@ -239,7 +232,7 @@ module RHACK
|
|
239
232
|
L.log << "Got Curl::Err::CurlOK, response was: #{c.res}"
|
240
233
|
else
|
241
234
|
@http.on_complete &Proc::NULL
|
242
|
-
|
235
|
+
c.outdate!
|
243
236
|
if retry? e
|
244
237
|
L.debug "#{e[0]} -> reloading scout"
|
245
238
|
#load uri, headers, not_redir, relvl, &callback
|
data/lib/rhack/services.rb
CHANGED
data/lib/rhack/services/base.rb
CHANGED
@@ -1,19 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
def #{name}(key=nil)
|
7
|
-
key ? self.class::#{name}[key] : self.class::#{name}
|
8
|
-
end}
|
9
|
-
end unless defined? alias_constant
|
10
|
-
end
|
11
|
-
|
12
|
-
# Вызовы сервисов всегда ждут и возвращают обработанный ответ, если вызвваны без блока.
|
13
|
-
# В противном случае используется событийная модель и обработанный ответ передаётся в блок.
|
3
|
+
# TODO 1.0+: опция для клиента, чтобы это описание имело смысл, т.к. сейчас это ложь:
|
4
|
+
# Вызовам клиентов всегда следует ждут и возвращают обработанный ответ, если вызвваны без блока.
|
5
|
+
# В противном случае используется событийная модель и обработанный ответ передаётся в блок.
|
14
6
|
module RHACK
|
15
7
|
|
16
|
-
class
|
8
|
+
class Client
|
17
9
|
attr_accessor :f
|
18
10
|
alias_constant :URI
|
19
11
|
|
@@ -54,6 +46,5 @@ module RHACK
|
|
54
46
|
|
55
47
|
end
|
56
48
|
|
57
|
-
class
|
58
|
-
|
59
|
-
end
|
49
|
+
class ClientError < Exception; end
|
50
|
+
end
|
data/lib/rhack/version.rb
CHANGED
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rhack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0
|
5
|
-
prerelease:
|
4
|
+
version: 1.0.0
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Sergey Baev
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-05-
|
12
|
+
date: 2013-05-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -180,6 +180,7 @@ files:
|
|
180
180
|
- lib/rhack/scout_squad.rb
|
181
181
|
- lib/rhack/services.rb
|
182
182
|
- lib/rhack/services/base.rb
|
183
|
+
- lib/rhack/services/compatibility.rb
|
183
184
|
- lib/rhack/services/examples.rb
|
184
185
|
- lib/rhack/version.rb
|
185
186
|
- lib/rhack_in.rb
|
@@ -203,9 +204,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
203
204
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
204
205
|
none: false
|
205
206
|
requirements:
|
206
|
-
- - ! '
|
207
|
+
- - ! '>='
|
207
208
|
- !ruby/object:Gem::Version
|
208
|
-
version:
|
209
|
+
version: '0'
|
209
210
|
requirements: []
|
210
211
|
rubyforge_project:
|
211
212
|
rubygems_version: 1.8.24
|