typhoeus 1.0.1 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +12 -7
- data/CHANGELOG.md +34 -2
- data/CONTRIBUTING.md +4 -0
- data/Gemfile +14 -3
- data/LICENSE +1 -1
- data/README.md +67 -42
- data/lib/typhoeus/adapters/faraday.rb +30 -9
- data/lib/typhoeus/cache/dalli.rb +28 -0
- data/lib/typhoeus/cache/rails.rb +28 -0
- data/lib/typhoeus/cache/redis.rb +35 -0
- data/lib/typhoeus/config.rb +8 -1
- data/lib/typhoeus/easy_factory.rb +8 -3
- data/lib/typhoeus/hydra/cacheable.rb +1 -1
- data/lib/typhoeus/pool.rb +2 -0
- data/lib/typhoeus/request/actions.rb +7 -7
- data/lib/typhoeus/request/cacheable.rb +14 -3
- data/lib/typhoeus/request/callbacks.rb +21 -3
- data/lib/typhoeus/request/marshal.rb +2 -2
- data/lib/typhoeus/request/streamable.rb +1 -1
- data/lib/typhoeus/request.rb +2 -0
- data/lib/typhoeus/response/header.rb +13 -5
- data/lib/typhoeus/response/informations.rb +7 -3
- data/lib/typhoeus/response/status.rb +22 -2
- data/lib/typhoeus/response.rb +1 -1
- data/lib/typhoeus/version.rb +1 -1
- data/lib/typhoeus.rb +19 -3
- data/spec/support/server.rb +8 -0
- data/spec/typhoeus/adapters/faraday_spec.rb +237 -191
- data/spec/typhoeus/cache/dalli_spec.rb +41 -0
- data/spec/typhoeus/cache/redis_spec.rb +41 -0
- data/spec/typhoeus/config_spec.rb +1 -1
- data/spec/typhoeus/easy_factory_spec.rb +6 -0
- data/spec/typhoeus/hydra/before_spec.rb +9 -8
- data/spec/typhoeus/hydra/cacheable_spec.rb +31 -1
- data/spec/typhoeus/hydra/runnable_spec.rb +4 -3
- data/spec/typhoeus/pool_spec.rb +43 -2
- data/spec/typhoeus/request/before_spec.rb +9 -8
- data/spec/typhoeus/request/cacheable_spec.rb +24 -0
- data/spec/typhoeus/request/callbacks_spec.rb +2 -2
- data/spec/typhoeus/request/marshal_spec.rb +1 -1
- data/spec/typhoeus/request_spec.rb +21 -3
- data/spec/typhoeus/response/header_spec.rb +51 -1
- data/spec/typhoeus/response/informations_spec.rb +12 -1
- data/spec/typhoeus/response/status_spec.rb +54 -0
- data/typhoeus.gemspec +1 -1
- metadata +12 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 79bd3451f6ec67ea973d9dc62dc4645c21e2eba1
|
4
|
+
data.tar.gz: 762e3c44e97fc0f2241d5cbf9d2a9ff2caa8a53a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e0a7c0b640b05b4527a4a9a7b3ee7db526fc25e31e001b7287f2423fecebbf6d673adb5c17b7cda32c1206872a3a27d82825b9ee74974607f691e9ee573fb39a
|
7
|
+
data.tar.gz: 444398bc7499402359de7a5534c265bf91c39346903c1b82c4767b1b7f9eb9c72741a979986b905e388301055e6816baa6ef999f9f35bed06a07a4bec37d7b99
|
data/.travis.yml
CHANGED
@@ -1,21 +1,26 @@
|
|
1
|
+
language: ruby
|
1
2
|
script: "bundle exec rake"
|
2
|
-
sudo: false
|
3
3
|
rvm:
|
4
|
-
- 1.8.7
|
5
|
-
- 1.9.2
|
6
4
|
- 1.9.3
|
7
5
|
- 2.0.0
|
6
|
+
- 2.1.10
|
7
|
+
- 2.2.10
|
8
|
+
- 2.3.8
|
9
|
+
- 2.4.7
|
10
|
+
- 2.5.6
|
11
|
+
- 2.6.4
|
8
12
|
- ruby-head
|
9
13
|
- jruby-head
|
10
14
|
- jruby-18mode
|
11
15
|
- jruby-19mode
|
12
|
-
- rbx-18mode
|
13
|
-
- rbx-19mode
|
14
16
|
matrix:
|
15
17
|
fast_finish: true
|
16
18
|
allow_failures:
|
17
19
|
- rvm: ruby-head
|
18
20
|
- rvm: jruby-head
|
19
|
-
- rvm: rbx-18mode
|
20
|
-
- rvm: rbx-19mode
|
21
21
|
- rvm: ree
|
22
|
+
include:
|
23
|
+
- rvm: 1.8.7
|
24
|
+
dist: precise
|
25
|
+
- rvm: 1.9.2
|
26
|
+
dist: trusty
|
data/CHANGELOG.md
CHANGED
@@ -2,7 +2,39 @@
|
|
2
2
|
|
3
3
|
## Master
|
4
4
|
|
5
|
-
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.0
|
5
|
+
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.4.0...master)
|
6
|
+
|
7
|
+
## 1.4.0
|
8
|
+
|
9
|
+
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.1.2...v1.4.0)
|
10
|
+
|
11
|
+
#### 1 feature
|
12
|
+
- Faraday adapter exceptions namespace compatibility with Faraday v1 ([@iMacTia](https://github.com/iMacTia) in [#616](https://github.com/typhoeus/typhoeus/pull/616))
|
13
|
+
|
14
|
+
#### 3 Others
|
15
|
+
- Yard warning fixes ([@olleolleolle](https://github.com/olleolleolle) in [#622](https://github.com/typhoeus/typhoeus/pull/622))
|
16
|
+
- Add more Ruby versions in CI matrix ([@olleolleolle](https://github.com/olleolleolle) in [#623](https://github.com/typhoeus/typhoeus/pull/623))
|
17
|
+
- Use of argument passed in function instead of `attr_reader` ([@v-kolesnikov](https://github.com/v-kolesnikov) in [#625](https://github.com/typhoeus/typhoeus/pull/625))
|
18
|
+
|
19
|
+
## 1.1.2
|
20
|
+
|
21
|
+
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.1.1...v1.1.2)
|
22
|
+
|
23
|
+
## 1.1.1
|
24
|
+
|
25
|
+
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.1.0...v1.1.1)
|
26
|
+
|
27
|
+
## 1.1.0
|
28
|
+
|
29
|
+
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.0.2...v1.1.0)
|
30
|
+
|
31
|
+
* Unless specified `Expect` header is set to be empty to avoid `100 continue`
|
32
|
+
to be set when using `PUT`
|
33
|
+
* Add global config option `Typhoeus::Config.proxy`
|
34
|
+
|
35
|
+
## 1.0.2
|
36
|
+
|
37
|
+
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.0.1...v1.0.2)
|
6
38
|
|
7
39
|
## 1.0.1
|
8
40
|
|
@@ -12,7 +44,7 @@
|
|
12
44
|
|
13
45
|
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.8.0...v1.0.0)
|
14
46
|
|
15
|
-
##
|
47
|
+
## 0.8.0
|
16
48
|
|
17
49
|
[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.7.3...v0.8.0)
|
18
50
|
|
data/CONTRIBUTING.md
CHANGED
@@ -14,3 +14,7 @@ a test!
|
|
14
14
|
5. Push to your fork and submit a pull request.
|
15
15
|
|
16
16
|
And in case we didn't emphasize it enough: we love tests!
|
17
|
+
|
18
|
+
## Issue triage [![Open Source Helpers](https://www.codetriage.com/typhoeus/typhoeus/badges/users.svg)](https://www.codetriage.com/typhoeus/typhoeus)
|
19
|
+
|
20
|
+
You can contribute by triaging issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to typhoeus on CodeTriage](https://www.codetriage.com/typhoeus/typhoeus).
|
data/Gemfile
CHANGED
@@ -1,14 +1,25 @@
|
|
1
1
|
source "https://rubygems.org"
|
2
2
|
gemspec
|
3
3
|
|
4
|
-
|
4
|
+
if Gem.ruby_version < Gem::Version.new("2.0.0")
|
5
|
+
gem "rake", "< 11"
|
6
|
+
gem "json", "< 2"
|
7
|
+
else
|
8
|
+
gem "json"
|
9
|
+
gem "rake"
|
10
|
+
end
|
5
11
|
|
6
12
|
group :development, :test do
|
7
13
|
gem "rspec", "~> 3.0"
|
8
14
|
|
9
15
|
gem "sinatra", "~> 1.3"
|
10
|
-
|
11
|
-
|
16
|
+
|
17
|
+
if Gem.ruby_version >= Gem::Version.new("1.9.0")
|
18
|
+
gem "faraday", ">= 0.9"
|
19
|
+
gem "dalli", "~> 2.0"
|
20
|
+
end
|
21
|
+
|
22
|
+
gem "redis", "~> 3.0"
|
12
23
|
|
13
24
|
if RUBY_PLATFORM == "java"
|
14
25
|
gem "spoon"
|
data/LICENSE
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Copyright (c) 2009-2010 Paul Dix
|
2
2
|
Copyright (c) 2011 David Balatero
|
3
|
-
Copyright (c) 2012-
|
3
|
+
Copyright (c) 2012-2016 Hans Hasselberg
|
4
4
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining
|
6
6
|
a copy of this software and associated documentation files (the
|
data/README.md
CHANGED
@@ -1,17 +1,7 @@
|
|
1
|
-
# Typhoeus [![Build Status](https://img.shields.io/travis/typhoeus/typhoeus/master.svg)](https://travis-ci.org/typhoeus/typhoeus) [![Code Climate](https://img.shields.io/codeclimate/
|
1
|
+
# Typhoeus [![Build Status](https://img.shields.io/travis/typhoeus/typhoeus/master.svg)](https://travis-ci.org/typhoeus/typhoeus) [![Code Climate](https://img.shields.io/codeclimate/maintainability/typhoeus/typhoeus.svg)](https://codeclimate.com/github/typhoeus/typhoeus) [![Gem Version](https://img.shields.io/gem/v/typhoeus.svg)](https://rubygems.org/gems/typhoeus)
|
2
2
|
|
3
3
|
Like a modern code version of the mythical beast with 100 serpent heads, Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling logic.
|
4
4
|
|
5
|
-
# Typhoeus needs your help!
|
6
|
-
|
7
|
-
I don't have enough time, but I think this is a nice project! If you or your company is using Typhoeus you should help keeping it alive! Pick any of:
|
8
|
-
|
9
|
-
* add docs
|
10
|
-
* respond to issues
|
11
|
-
* add features
|
12
|
-
|
13
|
-
Or send me an email! I would be more than happy to help getting you up to speed!
|
14
|
-
|
15
5
|
## Example
|
16
6
|
|
17
7
|
A single request:
|
@@ -29,13 +19,16 @@ hydra.run
|
|
29
19
|
```
|
30
20
|
|
31
21
|
## Installation
|
32
|
-
|
22
|
+
Add the following line to your Gemfile:
|
33
23
|
```
|
34
|
-
gem
|
24
|
+
gem "typhoeus"
|
35
25
|
```
|
26
|
+
Then run `bundle install`
|
27
|
+
|
28
|
+
Or install it yourself as:
|
36
29
|
|
37
30
|
```
|
38
|
-
gem
|
31
|
+
gem install typhoeus
|
39
32
|
```
|
40
33
|
|
41
34
|
## Project Tracking
|
@@ -201,6 +194,19 @@ end
|
|
201
194
|
request.run
|
202
195
|
```
|
203
196
|
|
197
|
+
If you need to interrupt the stream halfway,
|
198
|
+
you can return the `:abort` symbol from the `on_body` block, example:
|
199
|
+
|
200
|
+
```ruby
|
201
|
+
request.on_body do |chunk|
|
202
|
+
buffer << chunk
|
203
|
+
:abort if buffer.size > 1024 * 1024
|
204
|
+
end
|
205
|
+
```
|
206
|
+
|
207
|
+
This will properly stop the stream internally and avoid any memory leak which
|
208
|
+
may happen if you interrupt with something like a `return`, `throw` or `raise`.
|
209
|
+
|
204
210
|
### Making Parallel Requests
|
205
211
|
|
206
212
|
Generally, you should be running requests through hydra. Here is how that looks:
|
@@ -252,6 +258,33 @@ end
|
|
252
258
|
hydra.run
|
253
259
|
```
|
254
260
|
|
261
|
+
### Making Parallel Requests with Faraday + Typhoeus
|
262
|
+
|
263
|
+
```ruby
|
264
|
+
require 'faraday'
|
265
|
+
|
266
|
+
conn = Faraday.new(:url => 'http://httppage.com') do |builder|
|
267
|
+
builder.request :url_encoded
|
268
|
+
builder.response :logger
|
269
|
+
builder.adapter :typhoeus
|
270
|
+
end
|
271
|
+
|
272
|
+
conn.in_parallel do
|
273
|
+
response1 = conn.get('/first')
|
274
|
+
response2 = conn.get('/second')
|
275
|
+
|
276
|
+
# these will return nil here since the
|
277
|
+
# requests have not been completed
|
278
|
+
response1.body
|
279
|
+
response2.body
|
280
|
+
end
|
281
|
+
|
282
|
+
# after it has been completed the response information is fully available
|
283
|
+
# response1.status, etc
|
284
|
+
response1.body
|
285
|
+
response2.body
|
286
|
+
```
|
287
|
+
|
255
288
|
### Specifying Max Concurrency
|
256
289
|
|
257
290
|
Hydra will also handle how many requests you can make in parallel. Things will get flakey if you try to make too many requests at the same time. The built in limit is 200. When more requests than that are queued up, hydra will save them for later and start the requests as others are finished. You can raise or lower the concurrency limit through the Hydra constructor.
|
@@ -317,37 +350,32 @@ Typhoeus.get("www.example.com").cached?
|
|
317
350
|
For use with [Dalli](https://github.com/mperham/dalli):
|
318
351
|
|
319
352
|
```ruby
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
end
|
324
|
-
|
325
|
-
def get(request)
|
326
|
-
@client.get(request.cache_key)
|
327
|
-
end
|
353
|
+
dalli = Dalli::Client.new(...)
|
354
|
+
Typhoeus::Config.cache = Typhoeus::Cache::Dalli.new(dalli)
|
355
|
+
```
|
328
356
|
|
329
|
-
|
330
|
-
@client.set(request.cache_key, response)
|
331
|
-
end
|
332
|
-
end
|
357
|
+
For use with Rails:
|
333
358
|
|
334
|
-
|
359
|
+
```ruby
|
360
|
+
Typhoeus::Config.cache = Typhoeus::Cache::Rails.new
|
335
361
|
```
|
336
362
|
|
337
|
-
For use with
|
363
|
+
For use with [Redis](https://github.com/redis/redis-rb):
|
338
364
|
|
339
365
|
```ruby
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
end
|
366
|
+
redis = Redis.new(...)
|
367
|
+
Typhoeus::Config.cache = Typhoeus::Cache::Redis.new(redis)
|
368
|
+
```
|
344
369
|
|
345
|
-
|
346
|
-
|
347
|
-
end
|
348
|
-
end
|
370
|
+
All three of these adapters take an optional keyword argument `default_ttl`, which sets a default
|
371
|
+
TTL on cached responses (in seconds), for requests which do not have a cache TTL set.
|
349
372
|
|
350
|
-
|
373
|
+
You may also selectively choose not to cache by setting `cache` to `false` on a request or to use
|
374
|
+
a different adapter.
|
375
|
+
|
376
|
+
```ruby
|
377
|
+
cache = Cache.new
|
378
|
+
Typhoeus.get("www.example.com", cache: cache)
|
351
379
|
```
|
352
380
|
|
353
381
|
### Direct Stubbing
|
@@ -407,7 +435,7 @@ and [`connecttimeout`](http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLO
|
|
407
435
|
`timeout` is the time limit for the entire request in seconds.
|
408
436
|
`connecttimeout` is the time limit for just the connection phase, again in seconds.
|
409
437
|
|
410
|
-
There are two additional more fine grained
|
438
|
+
There are two additional more fine grained options `timeout_ms` and
|
411
439
|
`connecttimeout_ms`. These options offer millisecond precision but are not always available (for instance on linux if `nosignal` is not set to true).
|
412
440
|
|
413
441
|
When you pass a floating point `timeout` (or `connecttimeout`) Typhoeus will set `timeout_ms` for you if it has not been defined. The actual timeout values passed to curl will always be rounded up.
|
@@ -535,7 +563,7 @@ Copyright © 2009-2010 [Paul Dix](http://www.pauldix.net/)
|
|
535
563
|
|
536
564
|
Copyright © 2011-2012 [David Balatero](https://github.com/dbalatero/)
|
537
565
|
|
538
|
-
Copyright © 2012-
|
566
|
+
Copyright © 2012-2016 [Hans Hasselberg](http://github.com/i0rek/)
|
539
567
|
|
540
568
|
Permission is hereby granted, free of charge, to any person obtaining a
|
541
569
|
copy of this software and associated documentation files (the "Software"),
|
@@ -554,6 +582,3 @@ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
554
582
|
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
555
583
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
556
584
|
OTHER DEALINGS IN THE SOFTWARE.
|
557
|
-
|
558
|
-
|
559
|
-
[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/typhoeus/typhoeus/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
|
@@ -12,6 +12,9 @@ module Faraday # :nodoc:
|
|
12
12
|
#
|
13
13
|
# conn = Faraday.new(url: "www.example.com") do |faraday|
|
14
14
|
# faraday.adapter :typhoeus
|
15
|
+
#
|
16
|
+
# # You can include Typhoeus options to be used for every request
|
17
|
+
# # faraday.adapter :typhoeus, forbid_reuse: true, maxredirs: 1
|
15
18
|
# end
|
16
19
|
#
|
17
20
|
# response = conn.get("/")
|
@@ -32,6 +35,17 @@ module Faraday # :nodoc:
|
|
32
35
|
remove_method :configure_socket if method_defined? :configure_socket
|
33
36
|
remove_method :parallel? if method_defined? :parallel?
|
34
37
|
|
38
|
+
# Initialize the Typhoeus adapter
|
39
|
+
#
|
40
|
+
# @param [ App ] app Farday app
|
41
|
+
# @option [ Hash ] adapter_options Typhoeus options
|
42
|
+
#
|
43
|
+
# @return [ void ]
|
44
|
+
def initialize(app, adapter_options = {})
|
45
|
+
super(app)
|
46
|
+
@adapter_options = adapter_options
|
47
|
+
end
|
48
|
+
|
35
49
|
# Setup Hydra with provided options.
|
36
50
|
#
|
37
51
|
# @example Setup Hydra.
|
@@ -72,12 +86,7 @@ module Faraday # :nodoc:
|
|
72
86
|
def request(env)
|
73
87
|
read_body env
|
74
88
|
|
75
|
-
req =
|
76
|
-
env[:url].to_s,
|
77
|
-
:method => env[:method],
|
78
|
-
:body => env[:body],
|
79
|
-
:headers => env[:request_headers]
|
80
|
-
)
|
89
|
+
req = typhoeus_request(env)
|
81
90
|
|
82
91
|
configure_ssl req, env
|
83
92
|
configure_proxy req, env
|
@@ -88,12 +97,13 @@ module Faraday # :nodoc:
|
|
88
97
|
if resp.timed_out?
|
89
98
|
env[:typhoeus_timed_out] = true
|
90
99
|
unless parallel?(env)
|
91
|
-
raise Faraday::
|
100
|
+
raise Faraday::TimeoutError, "request timed out"
|
92
101
|
end
|
93
|
-
elsif resp.response_code == 0
|
102
|
+
elsif (resp.response_code == 0) || ((resp.return_code != :ok) && !resp.mock?)
|
94
103
|
env[:typhoeus_connection_failed] = true
|
104
|
+
env[:typhoeus_return_message] = resp.return_message
|
95
105
|
unless parallel?(env)
|
96
|
-
raise Faraday::
|
106
|
+
raise Faraday::ConnectionFailed, resp.return_message
|
97
107
|
end
|
98
108
|
end
|
99
109
|
|
@@ -107,6 +117,16 @@ module Faraday # :nodoc:
|
|
107
117
|
req
|
108
118
|
end
|
109
119
|
|
120
|
+
def typhoeus_request(env)
|
121
|
+
opts = {
|
122
|
+
:method => env[:method],
|
123
|
+
:body => env[:body],
|
124
|
+
:headers => env[:request_headers]
|
125
|
+
}.merge(@adapter_options)
|
126
|
+
|
127
|
+
::Typhoeus::Request.new(env[:url].to_s, opts)
|
128
|
+
end
|
129
|
+
|
110
130
|
def read_body(env)
|
111
131
|
env[:body] = env[:body].read if env[:body].respond_to? :read
|
112
132
|
end
|
@@ -135,6 +155,7 @@ module Faraday # :nodoc:
|
|
135
155
|
req.options[:proxy] = "#{proxy[:uri].scheme}://#{proxy[:uri].host}:#{proxy[:uri].port}"
|
136
156
|
|
137
157
|
if proxy[:user] && proxy[:password]
|
158
|
+
req.options[:proxyauth] = :any
|
138
159
|
req.options[:proxyuserpwd] = "#{proxy[:user]}:#{proxy[:password]}"
|
139
160
|
end
|
140
161
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
module Cache
|
3
|
+
# This module provides a simple way to cache HTTP responses using Dalli.
|
4
|
+
class Dalli
|
5
|
+
# @example Set Dalli as the Typhoeus cache backend
|
6
|
+
# Typhoeus::Config.cache = Typhoeus::Cache::Dalli.new
|
7
|
+
#
|
8
|
+
# @param [ Dalli::Client ] client
|
9
|
+
# A connection to the cache server. Defaults to `Dalli::Client.new`
|
10
|
+
# @param [ Hash ] options
|
11
|
+
# Options
|
12
|
+
# @option options [ Integer ] :default_ttl
|
13
|
+
# The default TTL of cached responses in seconds, for requests which do not set a cache_ttl.
|
14
|
+
def initialize(client = ::Dalli::Client.new, options = {})
|
15
|
+
@client = client
|
16
|
+
@default_ttl = options[:default_ttl]
|
17
|
+
end
|
18
|
+
|
19
|
+
def get(request)
|
20
|
+
@client.get(request.cache_key)
|
21
|
+
end
|
22
|
+
|
23
|
+
def set(request, response)
|
24
|
+
@client.set(request.cache_key, response, request.cache_ttl || @default_ttl)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
module Cache
|
3
|
+
# This module provides a simple way to cache HTTP responses in using the Rails cache.
|
4
|
+
class Rails
|
5
|
+
# @example Use the Rails cache setup to cache Typhoeus responses.
|
6
|
+
# Typhoeus::Config.cache = Typhoeus::Cache::Rails.new
|
7
|
+
#
|
8
|
+
# @param [ ActiveSupport::Cache::Store ] cache
|
9
|
+
# A Rails cache backend. Defaults to Rails.cache.
|
10
|
+
# @param [ Hash ] options
|
11
|
+
# Options
|
12
|
+
# @option options [ Integer ] :default_ttl
|
13
|
+
# The default TTL of cached responses in seconds, for requests which do not set a cache_ttl.
|
14
|
+
def initialize(cache = ::Rails.cache, options = {})
|
15
|
+
@cache = cache
|
16
|
+
@default_ttl = options[:default_ttl]
|
17
|
+
end
|
18
|
+
|
19
|
+
def get(request)
|
20
|
+
@cache.read(request)
|
21
|
+
end
|
22
|
+
|
23
|
+
def set(request, response)
|
24
|
+
@cache.write(request.cache_key, response, :expires_in => request.cache_ttl || @default_ttl)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
module Cache
|
3
|
+
# This module provides a simple way to cache HTTP responses in Redis.
|
4
|
+
class Redis
|
5
|
+
# @example Set Redis as the Typhoeus cache backend
|
6
|
+
# Typhoeus::Config.cache = Typhoeus::Cache::Redis.new
|
7
|
+
#
|
8
|
+
# @param [ Redis ] redis
|
9
|
+
# A connection to Redis. Defaults to `Redis.new`, which uses the
|
10
|
+
# `REDIS_URL` environment variable to connect
|
11
|
+
# @param [ Hash ] options
|
12
|
+
# Options
|
13
|
+
# @option options [ Integer ] :default_ttl
|
14
|
+
# The default TTL of cached responses in seconds, for requests which do not set a cache_ttl.
|
15
|
+
def initialize(redis = ::Redis.new, options = {})
|
16
|
+
@redis = redis
|
17
|
+
@default_ttl = options[:default_ttl]
|
18
|
+
end
|
19
|
+
|
20
|
+
def get(request)
|
21
|
+
serialized_response = @redis.get(request.cache_key)
|
22
|
+
return unless serialized_response
|
23
|
+
Marshal.load(serialized_response)
|
24
|
+
end
|
25
|
+
|
26
|
+
def set(request, response)
|
27
|
+
ttl = request.cache_ttl || @default_ttl
|
28
|
+
key = request.cache_key
|
29
|
+
serialized_response = Marshal.dump(response)
|
30
|
+
@redis.set(key, serialized_response)
|
31
|
+
@redis.expire(key, ttl) if ttl
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/typhoeus/config.rb
CHANGED
@@ -18,7 +18,7 @@ module Typhoeus
|
|
18
18
|
# {Typhoeus::Errors::NoStub} error is raised,
|
19
19
|
# when trying to do a real request. It's possible
|
20
20
|
# to work around inside
|
21
|
-
# {Typhoeus
|
21
|
+
# {Typhoeus.with_connection}.
|
22
22
|
#
|
23
23
|
# @return [ Boolean ]
|
24
24
|
#
|
@@ -58,5 +58,12 @@ module Typhoeus
|
|
58
58
|
#
|
59
59
|
# @see Typhoeus::Request#set_defaults
|
60
60
|
attr_accessor :user_agent
|
61
|
+
|
62
|
+
# Defines wether to use a proxy server for every request.
|
63
|
+
#
|
64
|
+
# @return [ String ]
|
65
|
+
#
|
66
|
+
# @see Typhoeus::Request#set_defaults
|
67
|
+
attr_accessor :proxy
|
61
68
|
end
|
62
69
|
end
|
@@ -35,7 +35,7 @@ module Typhoeus
|
|
35
35
|
|
36
36
|
REMOVED_OPTIONS = Set.new([:cache_key_basis, :cache_timeout, :user_agent])
|
37
37
|
|
38
|
-
SANITIZE_IGNORE = Set.new([:method, :cache_ttl])
|
38
|
+
SANITIZE_IGNORE = Set.new([:method, :cache_ttl, :cache])
|
39
39
|
SANITIZE_TIMEOUT = Set.new([:timeout_ms, :connecttimeout_ms])
|
40
40
|
|
41
41
|
# Returns the request provided.
|
@@ -97,7 +97,7 @@ module Typhoeus
|
|
97
97
|
# set nosignal to true by default
|
98
98
|
# this improves thread safety and timeout behavior
|
99
99
|
sanitized = {:nosignal => true}
|
100
|
-
|
100
|
+
options.each do |k,v|
|
101
101
|
s = k.to_sym
|
102
102
|
next if SANITIZE_IGNORE.include?(s)
|
103
103
|
if new_option = RENAMED_OPTIONS[k.to_sym]
|
@@ -121,7 +121,7 @@ module Typhoeus
|
|
121
121
|
end
|
122
122
|
|
123
123
|
def sanitize_timeout!(options, timeout)
|
124
|
-
timeout_ms =
|
124
|
+
timeout_ms = :"#{timeout}_ms"
|
125
125
|
if options[timeout] && options[timeout].round != options[timeout]
|
126
126
|
if !options[timeout_ms]
|
127
127
|
options[timeout_ms] = (options[timeout]*1000).ceil
|
@@ -155,6 +155,11 @@ module Typhoeus
|
|
155
155
|
request.execute_headers_callbacks(Response.new(Ethon::Easy::Mirror.from_easy(easy).options))
|
156
156
|
end
|
157
157
|
end
|
158
|
+
request.on_progress.each do |callback|
|
159
|
+
easy.on_progress do |dltotal, dlnow, ultotal, ulnow, easy|
|
160
|
+
callback.call(dltotal, dlnow, ultotal, ulnow, response)
|
161
|
+
end
|
162
|
+
end
|
158
163
|
easy.on_complete do |easy|
|
159
164
|
request.finish(Response.new(easy.mirror.options))
|
160
165
|
Typhoeus::Pool.release(easy)
|
data/lib/typhoeus/pool.rb
CHANGED
@@ -17,6 +17,8 @@ module Typhoeus
|
|
17
17
|
# @example Release easy.
|
18
18
|
# Typhoeus::Pool.release(easy)
|
19
19
|
def self.release(easy)
|
20
|
+
easy.cookielist = "flush" # dump all known cookies to 'cookiejar'
|
21
|
+
easy.cookielist = "all" # remove all cookies from memory for this handle
|
20
22
|
easy.reset
|
21
23
|
@mutex.synchronize { easies << easy }
|
22
24
|
end
|
@@ -15,7 +15,7 @@ module Typhoeus
|
|
15
15
|
#
|
16
16
|
# @option (see Typhoeus::Request#initialize)
|
17
17
|
#
|
18
|
-
# @return (see Typhoeus::
|
18
|
+
# @return (see Typhoeus::Response#initialize)
|
19
19
|
#
|
20
20
|
# @note (see Typhoeus::Request#initialize)
|
21
21
|
def get(base_url, options = {})
|
@@ -31,7 +31,7 @@ module Typhoeus
|
|
31
31
|
#
|
32
32
|
# @option (see Typhoeus::Request#initialize)
|
33
33
|
#
|
34
|
-
# @return (see Typhoeus::
|
34
|
+
# @return (see Typhoeus::Response#initialize)
|
35
35
|
#
|
36
36
|
# @note (see Typhoeus::Request#initialize)
|
37
37
|
def post(base_url, options = {})
|
@@ -50,7 +50,7 @@ module Typhoeus
|
|
50
50
|
# @option options :body [ Hash ] Body hash which
|
51
51
|
# becomes a PUT request body.
|
52
52
|
#
|
53
|
-
# @return (see Typhoeus::
|
53
|
+
# @return (see Typhoeus::Response#initialize)
|
54
54
|
#
|
55
55
|
# @note (see Typhoeus::Request#initialize)
|
56
56
|
def put(base_url, options = {})
|
@@ -66,7 +66,7 @@ module Typhoeus
|
|
66
66
|
#
|
67
67
|
# @option (see Typhoeus::Request#initialize)
|
68
68
|
#
|
69
|
-
# @return (see Typhoeus::
|
69
|
+
# @return (see Typhoeus::Response#initialize)
|
70
70
|
#
|
71
71
|
# @note (see Typhoeus::Request#initialize)
|
72
72
|
def delete(base_url, options = {})
|
@@ -82,7 +82,7 @@ module Typhoeus
|
|
82
82
|
#
|
83
83
|
# @option (see Typhoeus::Request#initialize)
|
84
84
|
#
|
85
|
-
# @return (see Typhoeus::
|
85
|
+
# @return (see Typhoeus::Response#initialize)
|
86
86
|
#
|
87
87
|
# @note (see Typhoeus::Request#initialize)
|
88
88
|
def head(base_url, options = {})
|
@@ -98,7 +98,7 @@ module Typhoeus
|
|
98
98
|
#
|
99
99
|
# @option (see Typhoeus::Request#initialize)
|
100
100
|
#
|
101
|
-
# @return (see Typhoeus::
|
101
|
+
# @return (see Typhoeus::Response#initialize)
|
102
102
|
#
|
103
103
|
# @note (see Typhoeus::Request#initialize)
|
104
104
|
def patch(base_url, options = {})
|
@@ -114,7 +114,7 @@ module Typhoeus
|
|
114
114
|
#
|
115
115
|
# @option (see Typhoeus::Request#initialize)
|
116
116
|
#
|
117
|
-
# @return (see Typhoeus::
|
117
|
+
# @return (see Typhoeus::Response#initialize)
|
118
118
|
#
|
119
119
|
# @note (see Typhoeus::Request#initialize)
|
120
120
|
def options(base_url, options = {})
|
@@ -2,16 +2,16 @@ module Typhoeus
|
|
2
2
|
class Request
|
3
3
|
module Cacheable
|
4
4
|
def response=(response)
|
5
|
-
|
5
|
+
cache.set(self, response) if cacheable? && !response.cached?
|
6
6
|
super
|
7
7
|
end
|
8
8
|
|
9
9
|
def cacheable?
|
10
|
-
|
10
|
+
cache
|
11
11
|
end
|
12
12
|
|
13
13
|
def run
|
14
|
-
if
|
14
|
+
if response = cached_response
|
15
15
|
response.cached = true
|
16
16
|
finish(response)
|
17
17
|
else
|
@@ -19,9 +19,20 @@ module Typhoeus
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
+
def cached_response
|
23
|
+
cacheable? && cache.get(self)
|
24
|
+
end
|
25
|
+
|
22
26
|
def cache_ttl
|
23
27
|
options[:cache_ttl]
|
24
28
|
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def cache
|
33
|
+
return nil if options[:cache] === false
|
34
|
+
options[:cache] || Typhoeus::Config.cache
|
35
|
+
end
|
25
36
|
end
|
26
37
|
end
|
27
38
|
end
|