down 2.4.3 → 2.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f666ebb30afe8b5e6faaeb0ac159808620a1dd21
4
- data.tar.gz: 8efca55dd0c9c7e8cfd6aafd82e857b7a3220b5e
3
+ metadata.gz: b36995e341763d157bd7dfc5e4de2aeee8f7d42e
4
+ data.tar.gz: f4f910e36886c4c64a1f6528a8f773716e28e940
5
5
  SHA512:
6
- metadata.gz: d5641778d393b156f5b1c32df00a890f62c6454af3bc7b8563921cea760e12c159ddd66dbf70f7c2b77183c7dee1cd6083c7456c58088f47f5ea5fb1b92fe7ef
7
- data.tar.gz: 7b5c5100c1418e4f494866efa45d8b14fd5e91fb84f7515d1bb3bdb0fdd0c0ac039f4dc02d8942d9ef1083dcda546ff47088e0e173ec3bbf19cb79dd58edada2
6
+ metadata.gz: 7c01afed653ed786903043d1b745afa99ec35d5e892d37df644171aad47d2d13a4bda3d98df5ebc890684180488fff62364537c78256e71d2a52c68fd2c69898
7
+ data.tar.gz: 1b61d561a3a0088c8df2e208cd71d15b8a4314de31a1f40d881eeda067396fbc5d0c80d6f78e6796f847717abbd0da3cf813b41cf7fb47dd10cd58e4a1f22454
data/README.md CHANGED
@@ -113,18 +113,6 @@ Down.download "http://example.com/image.jpg",
113
113
  read_timeout: 5
114
114
  ```
115
115
 
116
- ### Copying to tempfile
117
-
118
- Down has another "hidden" utility method, `#copy_to_tempfile`, which creates
119
- a Tempfile out of the given file. The `#download` method uses it internally,
120
- but it's also publicly available for direct use:
121
-
122
- ```rb
123
- io # IO object that you want to copy to tempfile
124
- tempfile = Down.copy_to_tempfile "basename.jpg", io
125
- tempfile.path #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/down20151116-77262-jgcx65.jpg"
126
- ```
127
-
128
116
  ## Streaming
129
117
 
130
118
  Down has the ability to access content of the remote file *as it is being
@@ -157,7 +145,18 @@ end
157
145
  remote_file.close
158
146
  ```
159
147
 
160
- It accepts the `:ssl_verify_mode` and `:ssl_ca_cert` options with the same
148
+ You can access the response status and headers of the HTTP request that was made:
149
+
150
+ ```rb
151
+ remote_file = Down.open("http://example.com/image.jpg")
152
+ remote_file.data[:status] #=> 200
153
+ remote_file.data[:headers] #=> { ... }
154
+ ```
155
+
156
+ Note that `Down::NotFound` error will automatically be raised if response
157
+ status was 4xx or 5xx.
158
+
159
+ `Down.open` accepts `:ssl_verify_mode` and `:ssl_ca_cert` options with the same
161
160
  semantics as in `open-uri`, and any options with String keys will be
162
161
  interpreted as request headers.
163
162
 
@@ -196,6 +195,28 @@ io = Down::ChunkedIO.new(
196
195
  )
197
196
  ```
198
197
 
198
+ ### Proxy
199
+
200
+ Both `Down.download` and `Down.open` support a `:proxy` option, where you can
201
+ specify a URL to an HTTP proxy which should be used when downloading.
202
+
203
+ ```rb
204
+ Down.download("http://example.com/image.jpg", proxy: "http://proxy.org")
205
+ Down.open("http://example.com/image.jpg", proxy: "http://user:password@proxy.org")
206
+ ```
207
+
208
+ ### Copying to tempfile
209
+
210
+ Down has another "hidden" utility method, `#copy_to_tempfile`, which creates
211
+ a Tempfile out of the given file. The `#download` method uses it internally,
212
+ but it's also publicly available for direct use:
213
+
214
+ ```rb
215
+ io # IO object that you want to copy to tempfile
216
+ tempfile = Down.copy_to_tempfile "basename.jpg", io
217
+ tempfile.path #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/down20151116-77262-jgcx65.jpg"
218
+ ```
219
+
199
220
  ## Supported Ruby versions
200
221
 
201
222
  * MRI 1.9.3
data/down.gemspec CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
15
15
 
16
16
  spec.add_development_dependency "rake"
17
17
  spec.add_development_dependency "minitest", "~> 5.8"
18
- spec.add_development_dependency "webmock"
18
+ spec.add_development_dependency "webmock", "~> 2.3"
19
19
  spec.add_development_dependency "addressable", "< 2.5"
20
20
  spec.add_development_dependency "mocha"
21
21
  end
data/lib/down.rb CHANGED
@@ -5,7 +5,7 @@ require "open-uri"
5
5
  require "net/http"
6
6
  require "tempfile"
7
7
  require "fileutils"
8
- require "cgi/util"
8
+ require "cgi"
9
9
 
10
10
  module Down
11
11
  class Error < StandardError; end
@@ -14,7 +14,7 @@ module Down
14
14
 
15
15
  module_function
16
16
 
17
- def download(url, options = {})
17
+ def download(uri, options = {})
18
18
  warn "Passing :timeout option to `Down.download` is deprecated and will be removed in Down 3. You should use open-uri's :open_timeout and/or :read_timeout." if options.key?(:timeout)
19
19
  warn "Passing :progress option to `Down.download` is deprecated and will be removed in Down 3. You should use open-uri's :progress_proc." if options.key?(:progress)
20
20
 
@@ -24,10 +24,24 @@ module Down
24
24
  content_length_proc = options.delete(:content_length_proc)
25
25
  timeout = options.delete(:timeout)
26
26
 
27
+ if options[:proxy]
28
+ proxy = URI(options[:proxy])
29
+ user = proxy.user
30
+ password = proxy.password
31
+
32
+ if user || password
33
+ proxy.user = nil
34
+ proxy.password = nil
35
+
36
+ options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
37
+ options.delete(:proxy)
38
+ end
39
+ end
40
+
27
41
  tries = max_redirects + 1
28
42
 
29
43
  begin
30
- uri = URI.parse(url)
44
+ uri = URI(uri)
31
45
 
32
46
  open_uri_options = {
33
47
  "User-Agent" => "Down/#{VERSION}",
@@ -57,12 +71,12 @@ module Down
57
71
 
58
72
  downloaded_file = uri.open(open_uri_options)
59
73
  rescue OpenURI::HTTPRedirect => redirect
60
- url = redirect.uri.to_s
74
+ uri = redirect.uri
61
75
  retry if (tries -= 1) > 0
62
- raise Down::NotFound, "too many redirects: #{url}"
76
+ raise Down::NotFound, "too many redirects: #{uri.to_s}"
63
77
  rescue => error
64
78
  raise if error.is_a?(Down::Error)
65
- raise Down::NotFound, "file not found: #{url}"
79
+ raise Down::NotFound, "file not found: #{uri.to_s}"
66
80
  end
67
81
 
68
82
  # open-uri will return a StringIO instead of a Tempfile if the filesize is
@@ -85,9 +99,16 @@ module Down
85
99
  io.close
86
100
  end
87
101
 
88
- def open(url, options = {})
89
- uri = URI.parse(url)
90
- http = Net::HTTP.new(uri.host, uri.port)
102
+ def open(uri, options = {})
103
+ uri = URI(uri)
104
+ http_class = Net::HTTP
105
+
106
+ if options[:proxy]
107
+ proxy = URI.parse(options[:proxy])
108
+ http_class = Net::HTTP::Proxy(proxy.hostname, proxy.port, proxy.user, proxy.password)
109
+ end
110
+
111
+ http = http_class.new(uri.host, uri.port)
91
112
 
92
113
  # taken from open-uri implementation
93
114
  if uri.is_a?(URI::HTTPS)
@@ -109,7 +130,9 @@ module Down
109
130
 
110
131
  request = Fiber.new do
111
132
  http.start do
112
- http.request_get(uri.request_uri, request_headers) do |response|
133
+ req = Net::HTTP::Get.new(uri.request_uri, request_headers)
134
+ req.basic_auth(uri.user, uri.password) if uri.user || uri.password
135
+ http.request(req) do |response|
113
136
  Fiber.yield response
114
137
  response.instance_variable_set("@read", true)
115
138
  end
@@ -118,11 +141,13 @@ module Down
118
141
 
119
142
  response = request.resume
120
143
 
144
+ raise Down::NotFound, "request to #{uri.to_s} returned status #{response.code} and body:\n#{response.body}" if response.code.to_i.between?(400, 599)
145
+
121
146
  if response.chunked?
122
147
  # Net::HTTP's implementation of reading "Transfer-Encoding: chunked"
123
148
  # raises a Fiber error, so we work around it by downloading the whole
124
149
  # response body without Enumerators (which internally use Fibers).
125
- warn "Response from #{url} returned as \"Transfer-Encoding: chunked\", which Down cannot partially download, so the whole response body will be downloaded instead."
150
+ warn "Response from #{uri.to_s} returned as \"Transfer-Encoding: chunked\", which Down cannot partially download, so the whole response body will be downloaded instead."
126
151
 
127
152
  tempfile = Tempfile.new("down", binmode: true)
128
153
  response.read_body { |chunk| tempfile << chunk }
@@ -130,18 +155,28 @@ module Down
130
155
 
131
156
  request.resume # close HTTP connection
132
157
 
133
- ChunkedIO.new(
158
+ chunked_io = ChunkedIO.new(
134
159
  chunks: Enumerator.new { |y| y << tempfile.read(16*1024) until tempfile.eof? },
135
160
  size: tempfile.size,
136
161
  on_close: -> { tempfile.close! },
137
162
  )
138
163
  else
139
- ChunkedIO.new(
164
+ chunked_io = ChunkedIO.new(
140
165
  chunks: response.enum_for(:read_body),
141
166
  size: response["Content-Length"] && response["Content-Length"].to_i,
142
167
  on_close: -> { request.resume }, # close HTTP connnection
143
168
  )
144
169
  end
170
+
171
+ chunked_io.data[:status] = response.code.to_i
172
+ chunked_io.data[:headers] = {}
173
+
174
+ response.each_header do |downcased_name, value|
175
+ name = downcased_name.split("-").map(&:capitalize).join("-")
176
+ chunked_io.data[:headers].merge!(name => value)
177
+ end
178
+
179
+ chunked_io
145
180
  end
146
181
 
147
182
  def copy_to_tempfile(basename, io)
@@ -2,12 +2,13 @@ require "tempfile"
2
2
 
3
3
  module Down
4
4
  class ChunkedIO
5
- attr_reader :tempfile
5
+ attr_reader :tempfile, :data
6
6
 
7
7
  def initialize(options)
8
8
  @size = options.fetch(:size)
9
9
  @chunks = options.fetch(:chunks)
10
10
  @on_close = options.fetch(:on_close, ->{})
11
+ @data = options.fetch(:data, {})
11
12
  @tempfile = Tempfile.new("down", binmode: true)
12
13
 
13
14
  peek_chunk
@@ -24,7 +25,7 @@ module Down
24
25
 
25
26
  def each_chunk
26
27
  return enum_for(__method__) if !block_given?
27
- yield download_chunk until download_finished?
28
+ yield retrieve_chunk until download_finished?
28
29
  end
29
30
 
30
31
  def eof?
@@ -43,8 +44,11 @@ module Down
43
44
  private
44
45
 
45
46
  def download_chunk
47
+ write(retrieve_chunk)
48
+ end
49
+
50
+ def retrieve_chunk
46
51
  chunk = @chunks.next
47
- write(chunk)
48
52
  peek_chunk
49
53
  chunk
50
54
  end
@@ -64,10 +68,8 @@ module Down
64
68
  end
65
69
 
66
70
  def terminate_download
67
- if @on_close
68
- @on_close.call
69
- @on_close = nil
70
- end
71
+ @on_close.call if @on_close
72
+ @on_close = nil
71
73
  end
72
74
 
73
75
  def write(chunk)
data/lib/down/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Down
2
- VERSION = "2.4.3"
2
+ VERSION = "2.5.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: down
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.4.3
4
+ version: 2.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Janko Marohnić
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-04-06 00:00:00.000000000 Z
11
+ date: 2017-05-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -42,16 +42,16 @@ dependencies:
42
42
  name: webmock
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '0'
47
+ version: '2.3'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ">="
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '0'
54
+ version: '2.3'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: addressable
57
57
  requirement: !ruby/object:Gem::Requirement