down 2.2.1 → 2.3.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1d8c92123cb9fa3a46484b51840f8363c74cdd6b
4
- data.tar.gz: 05d06a93dbe891dfe90332acfb0dd06bfd98e6da
3
+ metadata.gz: 351b0f1708bc156090b334caf7ad619cd439dee6
4
+ data.tar.gz: 72b4dbb2e6a62358292dddbea558c406477c5167
5
5
  SHA512:
6
- metadata.gz: 67f3abe4b028843b738d3e1e0cd4642664831dd3d4240a8cf0c7197b14bcbb8a5de0f1e9f982a90f651c82cb783ea09f83ac25a9713948d4d7f4f8ffc2b3ba3c
7
- data.tar.gz: cc63303532e1e748a997051d1349335c63ebf5f56d9725ffafa04eea2b4f36679e4f29830c889f2f6c483eec96c805a44cf06932fbe1aa44d2ae0809d17fff4a
6
+ metadata.gz: c1387a3d10dae8c1c22df454d7dc47d16e6b9a752329168a86f8c9934713dab3e32b079afb6d0231901129dfc6cea5c27eda2129271e9d825c5ca01259362aae
7
+ data.tar.gz: 6be19252cc1a7b9439c26ca87da199e4c009e5c5286bd3a3809db8ca9eff9600ad9bdd4bb3c7c8572e864990d03a6b3ba9e4246b140be9a6f58cc1432b931a25
data/README.md CHANGED
@@ -17,7 +17,7 @@ tempfile = Down.download("http://example.com/nature.jpg")
17
17
  tempfile #=> #<Tempfile:/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/20150925-55456-z7vxqz.jpg>
18
18
  ```
19
19
 
20
- ## Features
20
+ ## Downloading
21
21
 
22
22
  If you're downloading files from URLs that come from you, then it's probably
23
23
  enough to just use `open-uri`. However, if you're accepting URLs from your
@@ -113,18 +113,6 @@ Down.download "http://example.com/image.jpg",
113
113
  read_timeout: 5
114
114
  ```
115
115
 
116
- ### Streaming
117
-
118
- Down has the ability to stream remote files, yielding chunks when they're
119
- received:
120
-
121
- ```rb
122
- Down.stream("http://example.com/image.jpg") { |chunk, content_length| ... }
123
- ```
124
-
125
- The `content_length` argument is set from the `Content-Length` response header
126
- if it's present.
127
-
128
116
  ### Copying to tempfile
129
117
 
130
118
  Down has another "hidden" utility method, `#copy_to_tempfile`, which creates
@@ -137,6 +125,37 @@ tempfile = Down.copy_to_tempfile "basename.jpg", io
137
125
  tempfile.path #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/down20151116-77262-jgcx65.jpg"
138
126
  ```
139
127
 
128
+ ## Streaming
129
+
130
+ Down has the ability to stream the remote file as it is being downloaded. The
131
+ `Down.open` method returns an IO object which represents the remote file on the
132
+ given URL. When you read from it, Down internally downloads chunks of the
133
+ remote file, but only how much is needed.
134
+
135
+ ```rb
136
+ remote_file = Down.open("http://example.com/image.jpg")
137
+ remote_file.size # read from the "Content-Length" header
138
+
139
+ remote_file.read(1024) # downloads and returns first 1 KB
140
+ remote_file.read(1024) # downloads and returns next 1 KB
141
+ remote_file.read # downloads and returns the rest of the file
142
+
143
+ remote_file.eof? #=> true
144
+ remote_file.rewind
145
+ remote_file.eof? #=> false
146
+
147
+ remote_file.close # closes the HTTP connection and deletes the internal Tempfile
148
+ ```
149
+
150
+ You can also yield chunks directly as they're downloaded:
151
+
152
+ ```rb
153
+ remote_file = Down.open("http://example.com/image.jpg")
154
+ remote_file.each_chunk do |chunk|
155
+ # ...
156
+ end
157
+ ```
158
+
140
159
  ## Supported Ruby versions
141
160
 
142
161
  * MRI 1.9.3
@@ -1,18 +1,20 @@
1
+ require File.expand_path("../lib/down/version", __FILE__)
2
+
1
3
  Gem::Specification.new do |spec|
2
4
  spec.name = "down"
3
- spec.version = "2.2.1"
5
+ spec.version = Down::VERSION
4
6
  spec.authors = ["Janko Marohnić"]
5
7
  spec.email = ["janko.marohnic@gmail.com"]
6
8
 
7
- spec.summary = "Robust file download from URL using open-uri."
8
- spec.description = "Robust file download from URL using open-uri."
9
+ spec.summary = "Robust streaming downloads using net/http."
9
10
  spec.homepage = "https://github.com/janko-m/down"
10
11
  spec.license = "MIT"
11
12
 
12
- spec.files = ["README.md", "LICENSE.txt", "down.gemspec", "lib/down.rb"]
13
+ spec.files = Dir["README.md", "LICENSE.txt", "*.gemspec", "lib/**/*.rb"]
13
14
  spec.require_paths = ["lib"]
14
15
 
15
16
  spec.add_development_dependency "rake"
16
17
  spec.add_development_dependency "minitest", "~> 5.8"
17
18
  spec.add_development_dependency "webmock"
19
+ spec.add_development_dependency "mocha"
18
20
  end
@@ -1,3 +1,5 @@
1
+ require "down/version"
2
+
1
3
  require "open-uri"
2
4
  require "tempfile"
3
5
  require "fileutils"
@@ -24,8 +26,9 @@ module Down
24
26
 
25
27
  begin
26
28
  uri = URI.parse(url)
27
- downloaded_file = uri.open({
28
- "User-Agent" => "Down/1.0.0",
29
+
30
+ open_uri_options = {
31
+ "User-Agent" => "Down/#{VERSION}",
29
32
  content_length_proc: proc { |size|
30
33
  if size && max_size && size > max_size
31
34
  raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
@@ -40,7 +43,17 @@ module Down
40
43
  },
41
44
  read_timeout: timeout,
42
45
  redirect: false,
43
- }.merge(options))
46
+ }
47
+
48
+ if uri.user || uri.password
49
+ open_uri_options[:http_basic_authentication] = [uri.user, uri.password]
50
+ uri.user = nil
51
+ uri.password = nil
52
+ end
53
+
54
+ open_uri_options.update(options)
55
+
56
+ downloaded_file = uri.open(open_uri_options)
44
57
  rescue OpenURI::HTTPRedirect => redirect
45
58
  url = redirect.uri.to_s
46
59
  retry if (tries -= 1) > 0
@@ -64,6 +77,12 @@ module Down
64
77
  end
65
78
 
66
79
  def stream(url, options = {})
80
+ warn "Down.stream is deprecated and will be removed in Down 3. Use Down.open instead."
81
+ io = open(url, options)
82
+ io.each_chunk { |chunk| yield chunk, io.size }
83
+ end
84
+
85
+ def open(url, options = {})
67
86
  uri = URI.parse(url)
68
87
  http = Net::HTTP.new(uri.host, uri.port)
69
88
 
@@ -83,13 +102,20 @@ module Down
83
102
  http.cert_store = store
84
103
  end
85
104
 
86
- http.start do
87
- req = Net::HTTP::Get.new(uri.to_s)
88
- http.request(req) do |response|
89
- content_length = response["Content-Length"].to_i if response["Content-Length"]
90
- response.read_body { |chunk| yield chunk, content_length }
105
+ request = Fiber.new do
106
+ http.start do
107
+ http.request_get(url) do |response|
108
+ Fiber.yield response
109
+ end
91
110
  end
92
111
  end
112
+
113
+ response = request.resume
114
+ content_length = Integer(response["Content-Length"]) if response["Content-Length"]
115
+ chunks = response.to_enum(:read_body)
116
+ close_connection = -> { request.resume }
117
+
118
+ ChunkedIO.new(size: content_length, chunks: chunks, on_close: close_connection)
93
119
  end
94
120
 
95
121
  def copy_to_tempfile(basename, io)
@@ -106,6 +132,65 @@ module Down
106
132
  tempfile
107
133
  end
108
134
 
135
+ class ChunkedIO
136
+ attr_reader :tempfile
137
+
138
+ def initialize(options)
139
+ @size = options.fetch(:size)
140
+ @chunks = options.fetch(:chunks)
141
+ @on_close = options.fetch(:on_close, nil)
142
+ @tempfile = Tempfile.new("down", binmode: true)
143
+ @eof = false
144
+ end
145
+
146
+ def size
147
+ @size
148
+ end
149
+
150
+ def read(length = nil, outbuf = nil)
151
+ loop do
152
+ break if length && (@tempfile.pos + length <= @tempfile.size)
153
+ next_chunk or break
154
+ end
155
+ @tempfile.read(length, outbuf)
156
+ end
157
+
158
+ def each_chunk(&block)
159
+ @chunks.each(&block)
160
+ end
161
+
162
+ def eof?
163
+ @eof
164
+ end
165
+
166
+ def rewind
167
+ @tempfile.rewind
168
+ end
169
+
170
+ def close
171
+ @on_close.call if @on_close
172
+ @tempfile.close!
173
+ end
174
+
175
+ private
176
+
177
+ def next_chunk
178
+ chunk = @chunks.next
179
+ write(chunk)
180
+ chunk
181
+ rescue StopIteration
182
+ @eof = true
183
+ nil
184
+ end
185
+
186
+ def write(chunk)
187
+ current_pos = @tempfile.pos
188
+ @tempfile.pos = @tempfile.size
189
+ @tempfile.write(chunk)
190
+ @tempfile.pos = current_pos
191
+ end
192
+ end
193
+
109
194
  module DownloadedFile
110
195
  def original_filename
111
196
  path = base_uri.path
@@ -0,0 +1,3 @@
1
+ module Down
2
+ VERSION = "2.3.2"
3
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: down
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.1
4
+ version: 2.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Janko Marohnić
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-06 00:00:00.000000000 Z
11
+ date: 2016-06-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -52,7 +52,21 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
- description: Robust file download from URL using open-uri.
55
+ - !ruby/object:Gem::Dependency
56
+ name: mocha
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description:
56
70
  email:
57
71
  - janko.marohnic@gmail.com
58
72
  executables: []
@@ -63,6 +77,7 @@ files:
63
77
  - README.md
64
78
  - down.gemspec
65
79
  - lib/down.rb
80
+ - lib/down/version.rb
66
81
  homepage: https://github.com/janko-m/down
67
82
  licenses:
68
83
  - MIT
@@ -86,6 +101,6 @@ rubyforge_project:
86
101
  rubygems_version: 2.5.1
87
102
  signing_key:
88
103
  specification_version: 4
89
- summary: Robust file download from URL using open-uri.
104
+ summary: Robust streaming downloads using net/http.
90
105
  test_files: []
91
106
  has_rdoc: