down 2.3.5 → 2.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/down.rb +15 -77
- data/lib/down/chunked_io.rb +76 -0
- data/lib/down/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5742b5429e56268121f7c44d6f3f43a1d980ed93
|
|
4
|
+
data.tar.gz: 73baaef981286e0cfcc2ac51eb53dbbac168d240
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 667d87034446d1b54e41fd0bd871a7279da7a0970faa23a8e47a75c728d8cbaf10d8b5fc3bc779205250e14e45a4db16a8a79bdc878aea6e0860eb9decb6fe02
|
|
7
|
+
data.tar.gz: 1c23677ec51dd05c30a409e157406d3549ea052a6ce15e2f5351a52ef0a13f4491de2f14471162acbb75fd481b62d093d6349525ecbe3e32c811ba2bfe8fc145
|
data/README.md
CHANGED
|
@@ -127,7 +127,7 @@ tempfile.path #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/down20151116
|
|
|
127
127
|
|
|
128
128
|
## Streaming
|
|
129
129
|
|
|
130
|
-
Down has the ability to access
|
|
130
|
+
Down has the ability to access content of the remote file *as it is being
|
|
131
131
|
downloaded*. The `Down.open` method returns an IO object which represents the
|
|
132
132
|
remote file on the given URL. When you read from it, Down internally downloads
|
|
133
133
|
chunks of the remote file, but only how much is needed.
|
|
@@ -173,6 +173,8 @@ Down::ChunkedIO.new(...)
|
|
|
173
173
|
Here is an example of wrapping streaming MongoDB files:
|
|
174
174
|
|
|
175
175
|
```rb
|
|
176
|
+
require "down/chunked_io"
|
|
177
|
+
|
|
176
178
|
mongo = Mongo::Client.new(...)
|
|
177
179
|
bucket = mongo.database.fs
|
|
178
180
|
|
|
@@ -186,8 +188,6 @@ io = Down::ChunkedIO.new(
|
|
|
186
188
|
)
|
|
187
189
|
```
|
|
188
190
|
|
|
189
|
-
### Down
|
|
190
|
-
|
|
191
191
|
## Supported Ruby versions
|
|
192
192
|
|
|
193
193
|
* MRI 1.9.3
|
data/lib/down.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "down/version"
|
|
2
|
+
require "down/chunked_io"
|
|
2
3
|
|
|
3
4
|
require "open-uri"
|
|
4
5
|
require "net/http"
|
|
@@ -81,6 +82,7 @@ module Down
|
|
|
81
82
|
warn "Down.stream is deprecated and will be removed in Down 3. Use Down.open instead."
|
|
82
83
|
io = open(url, options)
|
|
83
84
|
io.each_chunk { |chunk| yield chunk, io.size }
|
|
85
|
+
io.close
|
|
84
86
|
end
|
|
85
87
|
|
|
86
88
|
def open(url, options = {})
|
|
@@ -113,11 +115,12 @@ module Down
|
|
|
113
115
|
end
|
|
114
116
|
|
|
115
117
|
response = request.resume
|
|
116
|
-
content_length = Integer(response["Content-Length"]) if response["Content-Length"]
|
|
117
|
-
chunks = response.to_enum(:read_body)
|
|
118
|
-
close_connection = -> { request.resume }
|
|
119
118
|
|
|
120
|
-
ChunkedIO.new(
|
|
119
|
+
ChunkedIO.new(
|
|
120
|
+
chunks: response.enum_for(:read_body),
|
|
121
|
+
size: response["Content-Length"] && response["Content-Length"].to_i,
|
|
122
|
+
on_close: -> { request.resume },
|
|
123
|
+
)
|
|
121
124
|
end
|
|
122
125
|
|
|
123
126
|
def copy_to_tempfile(basename, io)
|
|
@@ -134,86 +137,21 @@ module Down
|
|
|
134
137
|
tempfile
|
|
135
138
|
end
|
|
136
139
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
def initialize(options)
|
|
141
|
-
@size = options.fetch(:size)
|
|
142
|
-
@chunks = options.fetch(:chunks)
|
|
143
|
-
@on_close = options.fetch(:on_close, ->{})
|
|
144
|
-
@tempfile = Tempfile.new("down", binmode: true)
|
|
145
|
-
end
|
|
146
|
-
|
|
147
|
-
def size
|
|
148
|
-
@size
|
|
149
|
-
end
|
|
150
|
-
|
|
151
|
-
def read(length = nil, outbuf = nil)
|
|
152
|
-
download_chunk until enough_downloaded?(length) || download_finished?
|
|
153
|
-
@tempfile.read(length, outbuf)
|
|
154
|
-
end
|
|
155
|
-
|
|
156
|
-
def each_chunk
|
|
157
|
-
return enum_for(__method__) if !block_given?
|
|
158
|
-
yield download_chunk until download_finished?
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
def eof?
|
|
162
|
-
@tempfile.eof? && download_finished?
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
def rewind
|
|
166
|
-
@tempfile.rewind
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
def close
|
|
170
|
-
terminate_download
|
|
171
|
-
@tempfile.close!
|
|
140
|
+
module DownloadedFile
|
|
141
|
+
def original_filename
|
|
142
|
+
filename_from_content_disposition || filename_from_uri
|
|
172
143
|
end
|
|
173
144
|
|
|
174
145
|
private
|
|
175
146
|
|
|
176
|
-
def
|
|
177
|
-
|
|
178
|
-
write(chunk)
|
|
179
|
-
begin
|
|
180
|
-
@chunks.peek
|
|
181
|
-
rescue StopIteration
|
|
182
|
-
terminate_download
|
|
183
|
-
end
|
|
184
|
-
chunk
|
|
185
|
-
end
|
|
186
|
-
|
|
187
|
-
def enough_downloaded?(length)
|
|
188
|
-
length && (@tempfile.pos + length <= @tempfile.size)
|
|
189
|
-
end
|
|
190
|
-
|
|
191
|
-
def download_finished?
|
|
192
|
-
!@on_close
|
|
193
|
-
end
|
|
194
|
-
|
|
195
|
-
def terminate_download
|
|
196
|
-
if @on_close
|
|
197
|
-
@on_close.call
|
|
198
|
-
@on_close = nil
|
|
199
|
-
end
|
|
200
|
-
end
|
|
201
|
-
|
|
202
|
-
def write(chunk)
|
|
203
|
-
current_pos = @tempfile.pos
|
|
204
|
-
@tempfile.pos = @tempfile.size
|
|
205
|
-
@tempfile.write(chunk)
|
|
206
|
-
@tempfile.pos = current_pos
|
|
147
|
+
def filename_from_content_disposition
|
|
148
|
+
meta["content-disposition"].to_s[/filename="([^"]+)"/, 1]
|
|
207
149
|
end
|
|
208
|
-
end
|
|
209
150
|
|
|
210
|
-
|
|
211
|
-
def original_filename
|
|
151
|
+
def filename_from_uri
|
|
212
152
|
path = base_uri.path
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
CGI.unescape(filename)
|
|
216
|
-
end
|
|
153
|
+
filename = path.split("/").last
|
|
154
|
+
CGI.unescape(filename) if filename
|
|
217
155
|
end
|
|
218
156
|
end
|
|
219
157
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
require "tempfile"
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
class ChunkedIO
|
|
5
|
+
attr_reader :tempfile
|
|
6
|
+
|
|
7
|
+
def initialize(options)
|
|
8
|
+
@size = options.fetch(:size)
|
|
9
|
+
@chunks = options.fetch(:chunks)
|
|
10
|
+
@on_close = options.fetch(:on_close, ->{})
|
|
11
|
+
@tempfile = Tempfile.new("down", binmode: true)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def size
|
|
15
|
+
@size
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def read(length = nil, outbuf = nil)
|
|
19
|
+
download_chunk until enough_downloaded?(length) || download_finished?
|
|
20
|
+
@tempfile.read(length, outbuf)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def each_chunk
|
|
24
|
+
return enum_for(__method__) if !block_given?
|
|
25
|
+
yield download_chunk until download_finished?
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def eof?
|
|
29
|
+
@tempfile.eof? && download_finished?
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def rewind
|
|
33
|
+
@tempfile.rewind
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def close
|
|
37
|
+
terminate_download
|
|
38
|
+
@tempfile.close!
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def download_chunk
|
|
44
|
+
chunk = @chunks.next
|
|
45
|
+
write(chunk)
|
|
46
|
+
begin
|
|
47
|
+
@chunks.peek
|
|
48
|
+
rescue StopIteration
|
|
49
|
+
terminate_download
|
|
50
|
+
end
|
|
51
|
+
chunk
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def enough_downloaded?(length)
|
|
55
|
+
length && (@tempfile.pos + length <= @tempfile.size)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def download_finished?
|
|
59
|
+
!@on_close
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def terminate_download
|
|
63
|
+
if @on_close
|
|
64
|
+
@on_close.call
|
|
65
|
+
@on_close = nil
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def write(chunk)
|
|
70
|
+
current_pos = @tempfile.pos
|
|
71
|
+
@tempfile.pos = @tempfile.size
|
|
72
|
+
@tempfile.write(chunk)
|
|
73
|
+
@tempfile.pos = current_pos
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
data/lib/down/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: down
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.3.
|
|
4
|
+
version: 2.3.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Janko Marohnić
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-07-
|
|
11
|
+
date: 2016-07-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|
|
@@ -77,6 +77,7 @@ files:
|
|
|
77
77
|
- README.md
|
|
78
78
|
- down.gemspec
|
|
79
79
|
- lib/down.rb
|
|
80
|
+
- lib/down/chunked_io.rb
|
|
80
81
|
- lib/down/version.rb
|
|
81
82
|
homepage: https://github.com/janko-m/down
|
|
82
83
|
licenses:
|
|
@@ -103,4 +104,3 @@ signing_key:
|
|
|
103
104
|
specification_version: 4
|
|
104
105
|
summary: Robust streaming downloads using net/http.
|
|
105
106
|
test_files: []
|
|
106
|
-
has_rdoc:
|