down 5.6.3 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ # Generated from lib/down.rb with RBS::Inline
2
+
3
+ module Down
4
+ self.@backend: _Backend
5
+
6
+ @backend: _Backend
7
+
8
+ # : (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
9
+ # : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
10
+ def self?.download: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
11
+ | (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
12
+
13
+ # : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
14
+ def self?.open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
15
+
16
+ # Allows setting a backend via a symbol or a downloader object.
17
+ # : (Symbol | _Backend) -> _Backend
18
+ # : () -> _Backend
19
+ def self?.backend: (Symbol | _Backend) -> _Backend
20
+ | () -> _Backend
21
+ end
data/sig/manifest.yaml ADDED
@@ -0,0 +1,7 @@
1
+ dependencies:
2
+ - name: cgi-escape
3
+ - name: fileutils
4
+ - name: net-http
5
+ - name: open-uri
6
+ - name: tempfile
7
+ - name: uri
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: down
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.6.3
4
+ version: 6.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Janko Marohnić
@@ -37,6 +37,20 @@ dependencies:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
39
  version: '0.3'
40
+ - !ruby/object:Gem::Dependency
41
+ name: tempfile
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: 0.3.1
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: 0.3.1
40
54
  - !ruby/object:Gem::Dependency
41
55
  name: minitest
42
56
  requirement: !ruby/object:Gem::Requirement
@@ -160,7 +174,23 @@ files:
160
174
  - lib/down/net_http.rb
161
175
  - lib/down/utils.rb
162
176
  - lib/down/version.rb
163
- - lib/down/wget.rb
177
+ - sig/_patches/addressable.rbs
178
+ - sig/_patches/http.rbs
179
+ - sig/_patches/io.rbs
180
+ - sig/_patches/net_http.rbs
181
+ - sig/_patches/open_uri.rbs
182
+ - sig/_patches/stringio.rbs
183
+ - sig/down.rbs
184
+ - sig/generated/down.rbs
185
+ - sig/generated/down/backend.rbs
186
+ - sig/generated/down/chunked_io.rbs
187
+ - sig/generated/down/errors.rbs
188
+ - sig/generated/down/http.rbs
189
+ - sig/generated/down/httpx.rbs
190
+ - sig/generated/down/net_http.rbs
191
+ - sig/generated/down/utils.rbs
192
+ - sig/generated/down/version.rbs
193
+ - sig/manifest.yaml
164
194
  homepage: https://github.com/janko/down
165
195
  licenses:
166
196
  - MIT
data/lib/down/wget.rb DELETED
@@ -1,250 +0,0 @@
1
- # frozen-string-literal: true
2
-
3
- warn "The Down::Wget backend has been deprecated and will be removed in the next major version. Please use the Net::HTTP/open-uri, http.rb or HTTPX backend instead."
4
-
5
- begin
6
- require "posix-spawn"
7
- rescue LoadError
8
- require "open3"
9
- end
10
- require "http_parser"
11
-
12
- require "down/backend"
13
-
14
- require "tempfile"
15
- require "uri"
16
-
17
- module Down
18
- # Provides streaming downloads implemented with the wget command-line tool.
19
- # The design is very similar to Down::Http.
20
- class Wget < Backend
21
- # Initializes the backend with common defaults.
22
- def initialize(*arguments)
23
- @arguments = [
24
- user_agent: "Down/#{Down::VERSION}",
25
- max_redirect: 2,
26
- dns_timeout: 30,
27
- connect_timeout: 30,
28
- read_timeout: 30,
29
- ] + arguments
30
- end
31
-
32
- # Downlods the remote file to disk. Accepts wget command-line options and
33
- # some additional options as well.
34
- def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, extension: nil, **options)
35
- io = open(url, *args, **options, rewindable: false)
36
-
37
- content_length_proc.call(io.size) if content_length_proc && io.size
38
-
39
- if max_size && io.size && io.size > max_size
40
- raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
41
- end
42
-
43
- extname = extension ? ".#{extension}" : File.extname(URI(url).path)
44
- tempfile = Tempfile.new(["down-wget", extname], binmode: true)
45
-
46
- until io.eof?
47
- chunk = io.readpartial(nil, buffer ||= String.new)
48
-
49
- tempfile.write(chunk)
50
-
51
- progress_proc.call(tempfile.size) if progress_proc
52
-
53
- if max_size && tempfile.size > max_size
54
- raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
55
- end
56
- end
57
-
58
- tempfile.open # flush written content
59
-
60
- tempfile.extend Down::Wget::DownloadedFile
61
- tempfile.url = url
62
- tempfile.headers = io.data[:headers]
63
-
64
- download_result(tempfile, destination)
65
- rescue
66
- tempfile.close! if tempfile
67
- raise
68
- ensure
69
- io.close if io
70
- end
71
-
72
- # Starts retrieving the remote file and returns an IO-like object which
73
- # downloads the response body on-demand. Accepts wget command-line options.
74
- def open(url, *args, rewindable: true, **options)
75
- arguments = generate_command(url, *args, **options)
76
-
77
- command = Down::Wget::Command.execute(arguments)
78
- # Wrap the wget command output in an IO-like object.
79
- output = Down::ChunkedIO.new(
80
- chunks: command.enum_for(:output),
81
- on_close: command.method(:terminate),
82
- rewindable: false,
83
- )
84
-
85
- # https://github.com/tmm1/http_parser.rb/issues/29#issuecomment-309976363
86
- header_string = output.readpartial
87
- header_string << output.readpartial until header_string.include?("\r\n\r\n")
88
- header_string, first_chunk = header_string.split("\r\n\r\n", 2)
89
-
90
- # Use an HTTP parser to parse out the response headers.
91
- parser = HTTP::Parser.new
92
- parser << header_string
93
-
94
- if parser.headers.nil?
95
- output.close
96
- raise Down::Error, "failed to parse response headers"
97
- end
98
-
99
- headers = normalize_headers(parser.headers)
100
- status = parser.status_code
101
-
102
- content_length = headers["Content-Length"].to_i if headers["Content-Length"]
103
- charset = headers["Content-Type"][/;\s*charset=([^;]+)/i, 1] if headers["Content-Type"]
104
-
105
- # Create an Enumerator which will lazily retrieve chunks of response body.
106
- chunks = Enumerator.new do |yielder|
107
- yielder << first_chunk if first_chunk
108
- yielder << output.readpartial until output.eof?
109
- end
110
-
111
- Down::ChunkedIO.new(
112
- chunks: chunks,
113
- size: content_length,
114
- encoding: charset,
115
- rewindable: rewindable,
116
- on_close: output.method(:close),
117
- data: { status: status, headers: headers },
118
- )
119
- end
120
-
121
- private
122
-
123
- # Generates the wget command.
124
- def generate_command(url, *args, **options)
125
- command = %W[wget --no-verbose --save-headers -O -]
126
-
127
- options = @arguments.grep(Hash).inject({}, :merge).merge(options)
128
- args = @arguments.grep(->(o){!o.is_a?(Hash)}) + args
129
-
130
- (args + options.to_a).each do |option, value|
131
- if option.is_a?(String)
132
- command << option
133
- elsif option.length == 1
134
- command << "-#{option}"
135
- else
136
- command << "--#{option.to_s.gsub("_", "-")}"
137
- end
138
-
139
- command << value.to_s unless value.nil?
140
- end
141
-
142
- command << url
143
- command
144
- end
145
-
146
- # Handles executing the wget command.
147
- class Command
148
- PIPE_BUFFER_SIZE = 64*1024
149
-
150
- def self.execute(arguments)
151
- # posix-spawn gem has better performance, so we use it if it's available
152
- if defined?(POSIX::Spawn)
153
- pid, stdin_pipe, stdout_pipe, stderr_pipe = POSIX::Spawn.popen4(*arguments)
154
- status_reaper = Process.detach(pid)
155
- else
156
- stdin_pipe, stdout_pipe, stderr_pipe, status_reaper = Open3.popen3(*arguments)
157
- end
158
-
159
- stdin_pipe.close
160
- [stdout_pipe, stderr_pipe].each(&:binmode)
161
-
162
- new(stdout_pipe, stderr_pipe, status_reaper)
163
- rescue Errno::ENOENT
164
- raise Down::Error, "wget is not installed"
165
- end
166
-
167
- def initialize(stdout_pipe, stderr_pipe, status_reaper)
168
- @status_reaper = status_reaper
169
- @stdout_pipe = stdout_pipe
170
- @stderr_pipe = stderr_pipe
171
- end
172
-
173
- # Yields chunks of stdout. At the end handles the exit status.
174
- def output
175
- # Keep emptying the stderr buffer, to allow the subprocess to send more
176
- # than 64KB if it wants to.
177
- stderr_reader = Thread.new { @stderr_pipe.read }
178
-
179
- yield @stdout_pipe.readpartial(PIPE_BUFFER_SIZE) until @stdout_pipe.eof?
180
-
181
- status = @status_reaper.value
182
- stderr = stderr_reader.value
183
-
184
- close
185
-
186
- handle_status(status, stderr)
187
- end
188
-
189
- def terminate
190
- begin
191
- Process.kill("TERM", @status_reaper[:pid])
192
- Process.waitpid(@status_reaper[:pid])
193
- rescue Errno::ESRCH
194
- # process has already terminated
195
- end
196
-
197
- close
198
- end
199
-
200
- def close
201
- @stdout_pipe.close unless @stdout_pipe.closed?
202
- @stderr_pipe.close unless @stderr_pipe.closed?
203
- end
204
-
205
- private
206
-
207
- # Translates nonzero wget exit statuses into exceptions.
208
- def handle_status(status, stderr)
209
- case status.exitstatus
210
- when 0 # No problems occurred
211
- # success
212
- when 1, # Generic error code
213
- 2, # Parse error---for instance, when parsing command-line options, the .wgetrc or .netrc...
214
- 3 # File I/O error
215
- raise Down::Error, stderr
216
- when 4 # Network failure
217
- raise Down::TimeoutError, stderr if stderr.include?("timed out")
218
- raise Down::ConnectionError, stderr
219
- when 5 # SSL verification failure
220
- raise Down::SSLError, stderr
221
- when 6 # Username/password authentication failure
222
- raise Down::ClientError, stderr
223
- when 7 # Protocol errors
224
- raise Down::Error, stderr
225
- when 8 # Server issued an error response
226
- raise Down::TooManyRedirects, stderr if stderr.include?("redirections exceeded")
227
- raise Down::ResponseError, stderr
228
- end
229
- end
230
- end
231
-
232
- # Adds additional attributes to the Tempfile returned in #download.
233
- module DownloadedFile
234
- attr_accessor :url, :headers
235
-
236
- def original_filename
237
- Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
238
- Utils.filename_from_path(URI.parse(url).path)
239
- end
240
-
241
- def content_type
242
- headers["Content-Type"].to_s.split(";").first
243
- end
244
-
245
- def charset
246
- headers["Content-Type"].to_s[/;\s*charset=([^;]+)/i, 1]
247
- end
248
- end
249
- end
250
- end