down 5.6.2 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,127 @@
1
+ # Generated from lib/down/net_http.rb with RBS::Inline
2
+
3
+ module Down
4
+ # Provides streaming downloads implemented with Net::HTTP and open-uri.
5
+ class NetHttp < Backend
6
+ @options: Hash[Symbol, untyped]
7
+
8
+ URI_NORMALIZER: ^(String) -> String
9
+
10
+ # Content encodings that Net::HTTP transparently decodes.
11
+ DECODED_ENCODINGS: Array[String]
12
+
13
+ DEFAULT_OPTIONS: Hash[Symbol, untyped]
14
+
15
+ # Initializes the backend with common defaults.
16
+ # : (**untyped options) -> void
17
+ def initialize: (**untyped options) -> void
18
+
19
+ # Downloads a remote file to disk using open-uri. Accepts any open-uri
20
+ # options, and a few more.
21
+ # : (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
22
+ # : (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
23
+ def download: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
24
+ | (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
25
+
26
+ # Starts retrieving the remote file using Net::HTTP and returns an IO-like
27
+ # object which downloads the response body on-demand.
28
+ # : (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
29
+ def open: (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
30
+
31
+ private
32
+
33
+ # Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
34
+ # Receives String-keyed header kwargs alongside the options, which RBS keyword types can't express.
35
+ # : (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
36
+ def open_uri: (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
37
+
38
+ # Adds name/value pairs from the given Set-Cookie header values to the
39
+ # Cookie request header, ignoring any cookie attributes.
40
+ # : (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
41
+ def forward_cookies: (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
42
+
43
+ # Parses the Cookie request header value into a hash.
44
+ # : (String? value) -> Hash[String, String?]
45
+ def parse_cookie_header: (String? value) -> Hash[String, String?]
46
+
47
+ # Returns the Cookie request header value from the given headers.
48
+ # : (Hash[untyped, untyped] headers) -> String?
49
+ def cookie_header: (Hash[untyped, untyped] headers) -> String?
50
+
51
+ # Sets the Cookie request header value in the given headers, removing it
52
+ # if the value is nil or empty.
53
+ # : (Hash[Symbol | String, untyped] headers, String? value) -> void
54
+ def set_cookie_header: (Hash[Symbol | String, untyped] headers, String? value) -> void
55
+
56
+ # Returns whether the given header name is the Cookie header.
57
+ # : (untyped name) -> bool
58
+ def cookie_header?: (untyped name) -> bool
59
+
60
+ # Returns whether the given URIs have the same scheme, host and port.
61
+ # : (URI::Generic uri, URI::Generic other_uri) -> bool
62
+ def same_origin?: (URI::Generic uri, URI::Generic other_uri) -> bool
63
+
64
+ # Converts the given IO into a Tempfile if it isn't one already (open-uri
65
+ # returns a StringIO when there is less than 10KB of content), and gives
66
+ # it the specified file extension.
67
+ # : (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
68
+ def ensure_tempfile: (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
69
+
70
+ # Makes a Net::HTTP request and follows redirects.
71
+ # : (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
72
+ def net_http_request: (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
73
+
74
+ # Build a Net::HTTP object for making a request.
75
+ # : (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [String?, String?]?, **untyped) -> [Net::HTTP, Net::HTTP::Get]
76
+ def create_net_http: (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [ String?, String? ]?, **untyped) -> [ Net::HTTP, Net::HTTP::Get ]
77
+
78
+ # Yields chunks of the response body to the block.
79
+ # : (Net::HTTPResponse response) { (String) -> void } -> void
80
+ def stream_body: (Net::HTTPResponse response) { (String) -> void } -> void
81
+
82
+ # Returns the Content-Length, unless a Transfer-Encoding is present or
83
+ # Net::HTTP will decode the body, in which case it doesn't reflect the
84
+ # number of bytes read. Decoding conditions mirror Net::HTTPResponse#inflater.
85
+ # : (Net::HTTPResponse response) -> Integer?
86
+ def content_length: (Net::HTTPResponse response) -> Integer?
87
+
88
+ # Checks that the url is a valid URI and that its scheme is http or https.
89
+ # : (untyped url) -> URI::HTTP
90
+ # : (untyped url, allow_relative: bool) -> URI::Generic
91
+ def ensure_uri: (untyped url) -> URI::HTTP
92
+ | (untyped url, allow_relative: bool) -> URI::Generic
93
+
94
+ # Makes sure that the URL is properly encoded.
95
+ # : (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
96
+ def normalize_uri: (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
97
+
98
+ # When open-uri raises an exception, it doesn't expose the response object.
99
+ # Fortunately, the exception object holds response data that can be used to
100
+ # rebuild the Net::HTTP response object.
101
+ # : (OpenURI::HTTPError exception) -> Net::HTTPResponse
102
+ def rebuild_response_from_open_uri_exception: (OpenURI::HTTPError exception) -> Net::HTTPResponse
103
+
104
+ # Raises non-sucessful response as a Down::ResponseError.
105
+ # : (Net::HTTPResponse response) -> bot
106
+ def response_error!: (Net::HTTPResponse response) -> bot
107
+
108
+ # Re-raise Net::HTTP exceptions as Down::Error exceptions.
109
+ # : (Exception exception) -> bot
110
+ def request_error!: (Exception exception) -> bot
111
+
112
+ # Merge default and ad-hoc options, merging nested headers.
113
+ # : (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
114
+ def merge_options: (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
115
+
116
+ # Defines some additional attributes for the returned Tempfile (on top of what
117
+ # OpenURI::Meta already defines).
118
+ # @rbs module-self OpenURI::Meta
119
+ module DownloadedFile : OpenURI::Meta
120
+ # : () -> String?
121
+ def original_filename: () -> String?
122
+
123
+ # : () -> String?
124
+ def content_type: () -> String?
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,13 @@
1
+ # Generated from lib/down/utils.rb with RBS::Inline
2
+
3
+ module Down
4
+ module Utils
5
+ # Retrieves potential filename from the "Content-Disposition" header.
6
+ # : (_ToS content_disposition) -> String?
7
+ def self?.filename_from_content_disposition: (_ToS content_disposition) -> String?
8
+
9
+ # Retrieves potential filename from the URL path.
10
+ # : (String path) -> String?
11
+ def self?.filename_from_path: (String path) -> String?
12
+ end
13
+ end
@@ -0,0 +1,5 @@
1
+ # Generated from lib/down/version.rb with RBS::Inline
2
+
3
+ module Down
4
+ VERSION: ::String
5
+ end
@@ -0,0 +1,21 @@
1
+ # Generated from lib/down.rb with RBS::Inline
2
+
3
+ module Down
4
+ self.@backend: _Backend
5
+
6
+ @backend: _Backend
7
+
8
+ # : (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
9
+ # : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
10
+ def self?.download: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
11
+ | (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
12
+
13
+ # : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
14
+ def self?.open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
15
+
16
+ # Allows setting a backend via a symbol or a downloader object.
17
+ # : (Symbol | _Backend) -> _Backend
18
+ # : () -> _Backend
19
+ def self?.backend: (Symbol | _Backend) -> _Backend
20
+ | () -> _Backend
21
+ end
data/sig/manifest.yaml ADDED
@@ -0,0 +1,7 @@
1
+ dependencies:
2
+ - name: cgi-escape
3
+ - name: fileutils
4
+ - name: net-http
5
+ - name: open-uri
6
+ - name: tempfile
7
+ - name: uri
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: down
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.6.2
4
+ version: 6.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Janko Marohnić
@@ -37,6 +37,20 @@ dependencies:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
39
  version: '0.3'
40
+ - !ruby/object:Gem::Dependency
41
+ name: tempfile
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: 0.3.1
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: 0.3.1
40
54
  - !ruby/object:Gem::Dependency
41
55
  name: minitest
42
56
  requirement: !ruby/object:Gem::Requirement
@@ -160,7 +174,23 @@ files:
160
174
  - lib/down/net_http.rb
161
175
  - lib/down/utils.rb
162
176
  - lib/down/version.rb
163
- - lib/down/wget.rb
177
+ - sig/_patches/addressable.rbs
178
+ - sig/_patches/http.rbs
179
+ - sig/_patches/io.rbs
180
+ - sig/_patches/net_http.rbs
181
+ - sig/_patches/open_uri.rbs
182
+ - sig/_patches/stringio.rbs
183
+ - sig/down.rbs
184
+ - sig/generated/down.rbs
185
+ - sig/generated/down/backend.rbs
186
+ - sig/generated/down/chunked_io.rbs
187
+ - sig/generated/down/errors.rbs
188
+ - sig/generated/down/http.rbs
189
+ - sig/generated/down/httpx.rbs
190
+ - sig/generated/down/net_http.rbs
191
+ - sig/generated/down/utils.rbs
192
+ - sig/generated/down/version.rbs
193
+ - sig/manifest.yaml
164
194
  homepage: https://github.com/janko/down
165
195
  licenses:
166
196
  - MIT
@@ -179,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
179
209
  - !ruby/object:Gem::Version
180
210
  version: '0'
181
211
  requirements: []
182
- rubygems_version: 4.0.20
212
+ rubygems_version: 4.0.13
183
213
  specification_version: 4
184
214
  summary: Robust streaming downloads using Net::HTTP, http.rb or HTTPX.
185
215
  test_files: []
data/lib/down/wget.rb DELETED
@@ -1,250 +0,0 @@
1
- # frozen-string-literal: true
2
-
3
- warn "The Down::Wget backend has been deprecated and will be removed in the next major version. Please use the Net::HTTP/open-uri, http.rb or HTTPX backend instead."
4
-
5
- begin
6
- require "posix-spawn"
7
- rescue LoadError
8
- require "open3"
9
- end
10
- require "http_parser"
11
-
12
- require "down/backend"
13
-
14
- require "tempfile"
15
- require "uri"
16
-
17
- module Down
18
- # Provides streaming downloads implemented with the wget command-line tool.
19
- # The design is very similar to Down::Http.
20
- class Wget < Backend
21
- # Initializes the backend with common defaults.
22
- def initialize(*arguments)
23
- @arguments = [
24
- user_agent: "Down/#{Down::VERSION}",
25
- max_redirect: 2,
26
- dns_timeout: 30,
27
- connect_timeout: 30,
28
- read_timeout: 30,
29
- ] + arguments
30
- end
31
-
32
- # Downlods the remote file to disk. Accepts wget command-line options and
33
- # some additional options as well.
34
- def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, extension: nil, **options)
35
- io = open(url, *args, **options, rewindable: false)
36
-
37
- content_length_proc.call(io.size) if content_length_proc && io.size
38
-
39
- if max_size && io.size && io.size > max_size
40
- raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
41
- end
42
-
43
- extname = extension ? ".#{extension}" : File.extname(URI(url).path)
44
- tempfile = Tempfile.new(["down-wget", extname], binmode: true)
45
-
46
- until io.eof?
47
- chunk = io.readpartial(nil, buffer ||= String.new)
48
-
49
- tempfile.write(chunk)
50
-
51
- progress_proc.call(tempfile.size) if progress_proc
52
-
53
- if max_size && tempfile.size > max_size
54
- raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
55
- end
56
- end
57
-
58
- tempfile.open # flush written content
59
-
60
- tempfile.extend Down::Wget::DownloadedFile
61
- tempfile.url = url
62
- tempfile.headers = io.data[:headers]
63
-
64
- download_result(tempfile, destination)
65
- rescue
66
- tempfile.close! if tempfile
67
- raise
68
- ensure
69
- io.close if io
70
- end
71
-
72
- # Starts retrieving the remote file and returns an IO-like object which
73
- # downloads the response body on-demand. Accepts wget command-line options.
74
- def open(url, *args, rewindable: true, **options)
75
- arguments = generate_command(url, *args, **options)
76
-
77
- command = Down::Wget::Command.execute(arguments)
78
- # Wrap the wget command output in an IO-like object.
79
- output = Down::ChunkedIO.new(
80
- chunks: command.enum_for(:output),
81
- on_close: command.method(:terminate),
82
- rewindable: false,
83
- )
84
-
85
- # https://github.com/tmm1/http_parser.rb/issues/29#issuecomment-309976363
86
- header_string = output.readpartial
87
- header_string << output.readpartial until header_string.include?("\r\n\r\n")
88
- header_string, first_chunk = header_string.split("\r\n\r\n", 2)
89
-
90
- # Use an HTTP parser to parse out the response headers.
91
- parser = HTTP::Parser.new
92
- parser << header_string
93
-
94
- if parser.headers.nil?
95
- output.close
96
- raise Down::Error, "failed to parse response headers"
97
- end
98
-
99
- headers = normalize_headers(parser.headers)
100
- status = parser.status_code
101
-
102
- content_length = headers["Content-Length"].to_i if headers["Content-Length"]
103
- charset = headers["Content-Type"][/;\s*charset=([^;]+)/i, 1] if headers["Content-Type"]
104
-
105
- # Create an Enumerator which will lazily retrieve chunks of response body.
106
- chunks = Enumerator.new do |yielder|
107
- yielder << first_chunk if first_chunk
108
- yielder << output.readpartial until output.eof?
109
- end
110
-
111
- Down::ChunkedIO.new(
112
- chunks: chunks,
113
- size: content_length,
114
- encoding: charset,
115
- rewindable: rewindable,
116
- on_close: output.method(:close),
117
- data: { status: status, headers: headers },
118
- )
119
- end
120
-
121
- private
122
-
123
- # Generates the wget command.
124
- def generate_command(url, *args, **options)
125
- command = %W[wget --no-verbose --save-headers -O -]
126
-
127
- options = @arguments.grep(Hash).inject({}, :merge).merge(options)
128
- args = @arguments.grep(->(o){!o.is_a?(Hash)}) + args
129
-
130
- (args + options.to_a).each do |option, value|
131
- if option.is_a?(String)
132
- command << option
133
- elsif option.length == 1
134
- command << "-#{option}"
135
- else
136
- command << "--#{option.to_s.gsub("_", "-")}"
137
- end
138
-
139
- command << value.to_s unless value.nil?
140
- end
141
-
142
- command << url
143
- command
144
- end
145
-
146
- # Handles executing the wget command.
147
- class Command
148
- PIPE_BUFFER_SIZE = 64*1024
149
-
150
- def self.execute(arguments)
151
- # posix-spawn gem has better performance, so we use it if it's available
152
- if defined?(POSIX::Spawn)
153
- pid, stdin_pipe, stdout_pipe, stderr_pipe = POSIX::Spawn.popen4(*arguments)
154
- status_reaper = Process.detach(pid)
155
- else
156
- stdin_pipe, stdout_pipe, stderr_pipe, status_reaper = Open3.popen3(*arguments)
157
- end
158
-
159
- stdin_pipe.close
160
- [stdout_pipe, stderr_pipe].each(&:binmode)
161
-
162
- new(stdout_pipe, stderr_pipe, status_reaper)
163
- rescue Errno::ENOENT
164
- raise Down::Error, "wget is not installed"
165
- end
166
-
167
- def initialize(stdout_pipe, stderr_pipe, status_reaper)
168
- @status_reaper = status_reaper
169
- @stdout_pipe = stdout_pipe
170
- @stderr_pipe = stderr_pipe
171
- end
172
-
173
- # Yields chunks of stdout. At the end handles the exit status.
174
- def output
175
- # Keep emptying the stderr buffer, to allow the subprocess to send more
176
- # than 64KB if it wants to.
177
- stderr_reader = Thread.new { @stderr_pipe.read }
178
-
179
- yield @stdout_pipe.readpartial(PIPE_BUFFER_SIZE) until @stdout_pipe.eof?
180
-
181
- status = @status_reaper.value
182
- stderr = stderr_reader.value
183
-
184
- close
185
-
186
- handle_status(status, stderr)
187
- end
188
-
189
- def terminate
190
- begin
191
- Process.kill("TERM", @status_reaper[:pid])
192
- Process.waitpid(@status_reaper[:pid])
193
- rescue Errno::ESRCH
194
- # process has already terminated
195
- end
196
-
197
- close
198
- end
199
-
200
- def close
201
- @stdout_pipe.close unless @stdout_pipe.closed?
202
- @stderr_pipe.close unless @stderr_pipe.closed?
203
- end
204
-
205
- private
206
-
207
- # Translates nonzero wget exit statuses into exceptions.
208
- def handle_status(status, stderr)
209
- case status.exitstatus
210
- when 0 # No problems occurred
211
- # success
212
- when 1, # Generic error code
213
- 2, # Parse error---for instance, when parsing command-line options, the .wgetrc or .netrc...
214
- 3 # File I/O error
215
- raise Down::Error, stderr
216
- when 4 # Network failure
217
- raise Down::TimeoutError, stderr if stderr.include?("timed out")
218
- raise Down::ConnectionError, stderr
219
- when 5 # SSL verification failure
220
- raise Down::SSLError, stderr
221
- when 6 # Username/password authentication failure
222
- raise Down::ClientError, stderr
223
- when 7 # Protocol errors
224
- raise Down::Error, stderr
225
- when 8 # Server issued an error response
226
- raise Down::TooManyRedirects, stderr if stderr.include?("redirections exceeded")
227
- raise Down::ResponseError, stderr
228
- end
229
- end
230
- end
231
-
232
- # Adds additional attributes to the Tempfile returned in #download.
233
- module DownloadedFile
234
- attr_accessor :url, :headers
235
-
236
- def original_filename
237
- Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
238
- Utils.filename_from_path(URI.parse(url).path)
239
- end
240
-
241
- def content_type
242
- headers["Content-Type"].to_s.split(";").first
243
- end
244
-
245
- def charset
246
- headers["Content-Type"].to_s[/;\s*charset=([^;]+)/i, 1]
247
- end
248
- end
249
- end
250
- end