open-uri 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 75aa86e8cda2121671eaf25a7566555fdb79f1d572a102d11b9866c31ba72889
4
+ data.tar.gz: 8c3061b74a9d593c0e2a848a23590a62562971c954fa127d40d5a60e604f9192
5
+ SHA512:
6
+ metadata.gz: b5c7fd8009abcb73e5ccff0cc7ad53fb35c631f00d28868f5a5b11615483e9c81e1b17ca0dcedc2272c62e9e6b3d34a1c3563afc1553279db43de9a948bcafce
7
+ data.tar.gz: 075c510184720259fb6116c428b4bcf33b8eadf3ae3ab7b9b8ac1039fe9dc30a2e3ad93758157bf8496fc3586c26a2f3369cf5ccfd9421e312a2d0fbc984f733
@@ -0,0 +1,24 @@
1
+ name: test
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ build:
7
+ name: build (${{ matrix.ruby }} / ${{ matrix.os }})
8
+ strategy:
9
+ matrix:
10
+ ruby: [ 2.7, 2.6, 2.5, head ]
11
+ os: [ ubuntu-latest, macos-latest ]
12
+ runs-on: ${{ matrix.os }}
13
+ steps:
14
+ - uses: actions/checkout@master
15
+ - name: Set up Ruby
16
+ uses: ruby/setup-ruby@v1
17
+ with:
18
+ ruby-version: ${{ matrix.ruby }}
19
+ - name: Install dependencies
20
+ run: |
21
+ gem install bundler --no-document
22
+ bundle install
23
+ - name: Run test
24
+ run: rake test
@@ -0,0 +1,8 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ gem "rake"
4
+ gem "test-unit"
@@ -0,0 +1,22 @@
1
+ Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved.
2
+
3
+ Redistribution and use in source and binary forms, with or without
4
+ modification, are permitted provided that the following conditions
5
+ are met:
6
+ 1. Redistributions of source code must retain the above copyright
7
+ notice, this list of conditions and the following disclaimer.
8
+ 2. Redistributions in binary form must reproduce the above copyright
9
+ notice, this list of conditions and the following disclaimer in the
10
+ documentation and/or other materials provided with the distribution.
11
+
12
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
13
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
14
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
15
+ ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
16
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
18
+ OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
19
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
20
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
21
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
22
+ SUCH DAMAGE.
@@ -0,0 +1,40 @@
1
+ # OpenURI
2
+
3
+ OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'open-uri'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle install
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install open-uri
20
+
21
+ ## Usage
22
+
23
+ It is possible to open an http, https or ftp URL as though it were a file:
24
+
25
+ ```ruby
26
+ URI.open("http://www.ruby-lang.org/") {|f|
27
+ f.each_line {|line| p line}
28
+ }
29
+ ```
30
+
31
+ ## Development
32
+
33
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
34
+
35
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
36
+
37
+ ## Contributing
38
+
39
+ Bug reports and pull requests are welcome on GitHub at https://github.com/ruby/open-uri.
40
+
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |t|
5
+ t.libs << "test/lib"
6
+ t.ruby_opts << "-rhelper"
7
+ t.test_files = FileList["test/**/test_*.rb"]
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "open/uri"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,806 @@
1
+ # frozen_string_literal: true
2
+ require 'uri'
3
+ require 'stringio'
4
+ require 'time'
5
+
6
+ module URI
7
+ # Allows the opening of various resources including URIs.
8
+ #
9
+ # If the first argument responds to the 'open' method, 'open' is called on
10
+ # it with the rest of the arguments.
11
+ #
12
+ # If the first argument is a string that begins with <code>(protocol)://<code>, it is parsed by
13
+ # URI.parse. If the parsed object responds to the 'open' method,
14
+ # 'open' is called on it with the rest of the arguments.
15
+ #
16
+ # Otherwise, Kernel#open is called.
17
+ #
18
+ # OpenURI::OpenRead#open provides URI::HTTP#open, URI::HTTPS#open and
19
+ # URI::FTP#open, Kernel#open.
20
+ #
21
+ # We can accept URIs and strings that begin with http://, https:// and
22
+ # ftp://. In these cases, the opened file object is extended by OpenURI::Meta.
23
+ def self.open(name, *rest, &block)
24
+ if name.respond_to?(:open)
25
+ name.open(*rest, &block)
26
+ elsif name.respond_to?(:to_str) &&
27
+ %r{\A[A-Za-z][A-Za-z0-9+\-\.]*://} =~ name &&
28
+ (uri = URI.parse(name)).respond_to?(:open)
29
+ uri.open(*rest, &block)
30
+ else
31
+ super
32
+ end
33
+ end
34
+ end
35
+
36
+ # OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
37
+ #
38
+ # == Example
39
+ #
40
+ # It is possible to open an http, https or ftp URL as though it were a file:
41
+ #
42
+ # URI.open("http://www.ruby-lang.org/") {|f|
43
+ # f.each_line {|line| p line}
44
+ # }
45
+ #
46
+ # The opened file has several getter methods for its meta-information, as
47
+ # follows, since it is extended by OpenURI::Meta.
48
+ #
49
+ # URI.open("http://www.ruby-lang.org/en") {|f|
50
+ # f.each_line {|line| p line}
51
+ # p f.base_uri # <URI::HTTP:0x40e6ef2 URL:http://www.ruby-lang.org/en/>
52
+ # p f.content_type # "text/html"
53
+ # p f.charset # "iso-8859-1"
54
+ # p f.content_encoding # []
55
+ # p f.last_modified # Thu Dec 05 02:45:02 UTC 2002
56
+ # }
57
+ #
58
+ # Additional header fields can be specified by an optional hash argument.
59
+ #
60
+ # URI.open("http://www.ruby-lang.org/en/",
61
+ # "User-Agent" => "Ruby/#{RUBY_VERSION}",
62
+ # "From" => "foo@bar.invalid",
63
+ # "Referer" => "http://www.ruby-lang.org/") {|f|
64
+ # # ...
65
+ # }
66
+ #
67
+ # The environment variables such as http_proxy, https_proxy and ftp_proxy
68
+ # are in effect by default. Here we disable proxy:
69
+ #
70
+ # URI.open("http://www.ruby-lang.org/en/", :proxy => nil) {|f|
71
+ # # ...
72
+ # }
73
+ #
74
+ # See OpenURI::OpenRead.open and URI.open for more on available options.
75
+ #
76
+ # URI objects can be opened in a similar way.
77
+ #
78
+ # uri = URI.parse("http://www.ruby-lang.org/en/")
79
+ # uri.open {|f|
80
+ # # ...
81
+ # }
82
+ #
83
+ # URI objects can be read directly. The returned string is also extended by
84
+ # OpenURI::Meta.
85
+ #
86
+ # str = uri.read
87
+ # p str.base_uri
88
+ #
89
+ # Author:: Tanaka Akira <akr@m17n.org>
90
+
91
+ module OpenURI
92
+ Options = {
93
+ :proxy => true,
94
+ :proxy_http_basic_authentication => true,
95
+ :progress_proc => true,
96
+ :content_length_proc => true,
97
+ :http_basic_authentication => true,
98
+ :read_timeout => true,
99
+ :open_timeout => true,
100
+ :ssl_ca_cert => nil,
101
+ :ssl_verify_mode => nil,
102
+ :ftp_active_mode => false,
103
+ :redirect => true,
104
+ :encoding => nil,
105
+ }
106
+
107
+ def OpenURI.check_options(options) # :nodoc:
108
+ options.each {|k, v|
109
+ next unless Symbol === k
110
+ unless Options.include? k
111
+ raise ArgumentError, "unrecognized option: #{k}"
112
+ end
113
+ }
114
+ end
115
+
116
+ def OpenURI.scan_open_optional_arguments(*rest) # :nodoc:
117
+ if !rest.empty? && (String === rest.first || Integer === rest.first)
118
+ mode = rest.shift
119
+ if !rest.empty? && Integer === rest.first
120
+ perm = rest.shift
121
+ end
122
+ end
123
+ return mode, perm, rest
124
+ end
125
+
126
+ def OpenURI.open_uri(name, *rest) # :nodoc:
127
+ uri = URI::Generic === name ? name : URI.parse(name)
128
+ mode, _, rest = OpenURI.scan_open_optional_arguments(*rest)
129
+ options = rest.shift if !rest.empty? && Hash === rest.first
130
+ raise ArgumentError.new("extra arguments") if !rest.empty?
131
+ options ||= {}
132
+ OpenURI.check_options(options)
133
+
134
+ if /\Arb?(?:\Z|:([^:]+))/ =~ mode
135
+ encoding, = $1,Encoding.find($1) if $1
136
+ mode = nil
137
+ end
138
+ if options.has_key? :encoding
139
+ if !encoding.nil?
140
+ raise ArgumentError, "encoding specified twice"
141
+ end
142
+ encoding = Encoding.find(options[:encoding])
143
+ end
144
+
145
+ unless mode == nil ||
146
+ mode == 'r' || mode == 'rb' ||
147
+ mode == File::RDONLY
148
+ raise ArgumentError.new("invalid access mode #{mode} (#{uri.class} resource is read only.)")
149
+ end
150
+
151
+ io = open_loop(uri, options)
152
+ io.set_encoding(encoding) if encoding
153
+ if block_given?
154
+ begin
155
+ yield io
156
+ ensure
157
+ if io.respond_to? :close!
158
+ io.close! # Tempfile
159
+ else
160
+ io.close if !io.closed?
161
+ end
162
+ end
163
+ else
164
+ io
165
+ end
166
+ end
167
+
168
+ def OpenURI.open_loop(uri, options) # :nodoc:
169
+ proxy_opts = []
170
+ proxy_opts << :proxy_http_basic_authentication if options.include? :proxy_http_basic_authentication
171
+ proxy_opts << :proxy if options.include? :proxy
172
+ proxy_opts.compact!
173
+ if 1 < proxy_opts.length
174
+ raise ArgumentError, "multiple proxy options specified"
175
+ end
176
+ case proxy_opts.first
177
+ when :proxy_http_basic_authentication
178
+ opt_proxy, proxy_user, proxy_pass = options.fetch(:proxy_http_basic_authentication)
179
+ proxy_user = proxy_user.to_str
180
+ proxy_pass = proxy_pass.to_str
181
+ if opt_proxy == true
182
+ raise ArgumentError.new("Invalid authenticated proxy option: #{options[:proxy_http_basic_authentication].inspect}")
183
+ end
184
+ when :proxy
185
+ opt_proxy = options.fetch(:proxy)
186
+ proxy_user = nil
187
+ proxy_pass = nil
188
+ when nil
189
+ opt_proxy = true
190
+ proxy_user = nil
191
+ proxy_pass = nil
192
+ end
193
+ case opt_proxy
194
+ when true
195
+ find_proxy = lambda {|u| pxy = u.find_proxy; pxy ? [pxy, nil, nil] : nil}
196
+ when nil, false
197
+ find_proxy = lambda {|u| nil}
198
+ when String
199
+ opt_proxy = URI.parse(opt_proxy)
200
+ find_proxy = lambda {|u| [opt_proxy, proxy_user, proxy_pass]}
201
+ when URI::Generic
202
+ find_proxy = lambda {|u| [opt_proxy, proxy_user, proxy_pass]}
203
+ else
204
+ raise ArgumentError.new("Invalid proxy option: #{opt_proxy}")
205
+ end
206
+
207
+ uri_set = {}
208
+ buf = nil
209
+ while true
210
+ redirect = catch(:open_uri_redirect) {
211
+ buf = Buffer.new
212
+ uri.buffer_open(buf, find_proxy.call(uri), options)
213
+ nil
214
+ }
215
+ if redirect
216
+ if redirect.relative?
217
+ # Although it violates RFC2616, Location: field may have relative
218
+ # URI. It is converted to absolute URI using uri as a base URI.
219
+ redirect = uri + redirect
220
+ end
221
+ if !options.fetch(:redirect, true)
222
+ raise HTTPRedirect.new(buf.io.status.join(' '), buf.io, redirect)
223
+ end
224
+ unless OpenURI.redirectable?(uri, redirect)
225
+ raise "redirection forbidden: #{uri} -> #{redirect}"
226
+ end
227
+ if options.include? :http_basic_authentication
228
+ # send authentication only for the URI directly specified.
229
+ options = options.dup
230
+ options.delete :http_basic_authentication
231
+ end
232
+ uri = redirect
233
+ raise "HTTP redirection loop: #{uri}" if uri_set.include? uri.to_s
234
+ uri_set[uri.to_s] = true
235
+ else
236
+ break
237
+ end
238
+ end
239
+ io = buf.io
240
+ io.base_uri = uri
241
+ io
242
+ end
243
+
244
+ def OpenURI.redirectable?(uri1, uri2) # :nodoc:
245
+ # This test is intended to forbid a redirection from http://... to
246
+ # file:///etc/passwd, file:///dev/zero, etc. CVE-2011-1521
247
+ # https to http redirect is also forbidden intentionally.
248
+ # It avoids sending secure cookie or referer by non-secure HTTP protocol.
249
+ # (RFC 2109 4.3.1, RFC 2965 3.3, RFC 2616 15.1.3)
250
+ # However this is ad hoc. It should be extensible/configurable.
251
+ uri1.scheme.downcase == uri2.scheme.downcase ||
252
+ (/\A(?:http|ftp)\z/i =~ uri1.scheme && /\A(?:https?|ftp)\z/i =~ uri2.scheme)
253
+ end
254
+
255
+ def OpenURI.open_http(buf, target, proxy, options) # :nodoc:
256
+ if proxy
257
+ proxy_uri, proxy_user, proxy_pass = proxy
258
+ raise "Non-HTTP proxy URI: #{proxy_uri}" if proxy_uri.class != URI::HTTP
259
+ end
260
+
261
+ if target.userinfo
262
+ raise ArgumentError, "userinfo not supported. [RFC3986]"
263
+ end
264
+
265
+ header = {}
266
+ options.each {|k, v| header[k] = v if String === k }
267
+
268
+ require 'net/http'
269
+ klass = Net::HTTP
270
+ if URI::HTTP === target
271
+ # HTTP or HTTPS
272
+ if proxy
273
+ unless proxy_user && proxy_pass
274
+ proxy_user, proxy_pass = proxy_uri.userinfo.split(':') if proxy_uri.userinfo
275
+ end
276
+ if proxy_user && proxy_pass
277
+ klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port, proxy_user, proxy_pass)
278
+ else
279
+ klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port)
280
+ end
281
+ end
282
+ target_host = target.hostname
283
+ target_port = target.port
284
+ request_uri = target.request_uri
285
+ else
286
+ # FTP over HTTP proxy
287
+ target_host = proxy_uri.hostname
288
+ target_port = proxy_uri.port
289
+ request_uri = target.to_s
290
+ if proxy_user && proxy_pass
291
+ header["Proxy-Authorization"] =
292
+ 'Basic ' + ["#{proxy_user}:#{proxy_pass}"].pack('m0')
293
+ end
294
+ end
295
+
296
+ http = proxy ? klass.new(target_host, target_port) : klass.new(target_host, target_port, nil)
297
+ if target.class == URI::HTTPS
298
+ require 'net/https'
299
+ http.use_ssl = true
300
+ http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
301
+ store = OpenSSL::X509::Store.new
302
+ if options[:ssl_ca_cert]
303
+ Array(options[:ssl_ca_cert]).each do |cert|
304
+ if File.directory? cert
305
+ store.add_path cert
306
+ else
307
+ store.add_file cert
308
+ end
309
+ end
310
+ else
311
+ store.set_default_paths
312
+ end
313
+ http.cert_store = store
314
+ end
315
+ if options.include? :read_timeout
316
+ http.read_timeout = options[:read_timeout]
317
+ end
318
+ if options.include? :open_timeout
319
+ http.open_timeout = options[:open_timeout]
320
+ end
321
+
322
+ resp = nil
323
+ http.start {
324
+ req = Net::HTTP::Get.new(request_uri, header)
325
+ if options.include? :http_basic_authentication
326
+ user, pass = options[:http_basic_authentication]
327
+ req.basic_auth user, pass
328
+ end
329
+ http.request(req) {|response|
330
+ resp = response
331
+ if options[:content_length_proc] && Net::HTTPSuccess === resp
332
+ if resp.key?('Content-Length')
333
+ options[:content_length_proc].call(resp['Content-Length'].to_i)
334
+ else
335
+ options[:content_length_proc].call(nil)
336
+ end
337
+ end
338
+ resp.read_body {|str|
339
+ buf << str
340
+ if options[:progress_proc] && Net::HTTPSuccess === resp
341
+ options[:progress_proc].call(buf.size)
342
+ end
343
+ str.clear
344
+ }
345
+ }
346
+ }
347
+ io = buf.io
348
+ io.rewind
349
+ io.status = [resp.code, resp.message]
350
+ resp.each_name {|name| buf.io.meta_add_field2 name, resp.get_fields(name) }
351
+ case resp
352
+ when Net::HTTPSuccess
353
+ when Net::HTTPMovedPermanently, # 301
354
+ Net::HTTPFound, # 302
355
+ Net::HTTPSeeOther, # 303
356
+ Net::HTTPTemporaryRedirect # 307
357
+ begin
358
+ loc_uri = URI.parse(resp['location'])
359
+ rescue URI::InvalidURIError
360
+ raise OpenURI::HTTPError.new(io.status.join(' ') + ' (Invalid Location URI)', io)
361
+ end
362
+ throw :open_uri_redirect, loc_uri
363
+ else
364
+ raise OpenURI::HTTPError.new(io.status.join(' '), io)
365
+ end
366
+ end
367
+
368
+ class HTTPError < StandardError
369
+ def initialize(message, io)
370
+ super(message)
371
+ @io = io
372
+ end
373
+ attr_reader :io
374
+ end
375
+
376
+ # Raised on redirection,
377
+ # only occurs when +redirect+ option for HTTP is +false+.
378
+ class HTTPRedirect < HTTPError
379
+ def initialize(message, io, uri)
380
+ super(message, io)
381
+ @uri = uri
382
+ end
383
+ attr_reader :uri
384
+ end
385
+
386
+ class Buffer # :nodoc: all
387
+ def initialize
388
+ @io = StringIO.new
389
+ @size = 0
390
+ end
391
+ attr_reader :size
392
+
393
+ StringMax = 10240
394
+ def <<(str)
395
+ @io << str
396
+ @size += str.length
397
+ if StringIO === @io && StringMax < @size
398
+ require 'tempfile'
399
+ io = Tempfile.new('open-uri')
400
+ io.binmode
401
+ Meta.init io, @io if Meta === @io
402
+ io << @io.string
403
+ @io = io
404
+ end
405
+ end
406
+
407
+ def io
408
+ Meta.init @io unless Meta === @io
409
+ @io
410
+ end
411
+ end
412
+
413
+ # Mixin for holding meta-information.
414
+ module Meta
415
+ def Meta.init(obj, src=nil) # :nodoc:
416
+ obj.extend Meta
417
+ obj.instance_eval {
418
+ @base_uri = nil
419
+ @meta = {} # name to string. legacy.
420
+ @metas = {} # name to array of strings.
421
+ }
422
+ if src
423
+ obj.status = src.status
424
+ obj.base_uri = src.base_uri
425
+ src.metas.each {|name, values|
426
+ obj.meta_add_field2(name, values)
427
+ }
428
+ end
429
+ end
430
+
431
+ # returns an Array that consists of status code and message.
432
+ attr_accessor :status
433
+
434
+ # returns a URI that is the base of relative URIs in the data.
435
+ # It may differ from the URI supplied by a user due to redirection.
436
+ attr_accessor :base_uri
437
+
438
+ # returns a Hash that represents header fields.
439
+ # The Hash keys are downcased for canonicalization.
440
+ # The Hash values are a field body.
441
+ # If there are multiple field with same field name,
442
+ # the field values are concatenated with a comma.
443
+ attr_reader :meta
444
+
445
+ # returns a Hash that represents header fields.
446
+ # The Hash keys are downcased for canonicalization.
447
+ # The Hash value are an array of field values.
448
+ attr_reader :metas
449
+
450
+ def meta_setup_encoding # :nodoc:
451
+ charset = self.charset
452
+ enc = nil
453
+ if charset
454
+ begin
455
+ enc = Encoding.find(charset)
456
+ rescue ArgumentError
457
+ end
458
+ end
459
+ enc = Encoding::ASCII_8BIT unless enc
460
+ if self.respond_to? :force_encoding
461
+ self.force_encoding(enc)
462
+ elsif self.respond_to? :string
463
+ self.string.force_encoding(enc)
464
+ else # Tempfile
465
+ self.set_encoding enc
466
+ end
467
+ end
468
+
469
+ def meta_add_field2(name, values) # :nodoc:
470
+ name = name.downcase
471
+ @metas[name] = values
472
+ @meta[name] = values.join(', ')
473
+ meta_setup_encoding if name == 'content-type'
474
+ end
475
+
476
+ def meta_add_field(name, value) # :nodoc:
477
+ meta_add_field2(name, [value])
478
+ end
479
+
480
+ # returns a Time that represents the Last-Modified field.
481
+ def last_modified
482
+ if vs = @metas['last-modified']
483
+ v = vs.join(', ')
484
+ Time.httpdate(v)
485
+ else
486
+ nil
487
+ end
488
+ end
489
+
490
+ # :stopdoc:
491
+ RE_LWS = /[\r\n\t ]+/n
492
+ RE_TOKEN = %r{[^\x00- ()<>@,;:\\"/\[\]?={}\x7f]+}n
493
+ RE_QUOTED_STRING = %r{"(?:[\r\n\t !#-\[\]-~\x80-\xff]|\\[\x00-\x7f])*"}n
494
+ RE_PARAMETERS = %r{(?:;#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?=#{RE_LWS}?(?:#{RE_TOKEN}|#{RE_QUOTED_STRING})#{RE_LWS}?)*}n
495
+ # :startdoc:
496
+
497
+ def content_type_parse # :nodoc:
498
+ vs = @metas['content-type']
499
+ # The last (?:;#{RE_LWS}?)? matches extra ";" which violates RFC2045.
500
+ if vs && %r{\A#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?/(#{RE_TOKEN})#{RE_LWS}?(#{RE_PARAMETERS})(?:;#{RE_LWS}?)?\z}no =~ vs.join(', ')
501
+ type = $1.downcase
502
+ subtype = $2.downcase
503
+ parameters = []
504
+ $3.scan(/;#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?=#{RE_LWS}?(?:(#{RE_TOKEN})|(#{RE_QUOTED_STRING}))/no) {|att, val, qval|
505
+ if qval
506
+ val = qval[1...-1].gsub(/[\r\n\t !#-\[\]-~\x80-\xff]+|(\\[\x00-\x7f])/n) { $1 ? $1[1,1] : $& }
507
+ end
508
+ parameters << [att.downcase, val]
509
+ }
510
+ ["#{type}/#{subtype}", *parameters]
511
+ else
512
+ nil
513
+ end
514
+ end
515
+
516
+ # returns "type/subtype" which is MIME Content-Type.
517
+ # It is downcased for canonicalization.
518
+ # Content-Type parameters are stripped.
519
+ def content_type
520
+ type, *_ = content_type_parse
521
+ type || 'application/octet-stream'
522
+ end
523
+
524
+ # returns a charset parameter in Content-Type field.
525
+ # It is downcased for canonicalization.
526
+ #
527
+ # If charset parameter is not given but a block is given,
528
+ # the block is called and its result is returned.
529
+ # It can be used to guess charset.
530
+ #
531
+ # If charset parameter and block is not given,
532
+ # nil is returned except text type.
533
+ # In that case, "utf-8" is returned as defined by RFC6838 4.2.1
534
+ def charset
535
+ type, *parameters = content_type_parse
536
+ if pair = parameters.assoc('charset')
537
+ pair.last.downcase
538
+ elsif block_given?
539
+ yield
540
+ elsif type && %r{\Atext/} =~ type
541
+ "utf-8" # RFC6838 4.2.1
542
+ else
543
+ nil
544
+ end
545
+ end
546
+
547
+ # Returns a list of encodings in Content-Encoding field as an array of
548
+ # strings.
549
+ #
550
+ # The encodings are downcased for canonicalization.
551
+ def content_encoding
552
+ vs = @metas['content-encoding']
553
+ if vs && %r{\A#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?(?:,#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?)*}o =~ (v = vs.join(', '))
554
+ v.scan(RE_TOKEN).map {|content_coding| content_coding.downcase}
555
+ else
556
+ []
557
+ end
558
+ end
559
+ end
560
+
561
+ # Mixin for HTTP and FTP URIs.
562
+ module OpenRead
563
+ # OpenURI::OpenRead#open provides `open' for URI::HTTP and URI::FTP.
564
+ #
565
+ # OpenURI::OpenRead#open takes optional 3 arguments as:
566
+ #
567
+ # OpenURI::OpenRead#open([mode [, perm]] [, options]) [{|io| ... }]
568
+ #
569
+ # OpenURI::OpenRead#open returns an IO-like object if block is not given.
570
+ # Otherwise it yields the IO object and return the value of the block.
571
+ # The IO object is extended with OpenURI::Meta.
572
+ #
573
+ # +mode+ and +perm+ are the same as Kernel#open.
574
+ #
575
+ # However, +mode+ must be read mode because OpenURI::OpenRead#open doesn't
576
+ # support write mode (yet).
577
+ # Also +perm+ is ignored because it is meaningful only for file creation.
578
+ #
579
+ # +options+ must be a hash.
580
+ #
581
+ # Each option with a string key specifies an extra header field for HTTP.
582
+ # I.e., it is ignored for FTP without HTTP proxy.
583
+ #
584
+ # The hash may include other options, where keys are symbols:
585
+ #
586
+ # [:proxy]
587
+ # Synopsis:
588
+ # :proxy => "http://proxy.foo.com:8000/"
589
+ # :proxy => URI.parse("http://proxy.foo.com:8000/")
590
+ # :proxy => true
591
+ # :proxy => false
592
+ # :proxy => nil
593
+ #
594
+ # If :proxy option is specified, the value should be String, URI,
595
+ # boolean or nil.
596
+ #
597
+ # When String or URI is given, it is treated as proxy URI.
598
+ #
599
+ # When true is given or the option itself is not specified,
600
+ # environment variable `scheme_proxy' is examined.
601
+ # `scheme' is replaced by `http', `https' or `ftp'.
602
+ #
603
+ # When false or nil is given, the environment variables are ignored and
604
+ # connection will be made to a server directly.
605
+ #
606
+ # [:proxy_http_basic_authentication]
607
+ # Synopsis:
608
+ # :proxy_http_basic_authentication =>
609
+ # ["http://proxy.foo.com:8000/", "proxy-user", "proxy-password"]
610
+ # :proxy_http_basic_authentication =>
611
+ # [URI.parse("http://proxy.foo.com:8000/"),
612
+ # "proxy-user", "proxy-password"]
613
+ #
614
+ # If :proxy option is specified, the value should be an Array with 3
615
+ # elements. It should contain a proxy URI, a proxy user name and a proxy
616
+ # password. The proxy URI should be a String, an URI or nil. The proxy
617
+ # user name and password should be a String.
618
+ #
619
+ # If nil is given for the proxy URI, this option is just ignored.
620
+ #
621
+ # If :proxy and :proxy_http_basic_authentication is specified,
622
+ # ArgumentError is raised.
623
+ #
624
+ # [:http_basic_authentication]
625
+ # Synopsis:
626
+ # :http_basic_authentication=>[user, password]
627
+ #
628
+ # If :http_basic_authentication is specified,
629
+ # the value should be an array which contains 2 strings:
630
+ # username and password.
631
+ # It is used for HTTP Basic authentication defined by RFC 2617.
632
+ #
633
+ # [:content_length_proc]
634
+ # Synopsis:
635
+ # :content_length_proc => lambda {|content_length| ... }
636
+ #
637
+ # If :content_length_proc option is specified, the option value procedure
638
+ # is called before actual transfer is started.
639
+ # It takes one argument, which is expected content length in bytes.
640
+ #
641
+ # If two or more transfers are performed by HTTP redirection, the
642
+ # procedure is called only once for the last transfer.
643
+ #
644
+ # When expected content length is unknown, the procedure is called with
645
+ # nil. This happens when the HTTP response has no Content-Length header.
646
+ #
647
+ # [:progress_proc]
648
+ # Synopsis:
649
+ # :progress_proc => lambda {|size| ...}
650
+ #
651
+ # If :progress_proc option is specified, the proc is called with one
652
+ # argument each time when `open' gets content fragment from network.
653
+ # The argument +size+ is the accumulated transferred size in bytes.
654
+ #
655
+ # If two or more transfer is done by HTTP redirection, the procedure
656
+ # is called only one for a last transfer.
657
+ #
658
+ # :progress_proc and :content_length_proc are intended to be used for
659
+ # progress bar.
660
+ # For example, it can be implemented as follows using Ruby/ProgressBar.
661
+ #
662
+ # pbar = nil
663
+ # open("http://...",
664
+ # :content_length_proc => lambda {|t|
665
+ # if t && 0 < t
666
+ # pbar = ProgressBar.new("...", t)
667
+ # pbar.file_transfer_mode
668
+ # end
669
+ # },
670
+ # :progress_proc => lambda {|s|
671
+ # pbar.set s if pbar
672
+ # }) {|f| ... }
673
+ #
674
+ # [:read_timeout]
675
+ # Synopsis:
676
+ # :read_timeout=>nil (no timeout)
677
+ # :read_timeout=>10 (10 second)
678
+ #
679
+ # :read_timeout option specifies a timeout of read for http connections.
680
+ #
681
+ # [:open_timeout]
682
+ # Synopsis:
683
+ # :open_timeout=>nil (no timeout)
684
+ # :open_timeout=>10 (10 second)
685
+ #
686
+ # :open_timeout option specifies a timeout of open for http connections.
687
+ #
688
+ # [:ssl_ca_cert]
689
+ # Synopsis:
690
+ # :ssl_ca_cert=>filename or an Array of filenames
691
+ #
692
+ # :ssl_ca_cert is used to specify CA certificate for SSL.
693
+ # If it is given, default certificates are not used.
694
+ #
695
+ # [:ssl_verify_mode]
696
+ # Synopsis:
697
+ # :ssl_verify_mode=>mode
698
+ #
699
+ # :ssl_verify_mode is used to specify openssl verify mode.
700
+ #
701
+ # [:ftp_active_mode]
702
+ # Synopsis:
703
+ # :ftp_active_mode=>bool
704
+ #
705
+ # <tt>:ftp_active_mode => true</tt> is used to make ftp active mode.
706
+ # Ruby 1.9 uses passive mode by default.
707
+ # Note that the active mode is default in Ruby 1.8 or prior.
708
+ #
709
+ # [:redirect]
710
+ # Synopsis:
711
+ # :redirect=>bool
712
+ #
713
+ # +:redirect+ is true by default. <tt>:redirect => false</tt> is used to
714
+ # disable all HTTP redirects.
715
+ #
716
+ # OpenURI::HTTPRedirect exception raised on redirection.
717
+ # Using +true+ also means that redirections between http and ftp are
718
+ # permitted.
719
+ #
720
+ def open(*rest, &block)
721
+ OpenURI.open_uri(self, *rest, &block)
722
+ end
723
+
724
+ # OpenURI::OpenRead#read([ options ]) reads a content referenced by self and
725
+ # returns the content as string.
726
+ # The string is extended with OpenURI::Meta.
727
+ # The argument +options+ is same as OpenURI::OpenRead#open.
728
+ def read(options={})
729
+ self.open(options) {|f|
730
+ str = f.read
731
+ Meta.init str, f
732
+ str
733
+ }
734
+ end
735
+ end
736
+ end
737
+
738
+ module URI
739
+ class HTTP
740
+ def buffer_open(buf, proxy, options) # :nodoc:
741
+ OpenURI.open_http(buf, self, proxy, options)
742
+ end
743
+
744
+ include OpenURI::OpenRead
745
+ end
746
+
747
+ class FTP
748
+ def buffer_open(buf, proxy, options) # :nodoc:
749
+ if proxy
750
+ OpenURI.open_http(buf, self, proxy, options)
751
+ return
752
+ end
753
+ require 'net/ftp'
754
+
755
+ path = self.path
756
+ path = path.sub(%r{\A/}, '%2F') # re-encode the beginning slash because uri library decodes it.
757
+ directories = path.split(%r{/}, -1)
758
+ directories.each {|d|
759
+ d.gsub!(/%([0-9A-Fa-f][0-9A-Fa-f])/) { [$1].pack("H2") }
760
+ }
761
+ unless filename = directories.pop
762
+ raise ArgumentError, "no filename: #{self.inspect}"
763
+ end
764
+ directories.each {|d|
765
+ if /[\r\n]/ =~ d
766
+ raise ArgumentError, "invalid directory: #{d.inspect}"
767
+ end
768
+ }
769
+ if /[\r\n]/ =~ filename
770
+ raise ArgumentError, "invalid filename: #{filename.inspect}"
771
+ end
772
+ typecode = self.typecode
773
+ if typecode && /\A[aid]\z/ !~ typecode
774
+ raise ArgumentError, "invalid typecode: #{typecode.inspect}"
775
+ end
776
+
777
+ # The access sequence is defined by RFC 1738
778
+ ftp = Net::FTP.new
779
+ ftp.connect(self.hostname, self.port)
780
+ ftp.passive = !options[:ftp_active_mode]
781
+ # todo: extract user/passwd from .netrc.
782
+ user = 'anonymous'
783
+ passwd = nil
784
+ user, passwd = self.userinfo.split(/:/) if self.userinfo
785
+ ftp.login(user, passwd)
786
+ directories.each {|cwd|
787
+ ftp.voidcmd("CWD #{cwd}")
788
+ }
789
+ if typecode
790
+ # xxx: typecode D is not handled.
791
+ ftp.voidcmd("TYPE #{typecode.upcase}")
792
+ end
793
+ if options[:content_length_proc]
794
+ options[:content_length_proc].call(ftp.size(filename))
795
+ end
796
+ ftp.retrbinary("RETR #{filename}", 4096) { |str|
797
+ buf << str
798
+ options[:progress_proc].call(buf.size) if options[:progress_proc]
799
+ }
800
+ ftp.close
801
+ buf.io.rewind
802
+ end
803
+
804
+ include OpenURI::OpenRead
805
+ end
806
+ end
@@ -0,0 +1,22 @@
1
+ Gem::Specification.new do |spec|
2
+ spec.name = "open-uri"
3
+ spec.version = "0.1.0"
4
+ spec.authors = ["Tanaka Akira"]
5
+ spec.email = ["akr@fsij.org"]
6
+
7
+ spec.summary = %q{An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.}
8
+ spec.description = %q{An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.}
9
+ spec.homepage = "https://github.com/ruby/open-uri"
10
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
11
+ spec.licenses = ["Ruby", "BSD-2-Clause"]
12
+
13
+ spec.metadata["homepage_uri"] = spec.homepage
14
+ spec.metadata["source_code_uri"] = spec.homepage
15
+
16
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
17
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ end
19
+ spec.bindir = "exe"
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ["lib"]
22
+ end
metadata ADDED
@@ -0,0 +1,56 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: open-uri
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Tanaka Akira
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-09-18 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
14
+ email:
15
+ - akr@fsij.org
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - ".github/workflows/test.yml"
21
+ - ".gitignore"
22
+ - Gemfile
23
+ - LICENSE.txt
24
+ - README.md
25
+ - Rakefile
26
+ - bin/console
27
+ - bin/setup
28
+ - lib/open-uri.rb
29
+ - open-uri.gemspec
30
+ homepage: https://github.com/ruby/open-uri
31
+ licenses:
32
+ - Ruby
33
+ - BSD-2-Clause
34
+ metadata:
35
+ homepage_uri: https://github.com/ruby/open-uri
36
+ source_code_uri: https://github.com/ruby/open-uri
37
+ post_install_message:
38
+ rdoc_options: []
39
+ require_paths:
40
+ - lib
41
+ required_ruby_version: !ruby/object:Gem::Requirement
42
+ requirements:
43
+ - - ">="
44
+ - !ruby/object:Gem::Version
45
+ version: 2.3.0
46
+ required_rubygems_version: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ - - ">="
49
+ - !ruby/object:Gem::Version
50
+ version: '0'
51
+ requirements: []
52
+ rubygems_version: 3.2.0.rc.1
53
+ signing_key:
54
+ specification_version: 4
55
+ summary: An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
56
+ test_files: []