open-uri 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 75aa86e8cda2121671eaf25a7566555fdb79f1d572a102d11b9866c31ba72889
4
+ data.tar.gz: 8c3061b74a9d593c0e2a848a23590a62562971c954fa127d40d5a60e604f9192
5
+ SHA512:
6
+ metadata.gz: b5c7fd8009abcb73e5ccff0cc7ad53fb35c631f00d28868f5a5b11615483e9c81e1b17ca0dcedc2272c62e9e6b3d34a1c3563afc1553279db43de9a948bcafce
7
+ data.tar.gz: 075c510184720259fb6116c428b4bcf33b8eadf3ae3ab7b9b8ac1039fe9dc30a2e3ad93758157bf8496fc3586c26a2f3369cf5ccfd9421e312a2d0fbc984f733
@@ -0,0 +1,24 @@
1
+ name: test
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ build:
7
+ name: build (${{ matrix.ruby }} / ${{ matrix.os }})
8
+ strategy:
9
+ matrix:
10
+ ruby: [ 2.7, 2.6, 2.5, head ]
11
+ os: [ ubuntu-latest, macos-latest ]
12
+ runs-on: ${{ matrix.os }}
13
+ steps:
14
+ - uses: actions/checkout@master
15
+ - name: Set up Ruby
16
+ uses: ruby/setup-ruby@v1
17
+ with:
18
+ ruby-version: ${{ matrix.ruby }}
19
+ - name: Install dependencies
20
+ run: |
21
+ gem install bundler --no-document
22
+ bundle install
23
+ - name: Run test
24
+ run: rake test
@@ -0,0 +1,8 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ gem "rake"
4
+ gem "test-unit"
@@ -0,0 +1,22 @@
1
+ Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved.
2
+
3
+ Redistribution and use in source and binary forms, with or without
4
+ modification, are permitted provided that the following conditions
5
+ are met:
6
+ 1. Redistributions of source code must retain the above copyright
7
+ notice, this list of conditions and the following disclaimer.
8
+ 2. Redistributions in binary form must reproduce the above copyright
9
+ notice, this list of conditions and the following disclaimer in the
10
+ documentation and/or other materials provided with the distribution.
11
+
12
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
13
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
14
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
15
+ ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
16
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
18
+ OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
19
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
20
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
21
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
22
+ SUCH DAMAGE.
@@ -0,0 +1,40 @@
1
+ # OpenURI
2
+
3
+ OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'open-uri'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle install
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install open-uri
20
+
21
+ ## Usage
22
+
23
+ It is possible to open an http, https or ftp URL as though it were a file:
24
+
25
+ ```ruby
26
+ URI.open("http://www.ruby-lang.org/") {|f|
27
+ f.each_line {|line| p line}
28
+ }
29
+ ```
30
+
31
+ ## Development
32
+
33
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
34
+
35
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
36
+
37
+ ## Contributing
38
+
39
+ Bug reports and pull requests are welcome on GitHub at https://github.com/ruby/open-uri.
40
+
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |t|
5
+ t.libs << "test/lib"
6
+ t.ruby_opts << "-rhelper"
7
+ t.test_files = FileList["test/**/test_*.rb"]
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "open/uri"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,806 @@
1
+ # frozen_string_literal: true
2
+ require 'uri'
3
+ require 'stringio'
4
+ require 'time'
5
+
6
+ module URI
7
+ # Allows the opening of various resources including URIs.
8
+ #
9
+ # If the first argument responds to the 'open' method, 'open' is called on
10
+ # it with the rest of the arguments.
11
+ #
12
+ # If the first argument is a string that begins with <code>(protocol)://<code>, it is parsed by
13
+ # URI.parse. If the parsed object responds to the 'open' method,
14
+ # 'open' is called on it with the rest of the arguments.
15
+ #
16
+ # Otherwise, Kernel#open is called.
17
+ #
18
+ # OpenURI::OpenRead#open provides URI::HTTP#open, URI::HTTPS#open and
19
+ # URI::FTP#open, Kernel#open.
20
+ #
21
+ # We can accept URIs and strings that begin with http://, https:// and
22
+ # ftp://. In these cases, the opened file object is extended by OpenURI::Meta.
23
+ def self.open(name, *rest, &block)
24
+ if name.respond_to?(:open)
25
+ name.open(*rest, &block)
26
+ elsif name.respond_to?(:to_str) &&
27
+ %r{\A[A-Za-z][A-Za-z0-9+\-\.]*://} =~ name &&
28
+ (uri = URI.parse(name)).respond_to?(:open)
29
+ uri.open(*rest, &block)
30
+ else
31
+ super
32
+ end
33
+ end
34
+ end
35
+
36
+ # OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
37
+ #
38
+ # == Example
39
+ #
40
+ # It is possible to open an http, https or ftp URL as though it were a file:
41
+ #
42
+ # URI.open("http://www.ruby-lang.org/") {|f|
43
+ # f.each_line {|line| p line}
44
+ # }
45
+ #
46
+ # The opened file has several getter methods for its meta-information, as
47
+ # follows, since it is extended by OpenURI::Meta.
48
+ #
49
+ # URI.open("http://www.ruby-lang.org/en") {|f|
50
+ # f.each_line {|line| p line}
51
+ # p f.base_uri # <URI::HTTP:0x40e6ef2 URL:http://www.ruby-lang.org/en/>
52
+ # p f.content_type # "text/html"
53
+ # p f.charset # "iso-8859-1"
54
+ # p f.content_encoding # []
55
+ # p f.last_modified # Thu Dec 05 02:45:02 UTC 2002
56
+ # }
57
+ #
58
+ # Additional header fields can be specified by an optional hash argument.
59
+ #
60
+ # URI.open("http://www.ruby-lang.org/en/",
61
+ # "User-Agent" => "Ruby/#{RUBY_VERSION}",
62
+ # "From" => "foo@bar.invalid",
63
+ # "Referer" => "http://www.ruby-lang.org/") {|f|
64
+ # # ...
65
+ # }
66
+ #
67
+ # The environment variables such as http_proxy, https_proxy and ftp_proxy
68
+ # are in effect by default. Here we disable proxy:
69
+ #
70
+ # URI.open("http://www.ruby-lang.org/en/", :proxy => nil) {|f|
71
+ # # ...
72
+ # }
73
+ #
74
+ # See OpenURI::OpenRead.open and URI.open for more on available options.
75
+ #
76
+ # URI objects can be opened in a similar way.
77
+ #
78
+ # uri = URI.parse("http://www.ruby-lang.org/en/")
79
+ # uri.open {|f|
80
+ # # ...
81
+ # }
82
+ #
83
+ # URI objects can be read directly. The returned string is also extended by
84
+ # OpenURI::Meta.
85
+ #
86
+ # str = uri.read
87
+ # p str.base_uri
88
+ #
89
+ # Author:: Tanaka Akira <akr@m17n.org>
90
+
91
+ module OpenURI
92
+ Options = {
93
+ :proxy => true,
94
+ :proxy_http_basic_authentication => true,
95
+ :progress_proc => true,
96
+ :content_length_proc => true,
97
+ :http_basic_authentication => true,
98
+ :read_timeout => true,
99
+ :open_timeout => true,
100
+ :ssl_ca_cert => nil,
101
+ :ssl_verify_mode => nil,
102
+ :ftp_active_mode => false,
103
+ :redirect => true,
104
+ :encoding => nil,
105
+ }
106
+
107
+ def OpenURI.check_options(options) # :nodoc:
108
+ options.each {|k, v|
109
+ next unless Symbol === k
110
+ unless Options.include? k
111
+ raise ArgumentError, "unrecognized option: #{k}"
112
+ end
113
+ }
114
+ end
115
+
116
+ def OpenURI.scan_open_optional_arguments(*rest) # :nodoc:
117
+ if !rest.empty? && (String === rest.first || Integer === rest.first)
118
+ mode = rest.shift
119
+ if !rest.empty? && Integer === rest.first
120
+ perm = rest.shift
121
+ end
122
+ end
123
+ return mode, perm, rest
124
+ end
125
+
126
+ def OpenURI.open_uri(name, *rest) # :nodoc:
127
+ uri = URI::Generic === name ? name : URI.parse(name)
128
+ mode, _, rest = OpenURI.scan_open_optional_arguments(*rest)
129
+ options = rest.shift if !rest.empty? && Hash === rest.first
130
+ raise ArgumentError.new("extra arguments") if !rest.empty?
131
+ options ||= {}
132
+ OpenURI.check_options(options)
133
+
134
+ if /\Arb?(?:\Z|:([^:]+))/ =~ mode
135
+ encoding, = $1,Encoding.find($1) if $1
136
+ mode = nil
137
+ end
138
+ if options.has_key? :encoding
139
+ if !encoding.nil?
140
+ raise ArgumentError, "encoding specified twice"
141
+ end
142
+ encoding = Encoding.find(options[:encoding])
143
+ end
144
+
145
+ unless mode == nil ||
146
+ mode == 'r' || mode == 'rb' ||
147
+ mode == File::RDONLY
148
+ raise ArgumentError.new("invalid access mode #{mode} (#{uri.class} resource is read only.)")
149
+ end
150
+
151
+ io = open_loop(uri, options)
152
+ io.set_encoding(encoding) if encoding
153
+ if block_given?
154
+ begin
155
+ yield io
156
+ ensure
157
+ if io.respond_to? :close!
158
+ io.close! # Tempfile
159
+ else
160
+ io.close if !io.closed?
161
+ end
162
+ end
163
+ else
164
+ io
165
+ end
166
+ end
167
+
168
+ def OpenURI.open_loop(uri, options) # :nodoc:
169
+ proxy_opts = []
170
+ proxy_opts << :proxy_http_basic_authentication if options.include? :proxy_http_basic_authentication
171
+ proxy_opts << :proxy if options.include? :proxy
172
+ proxy_opts.compact!
173
+ if 1 < proxy_opts.length
174
+ raise ArgumentError, "multiple proxy options specified"
175
+ end
176
+ case proxy_opts.first
177
+ when :proxy_http_basic_authentication
178
+ opt_proxy, proxy_user, proxy_pass = options.fetch(:proxy_http_basic_authentication)
179
+ proxy_user = proxy_user.to_str
180
+ proxy_pass = proxy_pass.to_str
181
+ if opt_proxy == true
182
+ raise ArgumentError.new("Invalid authenticated proxy option: #{options[:proxy_http_basic_authentication].inspect}")
183
+ end
184
+ when :proxy
185
+ opt_proxy = options.fetch(:proxy)
186
+ proxy_user = nil
187
+ proxy_pass = nil
188
+ when nil
189
+ opt_proxy = true
190
+ proxy_user = nil
191
+ proxy_pass = nil
192
+ end
193
+ case opt_proxy
194
+ when true
195
+ find_proxy = lambda {|u| pxy = u.find_proxy; pxy ? [pxy, nil, nil] : nil}
196
+ when nil, false
197
+ find_proxy = lambda {|u| nil}
198
+ when String
199
+ opt_proxy = URI.parse(opt_proxy)
200
+ find_proxy = lambda {|u| [opt_proxy, proxy_user, proxy_pass]}
201
+ when URI::Generic
202
+ find_proxy = lambda {|u| [opt_proxy, proxy_user, proxy_pass]}
203
+ else
204
+ raise ArgumentError.new("Invalid proxy option: #{opt_proxy}")
205
+ end
206
+
207
+ uri_set = {}
208
+ buf = nil
209
+ while true
210
+ redirect = catch(:open_uri_redirect) {
211
+ buf = Buffer.new
212
+ uri.buffer_open(buf, find_proxy.call(uri), options)
213
+ nil
214
+ }
215
+ if redirect
216
+ if redirect.relative?
217
+ # Although it violates RFC2616, Location: field may have relative
218
+ # URI. It is converted to absolute URI using uri as a base URI.
219
+ redirect = uri + redirect
220
+ end
221
+ if !options.fetch(:redirect, true)
222
+ raise HTTPRedirect.new(buf.io.status.join(' '), buf.io, redirect)
223
+ end
224
+ unless OpenURI.redirectable?(uri, redirect)
225
+ raise "redirection forbidden: #{uri} -> #{redirect}"
226
+ end
227
+ if options.include? :http_basic_authentication
228
+ # send authentication only for the URI directly specified.
229
+ options = options.dup
230
+ options.delete :http_basic_authentication
231
+ end
232
+ uri = redirect
233
+ raise "HTTP redirection loop: #{uri}" if uri_set.include? uri.to_s
234
+ uri_set[uri.to_s] = true
235
+ else
236
+ break
237
+ end
238
+ end
239
+ io = buf.io
240
+ io.base_uri = uri
241
+ io
242
+ end
243
+
244
+ def OpenURI.redirectable?(uri1, uri2) # :nodoc:
245
+ # This test is intended to forbid a redirection from http://... to
246
+ # file:///etc/passwd, file:///dev/zero, etc. CVE-2011-1521
247
+ # https to http redirect is also forbidden intentionally.
248
+ # It avoids sending secure cookie or referer by non-secure HTTP protocol.
249
+ # (RFC 2109 4.3.1, RFC 2965 3.3, RFC 2616 15.1.3)
250
+ # However this is ad hoc. It should be extensible/configurable.
251
+ uri1.scheme.downcase == uri2.scheme.downcase ||
252
+ (/\A(?:http|ftp)\z/i =~ uri1.scheme && /\A(?:https?|ftp)\z/i =~ uri2.scheme)
253
+ end
254
+
255
+ def OpenURI.open_http(buf, target, proxy, options) # :nodoc:
256
+ if proxy
257
+ proxy_uri, proxy_user, proxy_pass = proxy
258
+ raise "Non-HTTP proxy URI: #{proxy_uri}" if proxy_uri.class != URI::HTTP
259
+ end
260
+
261
+ if target.userinfo
262
+ raise ArgumentError, "userinfo not supported. [RFC3986]"
263
+ end
264
+
265
+ header = {}
266
+ options.each {|k, v| header[k] = v if String === k }
267
+
268
+ require 'net/http'
269
+ klass = Net::HTTP
270
+ if URI::HTTP === target
271
+ # HTTP or HTTPS
272
+ if proxy
273
+ unless proxy_user && proxy_pass
274
+ proxy_user, proxy_pass = proxy_uri.userinfo.split(':') if proxy_uri.userinfo
275
+ end
276
+ if proxy_user && proxy_pass
277
+ klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port, proxy_user, proxy_pass)
278
+ else
279
+ klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port)
280
+ end
281
+ end
282
+ target_host = target.hostname
283
+ target_port = target.port
284
+ request_uri = target.request_uri
285
+ else
286
+ # FTP over HTTP proxy
287
+ target_host = proxy_uri.hostname
288
+ target_port = proxy_uri.port
289
+ request_uri = target.to_s
290
+ if proxy_user && proxy_pass
291
+ header["Proxy-Authorization"] =
292
+ 'Basic ' + ["#{proxy_user}:#{proxy_pass}"].pack('m0')
293
+ end
294
+ end
295
+
296
+ http = proxy ? klass.new(target_host, target_port) : klass.new(target_host, target_port, nil)
297
+ if target.class == URI::HTTPS
298
+ require 'net/https'
299
+ http.use_ssl = true
300
+ http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
301
+ store = OpenSSL::X509::Store.new
302
+ if options[:ssl_ca_cert]
303
+ Array(options[:ssl_ca_cert]).each do |cert|
304
+ if File.directory? cert
305
+ store.add_path cert
306
+ else
307
+ store.add_file cert
308
+ end
309
+ end
310
+ else
311
+ store.set_default_paths
312
+ end
313
+ http.cert_store = store
314
+ end
315
+ if options.include? :read_timeout
316
+ http.read_timeout = options[:read_timeout]
317
+ end
318
+ if options.include? :open_timeout
319
+ http.open_timeout = options[:open_timeout]
320
+ end
321
+
322
+ resp = nil
323
+ http.start {
324
+ req = Net::HTTP::Get.new(request_uri, header)
325
+ if options.include? :http_basic_authentication
326
+ user, pass = options[:http_basic_authentication]
327
+ req.basic_auth user, pass
328
+ end
329
+ http.request(req) {|response|
330
+ resp = response
331
+ if options[:content_length_proc] && Net::HTTPSuccess === resp
332
+ if resp.key?('Content-Length')
333
+ options[:content_length_proc].call(resp['Content-Length'].to_i)
334
+ else
335
+ options[:content_length_proc].call(nil)
336
+ end
337
+ end
338
+ resp.read_body {|str|
339
+ buf << str
340
+ if options[:progress_proc] && Net::HTTPSuccess === resp
341
+ options[:progress_proc].call(buf.size)
342
+ end
343
+ str.clear
344
+ }
345
+ }
346
+ }
347
+ io = buf.io
348
+ io.rewind
349
+ io.status = [resp.code, resp.message]
350
+ resp.each_name {|name| buf.io.meta_add_field2 name, resp.get_fields(name) }
351
+ case resp
352
+ when Net::HTTPSuccess
353
+ when Net::HTTPMovedPermanently, # 301
354
+ Net::HTTPFound, # 302
355
+ Net::HTTPSeeOther, # 303
356
+ Net::HTTPTemporaryRedirect # 307
357
+ begin
358
+ loc_uri = URI.parse(resp['location'])
359
+ rescue URI::InvalidURIError
360
+ raise OpenURI::HTTPError.new(io.status.join(' ') + ' (Invalid Location URI)', io)
361
+ end
362
+ throw :open_uri_redirect, loc_uri
363
+ else
364
+ raise OpenURI::HTTPError.new(io.status.join(' '), io)
365
+ end
366
+ end
367
+
368
+ class HTTPError < StandardError
369
+ def initialize(message, io)
370
+ super(message)
371
+ @io = io
372
+ end
373
+ attr_reader :io
374
+ end
375
+
376
+ # Raised on redirection,
377
+ # only occurs when +redirect+ option for HTTP is +false+.
378
+ class HTTPRedirect < HTTPError
379
+ def initialize(message, io, uri)
380
+ super(message, io)
381
+ @uri = uri
382
+ end
383
+ attr_reader :uri
384
+ end
385
+
386
+ class Buffer # :nodoc: all
387
+ def initialize
388
+ @io = StringIO.new
389
+ @size = 0
390
+ end
391
+ attr_reader :size
392
+
393
+ StringMax = 10240
394
+ def <<(str)
395
+ @io << str
396
+ @size += str.length
397
+ if StringIO === @io && StringMax < @size
398
+ require 'tempfile'
399
+ io = Tempfile.new('open-uri')
400
+ io.binmode
401
+ Meta.init io, @io if Meta === @io
402
+ io << @io.string
403
+ @io = io
404
+ end
405
+ end
406
+
407
+ def io
408
+ Meta.init @io unless Meta === @io
409
+ @io
410
+ end
411
+ end
412
+
413
+ # Mixin for holding meta-information.
414
+ module Meta
415
+ def Meta.init(obj, src=nil) # :nodoc:
416
+ obj.extend Meta
417
+ obj.instance_eval {
418
+ @base_uri = nil
419
+ @meta = {} # name to string. legacy.
420
+ @metas = {} # name to array of strings.
421
+ }
422
+ if src
423
+ obj.status = src.status
424
+ obj.base_uri = src.base_uri
425
+ src.metas.each {|name, values|
426
+ obj.meta_add_field2(name, values)
427
+ }
428
+ end
429
+ end
430
+
431
+ # returns an Array that consists of status code and message.
432
+ attr_accessor :status
433
+
434
+ # returns a URI that is the base of relative URIs in the data.
435
+ # It may differ from the URI supplied by a user due to redirection.
436
+ attr_accessor :base_uri
437
+
438
+ # returns a Hash that represents header fields.
439
+ # The Hash keys are downcased for canonicalization.
440
+ # The Hash values are a field body.
441
+ # If there are multiple field with same field name,
442
+ # the field values are concatenated with a comma.
443
+ attr_reader :meta
444
+
445
+ # returns a Hash that represents header fields.
446
+ # The Hash keys are downcased for canonicalization.
447
+ # The Hash value are an array of field values.
448
+ attr_reader :metas
449
+
450
+ def meta_setup_encoding # :nodoc:
451
+ charset = self.charset
452
+ enc = nil
453
+ if charset
454
+ begin
455
+ enc = Encoding.find(charset)
456
+ rescue ArgumentError
457
+ end
458
+ end
459
+ enc = Encoding::ASCII_8BIT unless enc
460
+ if self.respond_to? :force_encoding
461
+ self.force_encoding(enc)
462
+ elsif self.respond_to? :string
463
+ self.string.force_encoding(enc)
464
+ else # Tempfile
465
+ self.set_encoding enc
466
+ end
467
+ end
468
+
469
+ def meta_add_field2(name, values) # :nodoc:
470
+ name = name.downcase
471
+ @metas[name] = values
472
+ @meta[name] = values.join(', ')
473
+ meta_setup_encoding if name == 'content-type'
474
+ end
475
+
476
+ def meta_add_field(name, value) # :nodoc:
477
+ meta_add_field2(name, [value])
478
+ end
479
+
480
+ # returns a Time that represents the Last-Modified field.
481
+ def last_modified
482
+ if vs = @metas['last-modified']
483
+ v = vs.join(', ')
484
+ Time.httpdate(v)
485
+ else
486
+ nil
487
+ end
488
+ end
489
+
490
+ # :stopdoc:
491
+ RE_LWS = /[\r\n\t ]+/n
492
+ RE_TOKEN = %r{[^\x00- ()<>@,;:\\"/\[\]?={}\x7f]+}n
493
+ RE_QUOTED_STRING = %r{"(?:[\r\n\t !#-\[\]-~\x80-\xff]|\\[\x00-\x7f])*"}n
494
+ RE_PARAMETERS = %r{(?:;#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?=#{RE_LWS}?(?:#{RE_TOKEN}|#{RE_QUOTED_STRING})#{RE_LWS}?)*}n
495
+ # :startdoc:
496
+
497
+ def content_type_parse # :nodoc:
498
+ vs = @metas['content-type']
499
+ # The last (?:;#{RE_LWS}?)? matches extra ";" which violates RFC2045.
500
+ if vs && %r{\A#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?/(#{RE_TOKEN})#{RE_LWS}?(#{RE_PARAMETERS})(?:;#{RE_LWS}?)?\z}no =~ vs.join(', ')
501
+ type = $1.downcase
502
+ subtype = $2.downcase
503
+ parameters = []
504
+ $3.scan(/;#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?=#{RE_LWS}?(?:(#{RE_TOKEN})|(#{RE_QUOTED_STRING}))/no) {|att, val, qval|
505
+ if qval
506
+ val = qval[1...-1].gsub(/[\r\n\t !#-\[\]-~\x80-\xff]+|(\\[\x00-\x7f])/n) { $1 ? $1[1,1] : $& }
507
+ end
508
+ parameters << [att.downcase, val]
509
+ }
510
+ ["#{type}/#{subtype}", *parameters]
511
+ else
512
+ nil
513
+ end
514
+ end
515
+
516
+ # returns "type/subtype" which is MIME Content-Type.
517
+ # It is downcased for canonicalization.
518
+ # Content-Type parameters are stripped.
519
+ def content_type
520
+ type, *_ = content_type_parse
521
+ type || 'application/octet-stream'
522
+ end
523
+
524
+ # returns a charset parameter in Content-Type field.
525
+ # It is downcased for canonicalization.
526
+ #
527
+ # If charset parameter is not given but a block is given,
528
+ # the block is called and its result is returned.
529
+ # It can be used to guess charset.
530
+ #
531
+ # If charset parameter and block is not given,
532
+ # nil is returned except text type.
533
+ # In that case, "utf-8" is returned as defined by RFC6838 4.2.1
534
+ def charset
535
+ type, *parameters = content_type_parse
536
+ if pair = parameters.assoc('charset')
537
+ pair.last.downcase
538
+ elsif block_given?
539
+ yield
540
+ elsif type && %r{\Atext/} =~ type
541
+ "utf-8" # RFC6838 4.2.1
542
+ else
543
+ nil
544
+ end
545
+ end
546
+
547
+ # Returns a list of encodings in Content-Encoding field as an array of
548
+ # strings.
549
+ #
550
+ # The encodings are downcased for canonicalization.
551
+ def content_encoding
552
+ vs = @metas['content-encoding']
553
+ if vs && %r{\A#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?(?:,#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?)*}o =~ (v = vs.join(', '))
554
+ v.scan(RE_TOKEN).map {|content_coding| content_coding.downcase}
555
+ else
556
+ []
557
+ end
558
+ end
559
+ end
560
+
561
+ # Mixin for HTTP and FTP URIs.
562
+ module OpenRead
563
+ # OpenURI::OpenRead#open provides `open' for URI::HTTP and URI::FTP.
564
+ #
565
+ # OpenURI::OpenRead#open takes optional 3 arguments as:
566
+ #
567
+ # OpenURI::OpenRead#open([mode [, perm]] [, options]) [{|io| ... }]
568
+ #
569
+ # OpenURI::OpenRead#open returns an IO-like object if block is not given.
570
+ # Otherwise it yields the IO object and return the value of the block.
571
+ # The IO object is extended with OpenURI::Meta.
572
+ #
573
+ # +mode+ and +perm+ are the same as Kernel#open.
574
+ #
575
+ # However, +mode+ must be read mode because OpenURI::OpenRead#open doesn't
576
+ # support write mode (yet).
577
+ # Also +perm+ is ignored because it is meaningful only for file creation.
578
+ #
579
+ # +options+ must be a hash.
580
+ #
581
+ # Each option with a string key specifies an extra header field for HTTP.
582
+ # I.e., it is ignored for FTP without HTTP proxy.
583
+ #
584
+ # The hash may include other options, where keys are symbols:
585
+ #
586
+ # [:proxy]
587
+ # Synopsis:
588
+ # :proxy => "http://proxy.foo.com:8000/"
589
+ # :proxy => URI.parse("http://proxy.foo.com:8000/")
590
+ # :proxy => true
591
+ # :proxy => false
592
+ # :proxy => nil
593
+ #
594
+ # If :proxy option is specified, the value should be String, URI,
595
+ # boolean or nil.
596
+ #
597
+ # When String or URI is given, it is treated as proxy URI.
598
+ #
599
+ # When true is given or the option itself is not specified,
600
+ # environment variable `scheme_proxy' is examined.
601
+ # `scheme' is replaced by `http', `https' or `ftp'.
602
+ #
603
+ # When false or nil is given, the environment variables are ignored and
604
+ # connection will be made to a server directly.
605
+ #
606
+ # [:proxy_http_basic_authentication]
607
+ # Synopsis:
608
+ # :proxy_http_basic_authentication =>
609
+ # ["http://proxy.foo.com:8000/", "proxy-user", "proxy-password"]
610
+ # :proxy_http_basic_authentication =>
611
+ # [URI.parse("http://proxy.foo.com:8000/"),
612
+ # "proxy-user", "proxy-password"]
613
+ #
614
+ # If :proxy option is specified, the value should be an Array with 3
615
+ # elements. It should contain a proxy URI, a proxy user name and a proxy
616
+ # password. The proxy URI should be a String, an URI or nil. The proxy
617
+ # user name and password should be a String.
618
+ #
619
+ # If nil is given for the proxy URI, this option is just ignored.
620
+ #
621
+ # If :proxy and :proxy_http_basic_authentication is specified,
622
+ # ArgumentError is raised.
623
+ #
624
+ # [:http_basic_authentication]
625
+ # Synopsis:
626
+ # :http_basic_authentication=>[user, password]
627
+ #
628
+ # If :http_basic_authentication is specified,
629
+ # the value should be an array which contains 2 strings:
630
+ # username and password.
631
+ # It is used for HTTP Basic authentication defined by RFC 2617.
632
+ #
633
+ # [:content_length_proc]
634
+ # Synopsis:
635
+ # :content_length_proc => lambda {|content_length| ... }
636
+ #
637
+ # If :content_length_proc option is specified, the option value procedure
638
+ # is called before actual transfer is started.
639
+ # It takes one argument, which is expected content length in bytes.
640
+ #
641
+ # If two or more transfers are performed by HTTP redirection, the
642
+ # procedure is called only once for the last transfer.
643
+ #
644
+ # When expected content length is unknown, the procedure is called with
645
+ # nil. This happens when the HTTP response has no Content-Length header.
646
+ #
647
+ # [:progress_proc]
648
+ # Synopsis:
649
+ # :progress_proc => lambda {|size| ...}
650
+ #
651
+ # If :progress_proc option is specified, the proc is called with one
652
+ # argument each time when `open' gets content fragment from network.
653
+ # The argument +size+ is the accumulated transferred size in bytes.
654
+ #
655
+ # If two or more transfer is done by HTTP redirection, the procedure
656
+ # is called only one for a last transfer.
657
+ #
658
+ # :progress_proc and :content_length_proc are intended to be used for
659
+ # progress bar.
660
+ # For example, it can be implemented as follows using Ruby/ProgressBar.
661
+ #
662
+ # pbar = nil
663
+ # open("http://...",
664
+ # :content_length_proc => lambda {|t|
665
+ # if t && 0 < t
666
+ # pbar = ProgressBar.new("...", t)
667
+ # pbar.file_transfer_mode
668
+ # end
669
+ # },
670
+ # :progress_proc => lambda {|s|
671
+ # pbar.set s if pbar
672
+ # }) {|f| ... }
673
+ #
674
+ # [:read_timeout]
675
+ # Synopsis:
676
+ # :read_timeout=>nil (no timeout)
677
+ # :read_timeout=>10 (10 second)
678
+ #
679
+ # :read_timeout option specifies a timeout of read for http connections.
680
+ #
681
+ # [:open_timeout]
682
+ # Synopsis:
683
+ # :open_timeout=>nil (no timeout)
684
+ # :open_timeout=>10 (10 second)
685
+ #
686
+ # :open_timeout option specifies a timeout of open for http connections.
687
+ #
688
+ # [:ssl_ca_cert]
689
+ # Synopsis:
690
+ # :ssl_ca_cert=>filename or an Array of filenames
691
+ #
692
+ # :ssl_ca_cert is used to specify CA certificate for SSL.
693
+ # If it is given, default certificates are not used.
694
+ #
695
+ # [:ssl_verify_mode]
696
+ # Synopsis:
697
+ # :ssl_verify_mode=>mode
698
+ #
699
+ # :ssl_verify_mode is used to specify openssl verify mode.
700
+ #
701
+ # [:ftp_active_mode]
702
+ # Synopsis:
703
+ # :ftp_active_mode=>bool
704
+ #
705
+ # <tt>:ftp_active_mode => true</tt> is used to make ftp active mode.
706
+ # Ruby 1.9 uses passive mode by default.
707
+ # Note that the active mode is default in Ruby 1.8 or prior.
708
+ #
709
+ # [:redirect]
710
+ # Synopsis:
711
+ # :redirect=>bool
712
+ #
713
+ # +:redirect+ is true by default. <tt>:redirect => false</tt> is used to
714
+ # disable all HTTP redirects.
715
+ #
716
+ # OpenURI::HTTPRedirect exception raised on redirection.
717
+ # Using +true+ also means that redirections between http and ftp are
718
+ # permitted.
719
+ #
720
+ def open(*rest, &block)
721
+ OpenURI.open_uri(self, *rest, &block)
722
+ end
723
+
724
+ # OpenURI::OpenRead#read([ options ]) reads a content referenced by self and
725
+ # returns the content as string.
726
+ # The string is extended with OpenURI::Meta.
727
+ # The argument +options+ is same as OpenURI::OpenRead#open.
728
+ def read(options={})
729
+ self.open(options) {|f|
730
+ str = f.read
731
+ Meta.init str, f
732
+ str
733
+ }
734
+ end
735
+ end
736
+ end
737
+
738
+ module URI
739
+ class HTTP
740
+ def buffer_open(buf, proxy, options) # :nodoc:
741
+ OpenURI.open_http(buf, self, proxy, options)
742
+ end
743
+
744
+ include OpenURI::OpenRead
745
+ end
746
+
747
+ class FTP
748
+ def buffer_open(buf, proxy, options) # :nodoc:
749
+ if proxy
750
+ OpenURI.open_http(buf, self, proxy, options)
751
+ return
752
+ end
753
+ require 'net/ftp'
754
+
755
+ path = self.path
756
+ path = path.sub(%r{\A/}, '%2F') # re-encode the beginning slash because uri library decodes it.
757
+ directories = path.split(%r{/}, -1)
758
+ directories.each {|d|
759
+ d.gsub!(/%([0-9A-Fa-f][0-9A-Fa-f])/) { [$1].pack("H2") }
760
+ }
761
+ unless filename = directories.pop
762
+ raise ArgumentError, "no filename: #{self.inspect}"
763
+ end
764
+ directories.each {|d|
765
+ if /[\r\n]/ =~ d
766
+ raise ArgumentError, "invalid directory: #{d.inspect}"
767
+ end
768
+ }
769
+ if /[\r\n]/ =~ filename
770
+ raise ArgumentError, "invalid filename: #{filename.inspect}"
771
+ end
772
+ typecode = self.typecode
773
+ if typecode && /\A[aid]\z/ !~ typecode
774
+ raise ArgumentError, "invalid typecode: #{typecode.inspect}"
775
+ end
776
+
777
+ # The access sequence is defined by RFC 1738
778
+ ftp = Net::FTP.new
779
+ ftp.connect(self.hostname, self.port)
780
+ ftp.passive = !options[:ftp_active_mode]
781
+ # todo: extract user/passwd from .netrc.
782
+ user = 'anonymous'
783
+ passwd = nil
784
+ user, passwd = self.userinfo.split(/:/) if self.userinfo
785
+ ftp.login(user, passwd)
786
+ directories.each {|cwd|
787
+ ftp.voidcmd("CWD #{cwd}")
788
+ }
789
+ if typecode
790
+ # xxx: typecode D is not handled.
791
+ ftp.voidcmd("TYPE #{typecode.upcase}")
792
+ end
793
+ if options[:content_length_proc]
794
+ options[:content_length_proc].call(ftp.size(filename))
795
+ end
796
+ ftp.retrbinary("RETR #{filename}", 4096) { |str|
797
+ buf << str
798
+ options[:progress_proc].call(buf.size) if options[:progress_proc]
799
+ }
800
+ ftp.close
801
+ buf.io.rewind
802
+ end
803
+
804
+ include OpenURI::OpenRead
805
+ end
806
+ end
@@ -0,0 +1,22 @@
1
+ Gem::Specification.new do |spec|
2
+ spec.name = "open-uri"
3
+ spec.version = "0.1.0"
4
+ spec.authors = ["Tanaka Akira"]
5
+ spec.email = ["akr@fsij.org"]
6
+
7
+ spec.summary = %q{An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.}
8
+ spec.description = %q{An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.}
9
+ spec.homepage = "https://github.com/ruby/open-uri"
10
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
11
+ spec.licenses = ["Ruby", "BSD-2-Clause"]
12
+
13
+ spec.metadata["homepage_uri"] = spec.homepage
14
+ spec.metadata["source_code_uri"] = spec.homepage
15
+
16
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
17
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ end
19
+ spec.bindir = "exe"
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ["lib"]
22
+ end
metadata ADDED
@@ -0,0 +1,56 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: open-uri
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Tanaka Akira
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-09-18 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
14
+ email:
15
+ - akr@fsij.org
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - ".github/workflows/test.yml"
21
+ - ".gitignore"
22
+ - Gemfile
23
+ - LICENSE.txt
24
+ - README.md
25
+ - Rakefile
26
+ - bin/console
27
+ - bin/setup
28
+ - lib/open-uri.rb
29
+ - open-uri.gemspec
30
+ homepage: https://github.com/ruby/open-uri
31
+ licenses:
32
+ - Ruby
33
+ - BSD-2-Clause
34
+ metadata:
35
+ homepage_uri: https://github.com/ruby/open-uri
36
+ source_code_uri: https://github.com/ruby/open-uri
37
+ post_install_message:
38
+ rdoc_options: []
39
+ require_paths:
40
+ - lib
41
+ required_ruby_version: !ruby/object:Gem::Requirement
42
+ requirements:
43
+ - - ">="
44
+ - !ruby/object:Gem::Version
45
+ version: 2.3.0
46
+ required_rubygems_version: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ - - ">="
49
+ - !ruby/object:Gem::Version
50
+ version: '0'
51
+ requirements: []
52
+ rubygems_version: 3.2.0.rc.1
53
+ signing_key:
54
+ specification_version: 4
55
+ summary: An easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
56
+ test_files: []