format_parser 1.7.0 → 2.0.0.pre

data/lib/remote_io.rb

@@ -1,14 +1,14 @@
+require 'net/http'
+
 # Acts as a wrapper for turning a given URL into an IO object
-# you can read from and seek in. Uses Faraday under the hood
-# to perform fetches, so if you apply Faraday configuration
-# tweaks using `Faraday.default_connection = ...` these will
-# take effect for these RemoteIO objects as well
+# you can read from and seek in.
 class FormatParser::RemoteIO
   class UpstreamError < StandardError
     # @return Integer
     attr_reader :status_code
+
     def initialize(status_code, message)
-      @status_code = status_code
+      @status_code = Integer(status_code)
       super(message)
     end
   end
@@ -23,13 +23,19 @@ class FormatParser::RemoteIO
   class InvalidRequest < UpstreamError
   end
 
-  # @param uri[URI, String] the remote URL to obtain
+  # Represents a failure where the maximum amount of
+  # redirect requests are exceeded.
+  class RedirectLimitReached < UpstreamError
+    def initialize(uri)
+      super(504, "Too many redirects; last one to: #{uri}")
+    end
+  end
+
+  # @param uri[String, URI::Generic] the remote URL to obtain
   # @param headers[Hash] (optional) the HTTP headers to be used in the HTTP request
   def initialize(uri, headers: {})
-    require 'faraday'
-    require 'faraday_middleware/response/follow_redirects'
     @headers = headers
-    @uri = uri
+    @uri = URI(uri)
     @pos = 0
     @remote_size = false
   end
@@ -63,7 +69,7 @@ class FormatParser::RemoteIO
   # @return [String] the read bytes
   def read(n_bytes)
     http_range = (@pos..(@pos + n_bytes - 1))
-    maybe_size, maybe_body = Measurometer.instrument('format_parser.RemoteIO.read') { request_range(http_range) }
+    maybe_size, maybe_body = Measurometer.instrument('format_parser.remote_io.read') { request_range(http_range) }
     if maybe_size && maybe_body
       @remote_size = maybe_size
       @pos += maybe_body.bytesize
@@ -73,23 +79,39 @@ class FormatParser::RemoteIO
 
   protected
 
+  REDIRECT_LIMIT = 3
+  UNSAFE_URI_CHARS = %r{[^\-_.!~*'()a-zA-Z\d;/?:@&=+$,\[\]%]}
+
+  # Generate the URI to fetch from following a redirect response.
+  #
+  # @param location[String] The new URI reference, as provided by the Location header of the previous response.
+  # @param previous_uri[URI] The URI used in the previous request.
+  def redirect_uri(location, previous_uri)
+    # Escape unsafe characters in location. Use location as new URI if absolute, otherwise use it to replace the path of
+    # the previous URI.
+    new_uri = previous_uri.merge(location.to_s.gsub(UNSAFE_URI_CHARS) do |unsafe_char|
+      "%#{unsafe_char.unpack('H2' * unsafe_char.bytesize).join('%').upcase}"
+    end)
+    # Keep previous URI's fragment if not present in location (https://www.rfc-editor.org/rfc/rfc9110.html#section-10.2.2-5)
+    new_uri.fragment = previous_uri.fragment unless new_uri.fragment
+    new_uri
+  end
+
   # Only used internally when reading the remote file
   #
-  # @param range[Range] the HTTP range of data to fetch from remote
-  # @return [String] the response body of the ranged request
-  def request_range(range)
+  # @param range[Range] The HTTP range of data to fetch from remote
+  # @param uri[URI] The URI to fetch from
+  # @param redirects[Integer] The amount of remaining permitted redirects
+  # @return [[Integer, String]] The response body of the ranged request
+  def request_range(range, uri = @uri, redirects = REDIRECT_LIMIT)
     # We use a GET and not a HEAD request followed by a GET because
     # S3 does not allow HEAD requests if you only presigned your URL for GETs, so we
     # combine the first GET of a segment and retrieving the size of the resource
-    conn = Faraday.new(headers: @headers) do |faraday|
-      faraday.use FaradayMiddleware::FollowRedirects
-      # we still need the default adapter, more details: https://blog.thecodewhisperer.com/permalink/losing-time-to-faraday
-      faraday.adapter Faraday.default_adapter
+    response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+      http.request_get(uri, @headers.merge({ 'range' => 'bytes=%d-%d' % [range.begin, range.end] }))
     end
-    response = conn.get(@uri, nil, range: 'bytes=%d-%d' % [range.begin, range.end])
-
-    case response.status
-    when 200
+    case response
+    when Net::HTTPOK
       # S3 returns 200 when you request a Range that is fully satisfied by the entire object,
       # we take that into account here. Also, for very tiny responses (and also for empty responses)
       # the responses are going to be 200 which does not mean we cannot proceed
@@ -100,16 +122,16 @@ class FormatParser::RemoteIO
         error_message = [
           "We requested #{requested_range_size} bytes, but the server sent us more",
           "(#{response_size} bytes) - it likely has no `Range:` support.",
-          "The error occurred when talking to #{@uri})"
+          "The error occurred when talking to #{uri})"
         ]
-        raise InvalidRequest.new(response.status, error_message.join("\n"))
+        raise InvalidRequest.new(response.code, error_message.join("\n"))
       end
       [response_size, response.body]
-    when 206
+    when Net::HTTPPartialContent
       # Figure out of the server supports content ranges, if it doesn't we have no
       # business working with that server
-      range_header = response.headers['Content-Range']
-      raise InvalidRequest.new(response.status, "The server replied with 206 status but no Content-Range at #{@uri}") unless range_header
+      range_header = response['Content-Range']
+      raise InvalidRequest.new(response.code, "The server replied with 206 status but no Content-Range at #{uri}") unless range_header
 
       # "Content-Range: bytes 0-0/307404381" is how the response header is structured
       size = range_header[/\/(\d+)$/, 1].to_i
@@ -117,19 +139,27 @@ class FormatParser::RemoteIO
       # If we request a _larger_ range than what can be satisfied by the server,
       # the response is going to only contain what _can_ be sent and the status is also going
       # to be 206
-      return [size, response.body]
-    when 416
+      [size, response.body]
+    when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther, Net::HTTPTemporaryRedirect, Net::HTTPPermanentRedirect
+      raise RedirectLimitReached(uri) if redirects == 0
+      location = response['location']
+      if location
+        request_range(range, redirect_uri(location, uri), redirects - 1)
+      else
+        raise InvalidRequest.new(response.code, "Server at #{uri} replied with a #{response.code}, indicating redirection; however, the location header was empty.")
+      end
+    when Net::HTTPRangeNotSatisfiable
       # We return `nil` if we tried to read past the end of the IO,
       # which satisfies the Ruby IO convention. The caller should deal with `nil` being the result of a read()
       # S3 will also handily _not_ supply us with the Content-Range of the actual resource, so we
       # cannot hint size with this response - at lease not when working with S3
-      return
-    when 500..599
-      Measurometer.increment_counter('format_parser.RemoteIO.upstream50x_errors', 1)
-      raise IntermittentFailure.new(response.status, "Server at #{@uri} replied with a #{response.status} and we might want to retry")
+      nil
+    when Net::HTTPServerError
+      Measurometer.increment_counter('format_parser.remote_io.upstream50x_errors', 1)
+      raise IntermittentFailure.new(response.code, "Server at #{uri} replied with a #{response.code} and we might want to retry")
     else
-      Measurometer.increment_counter('format_parser.RemoteIO.invalid_request_errors', 1)
-      raise InvalidRequest.new(response.status, "Server at #{@uri} replied with a #{response.status} and refused our request")
+      Measurometer.increment_counter('format_parser.remote_io.invalid_request_errors', 1)
+      raise InvalidRequest.new(response.code, "Server at #{uri} replied with a #{response.code} and refused our request")
     end
   end
 end

data/lib/string.rb

@@ -0,0 +1,9 @@
+class String
+  def underscore
+    gsub(/::/, '/').
+    gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
+    gsub(/([a-z\d])([A-Z])/, '\1_\2').
+    tr('-', '_').
+    downcase
+  end
+end

data/spec/attributes_json_spec.rb

@@ -106,9 +106,6 @@ describe FormatParser::AttributesJSON do
       struct: Struct.new(:key).new('Value'),
       content: "\x01\xFF\xFEb\x00i\x00r\x00d\x00s\x00 \x005\x00 \x00m\x00o\x00r\x00e\x00 \x00c\x00o\x00m\x00p\x00".b
     }
-    expect {
-      JSON.pretty_generate(nasty_hash) # Should not raise an error
-    }.to raise_error(Encoding::UndefinedConversionError)
 
     anon_class = Struct.new(:evil)
     anon_class.include FormatParser::AttributesJSON

data/spec/remote_fetching_spec.rb

@@ -124,14 +124,9 @@ describe 'Fetching data from HTTP remotes' do
   end
 
   it 'sends provided HTTP headers in the request' do
-    # Faraday is required only after calling .parse_http
-    # This line is just to trigger this require, then it's possible to
-    # add an expectation of how Faraday is initialized after.
-    FormatParser.parse_http('invalid_url') rescue nil
-
-    expect(Faraday)
-      .to receive(:new)
-      .with(headers: {'test-header' => 'test-value'})
+    expect_any_instance_of(Net::HTTP)
+      .to receive(:request_get)
+      .with(anything, a_hash_including('test-header' => 'test-value'))
       .and_call_original
 
     file_information = FormatParser.parse_http(

data/spec/remote_io_spec.rb

@@ -4,130 +4,186 @@ describe FormatParser::RemoteIO do
   it_behaves_like 'an IO object compatible with IOConstraint'
 
   it 'returns the partial content when the server supplies a 206 status' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPPartialContent.new('2', '206', 'Partial Content')
+    response['Content-Range'] = '10-109/2577'
+    allow(response).to receive(:body).and_return('Response body')
 
-    fake_resp = double(headers: {'Content-Range' => '10-109/2577'}, status: 206, body: 'This is the response')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=10-109')
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| URI::HTTPS === uri && uri.to_s == url },
+      a_hash_including('range' => 'bytes=10-109')
+    )
+
+    rio = described_class.new(url)
     rio.seek(10)
     read_result = rio.read(100)
-    expect(read_result).to eq('This is the response')
+
+    expect(read_result).to eq(response.body)
   end
 
   it 'returns the entire content when the server supplies the Content-Range response but sends a 200 status' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPOK.new('2', '200', 'OK')
+    allow(response).to receive(:body).and_return('Response body')
+
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
-    fake_resp = double(headers: {'Content-Range' => '10-109/2577'}, status: 200, body: 'This is the response')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=10-109')
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| URI::HTTPS === uri && uri.to_s == url },
+      a_hash_including('range' => 'bytes=10-109')
+    )
 
+    rio = described_class.new(url)
     rio.seek(10)
     read_result = rio.read(100)
-    expect(read_result).to eq('This is the response')
+
+    expect(read_result).to eq(response.body)
   end
 
   it 'raises a specific error for all 4xx responses except 416' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPForbidden.new('2', '403', 'Forbidden')
+
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
-    fake_resp = double(headers: {}, status: 403, body: 'Please log in')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| uri.to_s == url },
+      a_hash_including('range' => 'bytes=100-199')
+    )
 
+    rio = described_class.new(url)
     rio.seek(100)
+
     expect { rio.read(100) }.to raise_error(/replied with a 403 and refused/)
   end
 
   it 'returns nil on a 416 response' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPRangeNotSatisfiable.new('2', '416', 'Range Not Satisfiable')
 
-    fake_resp = double(headers: {}, status: 416, body: 'You stepped off the ledge of the range')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| uri.to_s == url },
+      a_hash_including('range' => 'bytes=100-199')
+    )
+
+    rio = described_class.new(url)
     rio.seek(100)
+
     expect(rio.read(100)).to be_nil
   end
 
   it 'sets the status_code of the exception on a 4xx response from upstream' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPForbidden.new('2', '403', 'Forbidden')
+
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
-    fake_resp = double(headers: {}, status: 403, body: 'Please log in')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| uri.to_s == url },
+      a_hash_including('range' => 'bytes=100-199')
+    )
 
+    rio = described_class.new(url)
     rio.seek(100)
-    # rubocop: disable Lint/AmbiguousBlockAssociation
-    expect { rio.read(100) }.to raise_error { |e| expect(e.status_code).to eq(403) }
+    expect { rio.read(100) }.to(raise_error { |e| expect(e.status_code).to eq(403) })
   end
 
   it 'returns a nil when the range cannot be satisfied and the response is 416' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPRangeNotSatisfiable.new('2', '416', 'Range Not Satisfiable')
 
-    fake_resp = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| uri.to_s == url },
+      a_hash_including('range' => 'bytes=100-199')
+    )
+
+    rio = described_class.new(url)
     rio.seek(100)
+
     expect(rio.read(100)).to be_nil
   end
 
   it 'does not overwrite size when the range cannot be satisfied and the response is 416' do
-    rio = described_class.new('https://images.invalid/img.jpg')
-
-    fake_resp1 = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
-    fake_resp2 = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
-
-    faraday_conn = instance_double(Faraday::Connection)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get)
-      .with('https://images.invalid/img.jpg', nil, range: 'bytes=0-0')
+    url = 'https://images.invalid/img.jpg'
+    response_1 = Net::HTTPPartialContent.new('2', '206', 'Partial Content')
+    response_1['Content-Range'] = 'bytes 0-0/13'
+    allow(response_1).to receive(:body).and_return('Response body')
+    response_2 = Net::HTTPRangeNotSatisfiable.new('2', '416', 'Range Not Satisfiable')
+
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response_1, response_2)
+    allow(Net::HTTP).to receive(:request_get).and_return(response_1, response_2)
+
+    expect(Net::HTTP).to receive(:request_get)
+      .with(
+        an_object_satisfying { |uri| uri.to_s == url },
+        a_hash_including('range' => 'bytes=0-0')
+      )
       .ordered
-      .and_return(fake_resp1)
-    expect(faraday_conn).to receive(:get)
-      .with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
+    expect(Net::HTTP).to receive(:request_get)
+      .with(
+        an_object_satisfying { |uri| uri.to_s == url },
+        a_hash_including('range' => 'bytes=100-199')
+      )
       .ordered
-      .and_return(fake_resp2)
 
+    rio = described_class.new(url)
     rio.read(1)
 
     expect(rio.size).to eq(13)
 
     rio.seek(100)
-    expect(rio.read(100)).to be_nil
 
+    expect(rio.read(100)).to be_nil
     expect(rio.size).to eq(13)
   end
 
   it 'raises a specific error for all 5xx responses' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPBadGateway.new('2', '502', 'Bad Gateway')
+
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
-    fake_resp = double(headers: {}, status: 502, body: 'Guru meditation')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| uri.to_s == url },
+      a_hash_including('range' => 'bytes=100-199')
+    )
 
+    rio = described_class.new(url)
     rio.seek(100)
+
     expect { rio.read(100) }.to raise_error(/replied with a 502 and we might want to retry/)
   end
 
   it 'maintains and exposes #pos' do
-    rio = described_class.new('https://images.invalid/img.jpg')
+    url = 'https://images.invalid/img.jpg'
+    response = Net::HTTPPartialContent.new('2', '206', 'Partial Content')
+    response['Content-Range'] = 'bytes 0-0/13'
+    allow(response).to receive(:body).and_return('a')
 
-    expect(rio.pos).to eq(0)
+    allow(Net::HTTP).to receive(:start).and_yield(Net::HTTP).and_return(response)
+    allow(Net::HTTP).to receive(:request_get).and_return(response)
 
-    fake_resp = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
-    faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
-    allow(Faraday).to receive(:new).and_return(faraday_conn)
-    expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=0-0')
-    rio.read(1)
+    expect(Net::HTTP).to receive(:request_get).with(
+      an_object_satisfying { |uri| uri.to_s == url },
+      a_hash_including('range' => 'bytes=0-0')
+    )
 
+    rio = described_class.new(url)
+    expect(rio.pos).to eq(0)
+    rio.read(1)
     expect(rio.pos).to eq(1)
   end
 end