metainspector 5.18.1 → 5.20.0

data/lib/meta_inspector/request.rb

@@ -1,12 +1,5 @@
-require 'http/cookie'
-require 'faraday'
-require 'faraday-cookie_jar'
-require 'faraday-http-cache'
-require 'faraday/encoding'
-require 'faraday/follow_redirects'
-require 'faraday/gzip'
-require 'faraday/retry'
 require 'timeout'
+require_relative 'http_session'
 
 module MetaInspector
 
@@ -15,16 +8,15 @@ module MetaInspector
     def initialize(initial_url, options = {})
       @url                = initial_url
 
-      fail MetaInspector::RequestError.new('URL must be HTTP') unless @url.url =~ /http[s]?:\/\//i
+      URLPolicy.validate!(@url.url)
 
       @allow_redirections = options[:allow_redirections]
+      @head_first         = options[:head_first]
+      @allow_non_html_content = options[:allow_non_html_content]
       @connection_timeout = options[:connection_timeout]
       @read_timeout       = options[:read_timeout]
-      @retries            = options[:retries]
       @encoding           = options[:encoding]
-      @headers            = options[:headers]
-      @faraday_options    = options[:faraday_options] || {}
-      @faraday_http_cache = options[:faraday_http_cache]
+      @http_session       = options[:http_session] || HttpSession.new(options)
 
       response            # request early so we can fail early
     end
@@ -42,12 +34,13 @@ module MetaInspector
     end
 
     def content_type
-      return nil if response.headers['content-type'].nil?
-      response.headers['content-type'].split(';')[0] if response
+      response_content_type(response)
     end
 
     def response
       @response ||= fetch
+    rescue URLPolicy::Failure => failure
+      raise failure.original
     rescue Faraday::TimeoutError => e
       raise MetaInspector::TimeoutError.new(e)
     rescue Faraday::ConnectionFailed, Faraday::SSLError, URI::InvalidURIError, Faraday::FollowRedirects::RedirectLimitReached => e
@@ -56,36 +49,23 @@ module MetaInspector
 
     private
 
-    def fetch
-      Timeout::timeout(fatal_timeout) do
-        @faraday_options.merge!(:url => url)
-        follow_redirects_options = @faraday_options.delete(:redirect) || {}
-
-        session = Faraday.new(@faraday_options) do |faraday|
-          faraday.request :retry, max: @retries
-
-          faraday.request :gzip
-
-          if @allow_redirections
-            follow_redirects_options[:limit] ||= 10
-            faraday.use Faraday::FollowRedirects::Middleware, **follow_redirects_options
-            faraday.use :cookie_jar
-          end
+    # Once per process: apps fetch many pages with the same options, and a warning
+    # per request would flood their logs
+    def self.warn_faraday_http_cache_deprecation
+      return if @faraday_http_cache_deprecation_warned
 
-          if @faraday_http_cache.is_a?(Hash)
-            @faraday_http_cache[:serializer] ||= Marshal
-            faraday.use Faraday::HttpCache, **@faraday_http_cache
-          end
+      @faraday_http_cache_deprecation_warned = true
+      warn 'DEPRECATION WARNING: the top-level faraday_http_cache option is deprecated and will be removed ' \
+           'in the next major version. Pass it inside faraday_options instead: ' \
+           'faraday_options: { faraday_http_cache: { store: cache } }'
+    end
 
-          faraday.headers.merge!(@headers || {})
-          faraday.response :encoding
-          faraday.adapter :net_http
-        end
+    def fetch
+      Timeout::timeout(fatal_timeout) do
+        session = @http_session.build(url)
 
-        response = session.get do |req|
-          req.options.timeout      = @connection_timeout
-          req.options.open_timeout = @read_timeout
-        end
+        check_head_content_type(session) if @head_first
+        response = perform_request(session, :get)
 
         if @allow_redirections
           @url.url = response.env.url.to_s
@@ -97,6 +77,27 @@ module MetaInspector
       raise MetaInspector::TimeoutError.new(e)
     end
 
+    def check_head_content_type(session)
+      head_response = perform_request(session, :head)
+      return unless head_response.success?
+
+      type = response_content_type(head_response)
+      if !@allow_non_html_content && !type.nil? && type != 'text/html'
+        fail MetaInspector::NonHtmlError.new "The url provided contains #{type} content instead of text/html content"
+      end
+    end
+
+    def response_content_type(response)
+      response.headers['content-type']&.split(';')&.first
+    end
+
+    def perform_request(session, method)
+      session.public_send(method) do |req|
+        req.options.timeout      = @connection_timeout
+        req.options.open_timeout = @read_timeout
+      end
+    end
+
     # Timeouts when connecting / reading a request are handled by Faraday, but in the
     # case of URLs that respond with streaming, Faraday will never return. In that case,
     # we'll resort to our own timeout

data/lib/meta_inspector/url_policy.rb

@@ -0,0 +1,58 @@
+module MetaInspector
+  # Runs inside redirect handling, before cache access or the network adapter.
+  class URLPolicy < Faraday::Middleware
+    class Failure < StandardError
+      attr_reader :original
+
+      def initialize(original)
+        @original = original
+        super('URL policy failed')
+      end
+    end
+
+    IMAGE_HEADERS = %w[user-agent accept accept-language accept-encoding range].freeze
+
+    def self.validate!(value)
+      uri = URI.parse(value.to_s)
+      unless uri.is_a?(URI::HTTP) && uri.hostname && !uri.hostname.empty?
+        raise MetaInspector::RequestError, 'URL must be HTTP or HTTPS'
+      end
+      uri
+    rescue URI::InvalidURIError => error
+      raise MetaInspector::RequestError.new(error)
+    end
+
+    def initialize(app, filter, image_origin = nil)
+      super(app)
+      @filter = filter
+      @image_origin = image_origin && self.class.validate!(image_origin)
+    end
+
+    def call(env)
+      uri = self.class.validate!(env.url)
+      if @image_origin && origin(uri) != origin(@image_origin)
+        env.request_headers.delete_if { |key, _| !IMAGE_HEADERS.include?(key.downcase) }
+      end
+      check_filter(uri) if @filter
+      @app.call(env)
+    end
+
+    private
+
+    def origin(uri)
+      [uri.scheme.downcase, uri.hostname.downcase, uri.port]
+    end
+
+    def check_filter(uri)
+      # Parsing detaches the strings too: URI#dup alone shares its hostname.
+      begin
+        allowed = @filter.call(URI.parse(uri.to_s).freeze)
+      rescue StandardError => error
+        raise Failure.new(error)
+      end
+      unless allowed
+        raise Failure.new(MetaInspector::RequestError.new('Request blocked by url_filter'))
+      end
+    end
+  end
+end

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = '5.18.1'
+  VERSION = '5.20.0'
 end

data/spec/document_spec.rb

@@ -1,6 +1,18 @@
 require 'spec_helper'
 
 describe MetaInspector::Document do
+  describe 'max_image_downloads option' do
+    [-1, 1.5, '1', true, false].each do |limit|
+      it "should reject #{limit.inspect} before fetching the document" do
+        expect do
+          MetaInspector::Document.new('http://example.com/', max_image_downloads: limit)
+        end.to raise_error(ArgumentError, 'max_image_downloads must be a non-negative Integer or nil')
+
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/')
+      end
+    end
+  end
+
   describe 'passing the contents of the document as html' do
     let(:doc) { MetaInspector::Document.new('http://cnn.com/', :document => "<html><head><title>Hello From Passed Html</title><a href='/hello'>Hello link</a></head><body></body></html>") }
 
@@ -17,6 +29,13 @@ describe MetaInspector::Document do
     expect(MetaInspector::Document.new('http://pagerankalert.com').to_s.class).to eq(String)
   end
 
+  it 'should include a nil language in the hash when the document has no declared language' do
+    stub_request(:get, 'http://example.com/').to_return(body: '<html><body></body></html>', headers: { 'Content-Type' => 'text/html' })
+    doc = MetaInspector::Document.new('http://example.com/')
+
+    expect(doc.to_hash).to include('language' => nil)
+  end
+
   it "should return a Hash with all the values set" do
     doc = MetaInspector::Document.new('http://pagerankalert.com')
     expect(doc.to_hash).to eq({
@@ -24,6 +43,7 @@ describe MetaInspector::Document do
                             "scheme"          => "http",
                             "host"            => "pagerankalert.com",
                             "root_url"        => "http://pagerankalert.com/",
+                            "language"        => "en",
                             "title"           => "PageRankAlert.com :: Track your PageRank changes & receive alerts",
                             "best_title"      => "PageRankAlert.com :: Track your PageRank changes & receive alerts",
                             "author"          => nil,

data/spec/image_download_spec.rb

@@ -0,0 +1,114 @@
+require 'spec_helper'
+require 'socket'
+
+RSpec.describe 'bounded image downloads' do
+  let(:png) { "\x89PNG\r\n\x1a\n".b + [13].pack('N') + 'IHDR' + [37, 29].pack('NN') + "\x08\x02\x00\x00\x00".b }
+
+  it 'inherits proxy and TLS options without modifying caller-owned hashes' do
+    image = 'https://images.example/image.png'
+    stub_request(:get, image).to_return(body: png)
+    ssl = { verify: false }.freeze
+    redirect = { limit: 2 }.freeze
+    options = { proxy: 'http://proxy.example:3128', ssl: ssl, redirect: redirect }.freeze
+    expect(Faraday).to receive(:new).with(hash_including(proxy: options[:proxy], ssl: ssl)).and_call_original
+    page = MetaInspector.new('https://page.example/', document: %(<img src="#{image}">), faraday_options: options)
+    expect(page.images.with_size).to eq([[image, 37, 29]])
+    expect(options).to eq(proxy: 'http://proxy.example:3128', ssl: ssl, redirect: redirect)
+  end
+
+  it 'does not cache streamed image responses' do
+    cache = Class.new(Hash) do
+      def read(key); self[key]; end
+      def write(key, value); self[key] = value; end
+    end.new
+    stub_request(:get, 'https://images.example/image.png').to_return(body: png,
+      headers: { 'Cache-Control' => 'public, max-age=3600' })
+    2.times do
+      page = MetaInspector.new('https://page.example/', document: '<img src="https://images.example/image.png">',
+        faraday_options: { faraday_http_cache: { store: cache } })
+      expect(page.images.with_size.first.last(2)).to eq([37, 29])
+    end
+    expect(cache).to be_empty
+    expect(a_request(:get, 'https://images.example/image.png')).to have_been_made.twice
+  end
+
+  context 'with a real HTTP server' do
+    around do |example|
+      @server = TCPServer.new('127.0.0.1', 0)
+      @url = "http://127.0.0.1:#{@server.addr[1]}/image"
+      # WebMock buffers real responses before yielding to Net::HTTP's block.
+      # Disable its adapter here to exercise actual incremental reads.
+      WebMock.disable!
+      example.run
+    ensure
+      @server&.close
+      @worker&.kill
+      @worker&.join
+      WebMock.enable!
+      WebMock.disable_net_connect!
+    end
+
+    def serve(&block)
+      @worker = Thread.new do
+        socket = @server.accept
+        while (line = socket.gets) && line != "\r\n"; end
+        block.call(socket)
+      rescue Errno::EPIPE, Errno::ECONNRESET, IOError
+        # Reaching the byte/time limit closes the peer connection.
+      ensure
+        socket&.close
+      end
+    end
+
+    it 'bounds binary IO and stops a server that ignores Range before reading its full body' do
+      sent = 0
+      serve do |socket|
+        socket.write("HTTP/1.1 200 OK\r\nContent-Length: 4194304\r\nContent-Type: image/png\r\nConnection: close\r\n\r\n")
+        socket.write(png)
+        1024.times do
+          socket.write('x' * 4096)
+          sent += 4096
+          sleep 0.002
+        end
+      end
+      expect(FastImage).to receive(:size).with(an_instance_of(StringIO)) do |io|
+        expect(io.string.bytesize).to eq(256 * 1024)
+        expect(io.string.encoding).to eq(Encoding::BINARY)
+        [37, 29]
+      end
+      page = MetaInspector.new('http://page.example/', document: %(<img src="#{@url}">))
+      expect(page.images.with_size).to eq([[@url, 37, 29]])
+      expect(sent).to be < 4 * 1024 * 1024
+    end
+
+    it 'also stops oversized redirect bodies before following their location' do
+      sent = 0
+      serve do |socket|
+        socket.write("HTTP/1.1 302 Found\r\nLocation: /next\r\nContent-Length: 4194304\r\nConnection: close\r\n\r\n")
+        1024.times do
+          socket.write('x' * 4096)
+          sent += 4096
+          sleep 0.0005
+        end
+      end
+      page = MetaInspector.new('http://page.example/', document: %(<img src="#{@url}">))
+      expect(page.images.with_size).to eq([[@url, 0, 0]])
+      expect(sent).to be < 4 * 1024 * 1024
+    end
+
+    it 'bounds the entire image request even while response data keeps arriving' do
+      serve do |socket|
+        socket.write("HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\nConnection: close\r\n\r\n")
+        loop do
+          socket.write('x')
+          sleep 0.02
+        end
+      end
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      page = MetaInspector.new('http://page.example/', document: %(<img src="#{@url}">))
+      expect(page.images.with_size).to eq([[@url, 0, 0]])
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+      expect(elapsed).to be < 2.8
+    end
+  end
+end

data/spec/meta_inspector/head_first_spec.rb

@@ -0,0 +1,195 @@
+require 'spec_helper'
+
+describe MetaInspector, 'head_first option' do
+  let(:page_url) { 'http://example.com/head-first' }
+  let(:html) { '<html><head><title>Downloaded page</title></head></html>' }
+  let!(:get_request) do
+    stub_request(:get, page_url).to_return(body: html, headers: { 'Content-Type' => 'text/html' })
+  end
+
+  it 'does not send HEAD by default' do
+    expect(MetaInspector.new(page_url).title).to eq('Downloaded page')
+
+    expect(a_request(:head, page_url)).not_to have_been_made
+    expect(get_request).to have_been_requested.once
+  end
+
+  it 'does not send HEAD when explicitly disabled' do
+    expect(MetaInspector.new(page_url, head_first: false).title).to eq('Downloaded page')
+
+    expect(a_request(:head, page_url)).not_to have_been_made
+  end
+
+  it 'sends HEAD before GET for HTML and exposes the GET response' do
+    requests = []
+    head_request = stub_request(:head, page_url).to_return do
+      requests << :head
+      { headers: { 'Content-Type' => 'text/html; charset=UTF-8', 'X-Response' => 'head' } }
+    end
+    stub_request(:get, page_url).to_return do
+      requests << :get
+      { body: html, headers: { 'Content-Type' => 'text/html', 'X-Response' => 'get' } }
+    end
+
+    page = MetaInspector.new(page_url, head_first: true)
+
+    expect(page.title).to eq('Downloaded page')
+    expect(page.content_type).to eq('text/html')
+    expect(page.response.headers['x-response']).to eq('get')
+    expect(page.to_s).to eq(html)
+    expect(head_request).to have_been_requested.once
+    expect(requests).to eq([:head, :get])
+  end
+
+  it 'rejects non-HTML content without sending GET' do
+    head_request = stub_request(:head, page_url)
+      .to_return(headers: { 'Content-Type' => 'application/pdf; version=1.7' })
+
+    expect do
+      MetaInspector.new(page_url, head_first: true)
+    end.to raise_error(MetaInspector::NonHtmlError,
+      'The url provided contains application/pdf content instead of text/html content')
+
+    expect(head_request).to have_been_requested.once
+    expect(get_request).not_to have_been_requested
+  end
+
+  it 'downloads non-HTML content when explicitly allowed' do
+    head_request = stub_request(:head, page_url).to_return(headers: { 'Content-Type' => 'text/plain' })
+    stub_request(:get, page_url).to_return(body: 'Plain text', headers: { 'Content-Type' => 'text/plain' })
+
+    page = MetaInspector.new(page_url, head_first: true, allow_non_html_content: true)
+
+    expect(page.to_s).to eq('Plain text')
+    expect(page.content_type).to eq('text/plain')
+    expect(head_request).to have_been_requested.once
+    expect(get_request).to have_been_requested.once
+  end
+
+  it 'continues to GET when HEAD has no content type' do
+    head_request = stub_request(:head, page_url).to_return(status: 200)
+
+    expect(MetaInspector.new(page_url, head_first: true).title).to eq('Downloaded page')
+
+    expect(head_request).to have_been_requested.once
+    expect(get_request).to have_been_requested.once
+  end
+
+  [403, 404, 405, 501].each do |status|
+    it "continues to GET when HEAD returns #{status}" do
+      head_request = stub_request(:head, page_url)
+        .to_return(status: status, headers: { 'Content-Type' => 'text/plain' })
+
+      expect(MetaInspector.new(page_url, head_first: true).title).to eq('Downloaded page')
+
+      expect(head_request).to have_been_requested.once
+      expect(get_request).to have_been_requested.once
+    end
+  end
+
+  it 'still rejects a non-HTML GET response when HEAD reports HTML' do
+    head_request = stub_request(:head, page_url).to_return(headers: { 'Content-Type' => 'text/html' })
+    stub_request(:get, page_url).to_return(headers: { 'Content-Type' => 'application/pdf' })
+
+    expect do
+      MetaInspector.new(page_url, head_first: true)
+    end.to raise_error(MetaInspector::NonHtmlError)
+
+    expect(head_request).to have_been_requested.once
+    expect(get_request).to have_been_requested.once
+  end
+
+  it 'does not send requests for a supplied document' do
+    expect(MetaInspector.new(page_url, head_first: true, document: html).title).to eq('Downloaded page')
+
+    expect(a_request(:head, page_url)).not_to have_been_made
+    expect(get_request).not_to have_been_requested
+  end
+
+  it 'sends custom headers on both requests' do
+    headers = { 'User-Agent' => 'Custom agent', 'Authorization' => 'Bearer test-token' }
+    head_request = stub_request(:head, page_url).with(headers: headers)
+      .to_return(headers: { 'Content-Type' => 'text/html' })
+    get_with_headers = stub_request(:get, page_url).with(headers: headers).to_return(body: html)
+
+    expect(MetaInspector.new(page_url, head_first: true, headers: headers).title).to eq('Downloaded page')
+
+    expect(head_request).to have_been_requested.once
+    expect(get_with_headers).to have_been_requested.once
+  end
+
+  it 'follows HEAD redirects to reject non-HTML content before GET' do
+    target = 'http://example.com/large.pdf'
+    stub_request(:head, page_url).to_return(status: 302, headers: { 'Location' => target })
+    target_request = stub_request(:head, target).to_return(headers: { 'Content-Type' => 'application/pdf' })
+
+    expect do
+      MetaInspector.new(page_url, head_first: true)
+    end.to raise_error(MetaInspector::NonHtmlError)
+
+    expect(target_request).to have_been_requested.once
+    expect(a_request(:get, /example.com/)).not_to have_been_made
+  end
+
+  it 'follows GET redirects independently of HEAD and exposes the final GET URL' do
+    head_target = 'http://example.com/head-target'
+    get_target = 'http://example.com/get-target'
+    head_request = stub_request(:head, page_url).to_return(status: 302, headers: { 'Location' => head_target })
+    stub_request(:head, head_target).to_return(headers: { 'Content-Type' => 'text/html' })
+    stub_request(:get, page_url).to_return(status: 302, headers: { 'Location' => get_target })
+    final_request = stub_request(:get, get_target).to_return(body: html, headers: { 'Content-Type' => 'text/html' })
+
+    page = MetaInspector.new(page_url, head_first: true)
+
+    expect(page.title).to eq('Downloaded page')
+    expect(page.url).to eq(get_target)
+    expect(head_request).to have_been_requested.once
+    expect(final_request).to have_been_requested.once
+    expect(a_request(:get, head_target)).not_to have_been_made
+  end
+
+  it 'does not follow HEAD redirects when redirections are disabled' do
+    head_request = stub_request(:head, page_url)
+      .to_return(status: 302, headers: { 'Location' => 'http://example.com/target', 'Content-Type' => 'text/plain' })
+
+    page = MetaInspector.new(page_url, head_first: true, allow_redirections: false)
+
+    expect(page.title).to eq('Downloaded page')
+    expect(page.url).to eq(page_url)
+    expect(head_request).to have_been_requested.once
+    expect(a_request(:head, 'http://example.com/target')).not_to have_been_made
+  end
+
+  it 'keeps cookies from HEAD for the GET request' do
+    head_request = stub_request(:head, page_url)
+      .to_return(headers: { 'Content-Type' => 'text/html', 'Set-Cookie' => 'session=checked; path=/' })
+    get_with_cookie = stub_request(:get, page_url).with(headers: { 'Cookie' => 'session=checked' }).to_return(body: html)
+
+    expect(MetaInspector.new(page_url, head_first: true).title).to eq('Downloaded page')
+
+    expect(head_request).to have_been_requested.once
+    expect(get_with_cookie).to have_been_requested.once
+  end
+
+  it 'wraps HEAD timeouts without sending GET' do
+    head_request = stub_request(:head, page_url).to_raise(Net::ReadTimeout)
+
+    expect do
+      MetaInspector.new(page_url, head_first: true, retries: 0)
+    end.to raise_error(MetaInspector::TimeoutError)
+
+    expect(head_request).to have_been_requested.once
+    expect(get_request).not_to have_been_requested
+  end
+
+  it 'wraps HEAD connection failures without sending GET' do
+    head_request = stub_request(:head, page_url).to_raise(Faraday::ConnectionFailed.new('connection failed'))
+
+    expect do
+      MetaInspector.new(page_url, head_first: true, retries: 0)
+    end.to raise_error(MetaInspector::RequestError)
+
+    expect(head_request).to have_been_requested.once
+    expect(get_request).not_to have_been_requested
+  end
+end