metainspector 2.3.2 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 70997114bb41b0923e33d69ff5f0bfacee1c815d
-  data.tar.gz: 2d5e0c677d79ec0f27d3b964a986c6a90b3223f0
+  metadata.gz: 19cf1cb2c804ede2e36407d38a150840c5f19de6
+  data.tar.gz: 6e3cc4349b842ee52e5d4f9b993603662a573037
 SHA512:
-  metadata.gz: 6c5564c900e6a5716e87edd1c2f0d1b077ff197410691de0bb90a9315c5d40c8a34d912d40babdeb0c556b830b36f62f4319d2758b8a95618f62a6e795b17077
-  data.tar.gz: c5bbc6e6587f9ce53c87e9cba683be578d3ff416f10ae260da09946c9c7b683c3254040b72fcc6557bc31bac468efdb3a770f614fa1dfd327cc773d21590a80e
+  metadata.gz: f77d946f82fd31d6683c512cef527370e99b7edef0a84c355d25e0ce2c73c2c12de642438c9f4c28a80b8e82b73d9974e2f52a8b0eab2e0cfc1ba900aa7148d2
+  data.tar.gz: cfa3f9e830fb56a8e3202c26827158d8504ca0ef0b1787953599b282b9d5005aa47275979e7d212a6f8dbed5a56aafd84ca0fd9025c700e819da8fc3bbbd183f

data/.travis.yml

@@ -1,3 +1,3 @@
 rvm:
-  - 1.9.3
-  - 2.1.2
+- 2.0.0
+- 2.1.3

data/README.md

@@ -8,6 +8,17 @@ You give it an URL, and it lets you easily get its title, links, images, charset
 
 You can try MetaInspector live at this little demo: [https://metainspectordemo.herokuapp.com](https://metainspectordemo.herokuapp.com)
 
+## Changes in 3.0
+
+This latest release introduces some backwards-incompatible changes, so we've decided to do a major version upgrade:
+
+* The redirect API has been changed, now the `:allow_redirections` option will expect only a boolean, which by default is `true`. That is, no more specifying `:safe`, `:unsafe` or `:all`.
+* We've dropped support for Ruby < 2.
+
+Also, we've introduced a new feature:
+
+* Persist cookies across redirects. Now MetaInspector will include the received cookies when following redirects. This fixes some cases where a redirect would fail, sometimes caught in a redirection loop.
+
 ## Installation
 
 Install the gem from RubyGems:
@@ -18,7 +29,7 @@ If you're using it on a Rails application, just add it to your Gemfile and run `
 
     gem 'metainspector'
 
-This gem is tested on Ruby versions 1.9.3 and 2.1.2.
+This gem is tested on Ruby versions 2.0.0 and 2.1.3.
 
 ## Usage
 
@@ -182,15 +193,11 @@ You can set a different timeout with a second parameter, like this:
 
 ### Redirections
 
-By default, redirections from HTTP to HTTPS, and from HTTPS to HTTP are disallowed.
-
-However, you can tell MetaInspector to allow these redirections with the option `:allow_redirections`, like this:
+By default, MetaInspector will follow redirects (up to a limit of 10).
 
-     # This will allow HTTP => HTTPS redirections
-     page = MetaInspector.new('facebook.com', :allow_redirections => :safe)
+If you want to disallow redirects, you can do it like this:
 
-     # And this will allow HTTP => HTTPS ("safe") and HTTPS => HTTP ("unsafe") redirections
-     page = MetaInspector.new('facebook.com', :allow_redirections => :all)
+    page = MetaInspector.new('facebook.com', :allow_redirections => false)
 
 ### Headers
 

data/lib/meta_inspector/document.rb

@@ -11,7 +11,7 @@ module MetaInspector
     # Options:
     # => timeout: defaults to 20 seconds
     # => html_content_type_only: if an exception should be raised if request content-type is not text/html. Defaults to false
-    # => allow_redirections: when :safe, allows HTTP => HTTPS redirections. When :all, it also allows HTTPS => HTTP
+    # => allow_redirections: when true, follow HTTP redirects. Defaults to true
     # => document: the html of the url as a string
     # => warn_level: what to do when encountering exceptions. Can be :warn, :raise or nil
     # => headers: object containing custom headers for the request
@@ -66,7 +66,8 @@ module MetaInspector
       { :timeout => 20,
         :html_content_only => false,
         :warn_level => :raise,
-        :headers => {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"}
+        :headers => {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"},
+        :allow_redirections => true
       }
     end
 

data/lib/meta_inspector/request.rb

@@ -1,7 +1,8 @@
 # -*- encoding: utf-8 -*-
 
-require 'open-uri'
-require 'open_uri_redirections'
+require 'faraday'
+require 'faraday_middleware'
+require 'faraday-cookie_jar'
 require 'timeout'
 
 module MetaInspector
@@ -14,6 +15,7 @@ module MetaInspector
       options = defaults.merge(options)
 
       @url                = initial_url
+      
       @allow_redirections = options[:allow_redirections]
       @timeout            = options[:timeout]
       @exception_log      = options[:exception_log]
@@ -26,36 +28,40 @@ module MetaInspector
     def_delegators :@url, :url
 
     def read
-      response.read if response
+      response.body if response
     end
 
     def content_type
-      response.content_type if response
+      response.headers["content-type"].split(";")[0] if response
     end
 
     private
 
     def response
       Timeout::timeout(@timeout) { @response ||= fetch }
-    rescue TimeoutError, SocketError, RuntimeError => e
+    rescue TimeoutError, Faraday::ConnectionFailed, RuntimeError => e
       @exception_log << e
       nil
     end
 
     def fetch
-      options = {}
-      options.merge!(:allow_redirections => @allow_redirections) if @allow_redirections
-      options.merge!(@headers)                                   if @headers.is_a?(Hash)
-
-      request = open(url, options)
-
-      @url.url = request.base_uri.to_s
-
-      request
+      session = Faraday.new(:url => url) do |faraday|
+        if @allow_redirections
+          faraday.use FaradayMiddleware::FollowRedirects, limit: 10
+          faraday.use :cookie_jar
+        end
+        faraday.headers.merge!(@headers || {})
+        faraday.adapter :net_http
+      end
+      response = session.get
+
+      @url.url = response.env.url.to_s
+
+      response
     end
 
     def defaults
-      { timeout: 20, exception_log: MetaInspector::ExceptionLog.new }
+      { timeout: 20, exception_log: MetaInspector::ExceptionLog.new, allow_redirections: true }
     end
   end
 end

data/lib/meta_inspector/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
 
 module MetaInspector
-  VERSION = "2.3.2"
+  VERSION = "3.0.0"
 end

data/meta_inspector.gemspec

@@ -16,11 +16,14 @@ Gem::Specification.new do |gem|
   gem.version       = MetaInspector::VERSION
 
   gem.add_dependency 'nokogiri', '~> 1.6'
-  gem.add_dependency 'open_uri_redirections', '~> 0.1.4'
+  gem.add_dependency 'faraday'
+  gem.add_dependency 'faraday_middleware'
+  gem.add_dependency 'faraday-cookie_jar'
   gem.add_dependency 'addressable', '~> 2.3.5'
 
   gem.add_development_dependency 'rspec', '2.14.1'
   gem.add_development_dependency 'fakeweb', '1.3.0'
+  gem.add_development_dependency 'webmock'
   gem.add_development_dependency 'awesome_print', '~> 1.2.0'
   gem.add_development_dependency 'rake', '~> 10.1.0'
   gem.add_development_dependency 'pry'

data/spec/document_spec.rb

@@ -93,26 +93,22 @@ describe MetaInspector::Document do
 
   describe 'headers' do
     it "should include default headers" do
-      url     = 'http://example.com/headers'
-      request = double('Request', base_uri: url)
+      url = "http://pagerankalert.com/"
       expected_headers = {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"}
-
-      MetaInspector::Request.any_instance.should_receive(:open)
-                                         .with(url, expected_headers)
-                                         .and_return(request)
-
+                                         
+      headers = {}
+      headers.should_receive(:merge!).with(expected_headers)
+      Faraday::Connection.any_instance.stub(:headers){headers}
       MetaInspector::Document.new(url)
     end
 
     it "should include passed headers on the request" do
-      url     = 'http://example.com/headers'
+      url = "http://pagerankalert.com/"
       headers = {'User-Agent' => 'Mozilla', 'Referer' => 'https://github.com/'}
-      request = double('Request', base_uri: url)
-
-      MetaInspector::Request.any_instance.should_receive(:open)
-                                         .with(url, headers)
-                                         .and_return(request)
 
+      headers = {}
+      headers.should_receive(:merge!).with(headers)
+      Faraday::Connection.any_instance.stub(:headers){headers}
       MetaInspector::Document.new(url, headers: headers)
     end
   end

data/spec/redirections_spec.rb

@@ -6,64 +6,42 @@ describe MetaInspector do
   describe "redirections" do
     let(:logger) { MetaInspector::ExceptionLog.new }
 
-    describe "safe redirections (HTTP to HTTPS)" do
-      it "disallows safe redirections by default" do
-        logger.should receive(:<<).with(an_instance_of(RuntimeError))
-
-        MetaInspector.new("http://facebook.com", exception_log: logger)
-      end
-
-      it "allows safe redirections when :allow_redirections => :safe" do
-        logger.should_not receive(:<<)
-
-        m = MetaInspector.new("http://facebook.com", :allow_redirections => :safe, exception_log: logger)
-
-        m.url.should == "https://www.facebook.com/"
+    context "when redirections are turned off" do
+      it "disallows redirections" do
+        m = MetaInspector.new("http://facebook.com", :allow_redirections => false, exception_log: logger)
+        m.url.should == "http://facebook.com/"
       end
+    end
 
-      it "allows safe redirections when :allow_redirections => :all" do
+    context "when redirections are on (default)" do
+      it "allows follows redirections" do
         logger.should_not receive(:<<)
 
-        m = MetaInspector.new("http://facebook.com", :allow_redirections => :all, exception_log: logger)
+        m = MetaInspector.new("http://facebook.com", exception_log: logger)
 
         m.url.should == "https://www.facebook.com/"
       end
     end
 
-    describe "unsafe redirections (HTTPS to HTTP)" do
-      it "disallows unsafe redirections by default" do
-        logger.should receive(:<<).with(an_instance_of(RuntimeError))
-
-        MetaInspector.new("https://unsafe-facebook.com", exception_log: logger)
-      end
-
-      it "disallows unsafe redirections when :allow_redirections => :safe" do
-        logger.should receive(:<<).with(an_instance_of(RuntimeError))
-
-        MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :safe, exception_log: logger)
-      end
-
-      it "allows unsafe redirections when :allow_redirections => :all" do
+    context "when there are cookies required for proper redirection" do
+      before(:all){WebMock.enable!}
+      after(:all){WebMock.disable!}
+
+      it "allows follows redirections while sending the cookies" do
+        stub_request(:get, "http://blogs.clarionledger.com/dechols/2014/03/24/digital-medicine/").to_return(
+          :status => 302,
+          :headers => { 
+            "Location" => "http://blogs.clarionledger.com/dechols/2014/03/24/digital-medicine/?nclick_check=1",
+            "Set-Cookie" => "EMETA_COOKIE_CHECK=1; path=/; domain=clarionledger.com"
+          })
+        stub_request(:get, "http://blogs.clarionledger.com/dechols/2014/03/24/digital-medicine/?nclick_check=1")
+          .with(:headers => {"Cookie" => "EMETA_COOKIE_CHECK=1"})
         logger.should_not receive(:<<)
 
-        m = MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :all, exception_log: logger)
-
-        m.url.should == "http://unsafe-facebook.com/"
-      end
-    end
-
-    describe "Redirections should update the base_uri" do
-      it "updates the base_uri on safe redirections" do
-        m = MetaInspector.new("http://facebook.com", :allow_redirections => :safe)
+        m = MetaInspector.new("http://blogs.clarionledger.com/dechols/2014/03/24/digital-medicine/", exception_log: logger)
 
-        m.url.should == "https://www.facebook.com/"
-      end
-
-      it "updates the base_uri on all redirections" do
-        m = MetaInspector.new("http://facebook.com", :allow_redirections => :all)
-
-        m.url.should == "https://www.facebook.com/"
-      end
+        m.url.should == "http://blogs.clarionledger.com/dechols/2014/03/24/digital-medicine/?nclick_check=1"
+      end      
     end
   end
 end

data/spec/request_spec.rb

@@ -45,7 +45,7 @@ describe MetaInspector::Request do
 
     it "should handle socket errors" do
       TCPSocket.stub(:open).and_raise(SocketError)
-      logger.should receive(:<<).with(an_instance_of(SocketError))
+      logger.should receive(:<<).with(an_instance_of(Faraday::ConnectionFailed))
 
       MetaInspector::Request.new(url('http://caca232dsdsaer3sdsd-asd343.org'), exception_log: logger)
     end

data/spec/spec_helper.rb

@@ -3,9 +3,11 @@
 $: << File.join(File.dirname(__FILE__), "/../lib")
 require 'meta_inspector'
 require 'fakeweb'
+require "webmock/rspec"
 require "pry"
 
 FakeWeb.allow_net_connect = false
+WebMock.disable!
 
 def fixture_file(filename)
   return '' if filename == ''

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 3.0.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-05 00:00:00.000000000 Z
+date: 2014-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -25,19 +25,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: open_uri_redirections
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.4
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.4
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday-cookie_jar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +108,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement