metainspector 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fcb1f8adaeb835639f198ee9ede5f54da02211fd
-  data.tar.gz: 40bb733a1e3f2b48011c535951a2a2ac2bb097dc
+  metadata.gz: 9c230bbbcf325166d2e5906ed7f13af8e0f8ef06
+  data.tar.gz: 245e219fc420a20e8fd8eef722502a6cc23d8087
 SHA512:
-  metadata.gz: 5f502ee4f24fb04b8d0afb8e369a5da2ac7c7c4a4b4c8a743cfbfd24f0f15125d139c5f35bd65c2fe9bce59d4015305c2fdf05f4fe6e696e691e90bb60e445b2
-  data.tar.gz: 2f9252d0283f815b15bc7e4c43b33e588975b63a935354e3e0c34639a0bad1c96d52b63308f9902120829b3747c796fc43f36d95d5ee78237c9dbc4fb64baa80
+  metadata.gz: 791526a75ce5e23651a7cbded8ee4e9ab12fd75281d7089a2f2191ee44293c9f1b8d8587cae3284e2ed1c103d9020cfa2038fd1be4ddc90d2e78d1235fcc0374
+  data.tar.gz: f3c535c4a4b7618168e30cf2427a22365404dc00f698dde32669364a86770bc8445b20989974298afd124822fddea272afd000035663d7e0eb6cd5acb0676006

data/.travis.yml

@@ -2,4 +2,5 @@ rvm:
   - 1.9.2
   - 1.9.3
   - 2.0.0
-  - 2.1.0
+  - 2.1.0
+  - 2.1.1

data/README.md

@@ -191,6 +191,17 @@ However, you can tell MetaInspector to allow these redirections with the option
      # And this will allow HTTP => HTTPS ("safe") and HTTPS => HTTP ("unsafe") redirections
      page = MetaInspector.new('facebook.com', :allow_redirections => :all)
 
+### Headers
+
+By default, the following headers are set:
+
+    {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"}
+
+If you want to set custom headers then use the `headers` option:
+
+     # Set the User-Agent header
+     page = MetaInspector.new('example.com', :headers => {'User-Agent' => 'My custom User-Agent'})
+
 ### HTML Content Only
 
 MetaInspector will try to parse all URLs by default. If you want to raise an exception when trying to parse a non-html URL (one that has a content-type different than text/html), you can state it like this:
@@ -215,7 +226,7 @@ However, if you prefer you can also set the `warn_level: :warn` option, so that
 
 You can also set the `warn_level: :store` option so that exceptions found will be silenced, and left for you to inspect on `page.exceptions`. You can also ask for `page.ok?`, wich will return `true` if no exceptions are stored.
 
-You should avoid using the `:store` option, or use it wisely, as silencing errors can be problematic, it's always better to face the errors and treat them accordingly. 
+You should avoid using the `:store` option, or use it wisely, as silencing errors can be problematic, it's always better to face the errors and treat them accordingly.
 
 ## Examples
 

data/lib/meta_inspector.rb

@@ -7,7 +7,6 @@ require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/reque
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/url'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/parser'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/document'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/deprecations'))
 
 module MetaInspector
   extend self

data/lib/meta_inspector/document.rb

@@ -3,7 +3,7 @@
 module MetaInspector
   # A MetaInspector::Document knows about its URL and its contents
   class Document
-    attr_reader :timeout, :html_content_only, :allow_redirections, :warn_level
+    attr_reader :timeout, :html_content_only, :allow_redirections, :warn_level, :headers
 
     include MetaInspector::Exceptionable
 
@@ -14,25 +14,22 @@ module MetaInspector
     # => allow_redirections: when :safe, allows HTTP => HTTPS redirections. When :all, it also allows HTTPS => HTTP
     # => document: the html of the url as a string
     # => warn_level: what to do when encountering exceptions. Can be :warn, :raise or nil
+    # => headers: object containing custom headers for the request
     def initialize(initial_url, options = {})
       options             = defaults.merge(options)
       @timeout            = options[:timeout]
       @html_content_only  = options[:html_content_only]
       @allow_redirections = options[:allow_redirections]
       @document           = options[:document]
-
-      if options[:verbose] == true
-        warn "The verbose option is deprecated since 1.17, please use warn_level: :warn instead"
-        options[:warn_level] = :warn
-      end
-
-      @warn_level     = options[:warn_level]
-      @exception_log  = options[:exception_log] || MetaInspector::ExceptionLog.new(warn_level: warn_level)
-      @url            = MetaInspector::URL.new(initial_url, exception_log: @exception_log)
-      @request        = MetaInspector::Request.new(@url, allow_redirections: @allow_redirections,
-                                                         timeout:            @timeout,
-                                                         exception_log:      @exception_log) unless @document
-      @parser         = MetaInspector::Parser.new(self,  exception_log:      @exception_log)
+      @headers            = options[:headers]
+      @warn_level         = options[:warn_level]
+      @exception_log      = options[:exception_log] || MetaInspector::ExceptionLog.new(warn_level: warn_level)
+      @url                = MetaInspector::URL.new(initial_url, exception_log: @exception_log)
+      @request            = MetaInspector::Request.new(@url,  allow_redirections: @allow_redirections,
+                                                              timeout:            @timeout,
+                                                              exception_log:      @exception_log,
+                                                              headers:            @headers) unless @document
+      @parser             = MetaInspector::Parser.new(self,  exception_log:      @exception_log)
     end
 
     extend Forwardable
@@ -65,7 +62,11 @@ module MetaInspector
     private
 
     def defaults
-      { :timeout => 20, :html_content_only => false, :warn_level => :raise }
+      { :timeout => 20,
+        :html_content_only => false,
+        :warn_level => :raise,
+        :headers => {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"}
+      }
     end
 
     def document

data/lib/meta_inspector/request.rb

@@ -17,6 +17,7 @@ module MetaInspector
       @allow_redirections = options[:allow_redirections]
       @timeout            = options[:timeout]
       @exception_log      = options[:exception_log]
+      @headers            = options[:headers]
 
       response            # as soon as it is set up, we make the request so we can fail early
     end
@@ -43,7 +44,11 @@ module MetaInspector
     end
 
     def fetch
-      request = open(url, {:allow_redirections => @allow_redirections})
+      options = {}
+      options.merge!(:allow_redirections => @allow_redirections) if @allow_redirections
+      options.merge!(@headers)                                   if @headers.is_a?(Hash)
+
+      request = open(url, options)
 
       @url.url = request.base_uri.to_s
 
@@ -51,7 +56,7 @@ module MetaInspector
     end
 
     def defaults
-      { allow_redirections: false, timeout: 20, exception_log: MetaInspector::ExceptionLog.new }
+      { timeout: 20, exception_log: MetaInspector::ExceptionLog.new }
     end
   end
 end

data/lib/meta_inspector/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
 
 module MetaInspector
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/spec/document_spec.rb

@@ -89,4 +89,30 @@ describe MetaInspector::Document do
       tar_url.title
     end
   end
+
+  describe 'headers' do
+    it "should include default headers" do
+      url     = 'http://example.com/headers'
+      request = double('Request', base_uri: url)
+      expected_headers = {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"}
+
+      MetaInspector::Request.any_instance.should_receive(:open)
+                                         .with(url, expected_headers)
+                                         .and_return(request)
+
+      MetaInspector::Document.new(url)
+    end
+
+    it "should include passed headers on the request" do
+      url     = 'http://example.com/headers'
+      headers = {'User-Agent' => 'Mozilla', 'Referer' => 'https://github.com/'}
+      request = double('Request', base_uri: url)
+
+      MetaInspector::Request.any_instance.should_receive(:open)
+                                         .with(url, headers)
+                                         .and_return(request)
+
+      MetaInspector::Document.new(url, headers: headers)
+    end
+  end
 end

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-16 00:00:00.000000000 Z
+date: 2014-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: open_uri_redirections
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.4
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.3.5
 - !ruby/object:Gem::Dependency
@@ -84,28 +84,28 @@ dependencies:
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 10.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 10.1.0
 description: MetaInspector lets you scrape a web page and get its title, charset,
@@ -116,15 +116,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".rspec.example"
-- ".travis.yml"
+- .gitignore
+- .rspec.example
+- .travis.yml
 - Gemfile
 - MIT-LICENSE
 - README.md
 - Rakefile
 - lib/meta_inspector.rb
-- lib/meta_inspector/deprecations.rb
 - lib/meta_inspector/document.rb
 - lib/meta_inspector/exception_log.rb
 - lib/meta_inspector/exceptionable.rb
@@ -182,17 +181,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: MetaInspector is a ruby gem for web scraping purposes, that returns a hash

data/lib/meta_inspector/deprecations.rb

@@ -1,19 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-module MetaInspector
-  class Scraper < Document
-    def initialize
-      warn "The Scraper class is now deprecated since version 1.17, use Document instead"
-      super
-    end
-
-    def errors
-      warn "The #errors method is deprecated since version 1.17, use #exceptions instead"
-      exceptions
-    end
-
-    def document
-      warn "The #document method is deprecated since version 1.17, use #to_s instead"
-    end
-  end
-end