metainspector 4.3.3 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e10230be8608519d139bf58fac5520aec7f511ad
-  data.tar.gz: 738c8c089374e8e6dbe68212a445a475460d111a
+  metadata.gz: ccdca184756e92a93aee21d1c2ac23bd0acfad7e
+  data.tar.gz: 0039759535f19c26f0e70271bbceb8f53a79ec2e
 SHA512:
-  metadata.gz: 06107c66a9a420009f832de9c7a2d1593c509260dbad12c6a78d9e47af1ec3cbbd09de8ae2dd831024cb26c3f156f1ed23370d7f590164e346e7c7d719a76e6d
-  data.tar.gz: cb248b9019bec125b1b0623b4f27ffd5a6ce7f519cae490034abc6df2362673b44664d3cbb649225d8d4ff1b078f77a46eb8397385872dbe158030c22186b806
+  metadata.gz: f0ef85b48a07af588d53bb148b5e4badae0df4fd49ce46b890cec6a3471c817704ce127a0243c11f5e945cbd741f831a7964645af590ba5f8b33003e78cc43d5
+  data.tar.gz: 52bd1966da3e5fa8093d7b59d99880e26e8c40b03d390be0c276802edfdb0cf500892149f3fa7bccee631f086bf0db8a8701adc57b211dbd152754ee2cf14a66

data/README.md

@@ -8,6 +8,10 @@ You give it an URL, and it lets you easily get its title, links, images, charset
 
 You can try MetaInspector live at this little demo: [https://metainspectordemo.herokuapp.com](https://metainspectordemo.herokuapp.com)
 
+## Changes in 4.4
+
+The default headers now include `'Accept-Encoding' => 'identity'` to minimize trouble with servers that respond with malformed compressed responses, [as explained here](https://github.com/lostisland/faraday/issues/337).
+
 ## Changes in 4.3
 
 * The Document API has been extended with one new method `page.best_title` that returns the longest text available from a selection of candidates.
@@ -310,10 +314,15 @@ page = MetaInspector.new('facebook.com', :allow_redirections => false)
 By default, the following headers are set:
 
 ```ruby
-{'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"}
+{
+  'User-Agent'      => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)",
+  'Accept-Encoding' => 'identity'
+}
 ```
 
-If you want to set custom headers then use the `headers` option:
+The `Accept-Encoding` is set to `identity` to avoid exceptions being raised on servers that return malformed compressed responses, [as explained here](https://github.com/lostisland/faraday/issues/337).
+
+If you want to override the default headers then use the `headers` option:
 
 ```ruby
 # Set the User-Agent header

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "metainspector"
+
+require "pry"
+Pry.start

data/lib/meta_inspector/document.rb

@@ -87,7 +87,10 @@ module MetaInspector
         :retries            => 3,
         :html_content_only  => false,
         :warn_level         => :raise,
-        :headers            => { 'User-Agent' => default_user_agent },
+        :headers            => {
+                                 'User-Agent'      => default_user_agent,
+                                 'Accept-Encoding' => 'identity'
+                               },
         :allow_redirections => true,
         :normalize_url      => true,
         :download_images    => true }

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = "4.3.3"
+  VERSION = "4.4.0"
 end

data/spec/document_spec.rb

@@ -158,7 +158,7 @@ describe MetaInspector::Document do
   describe 'headers' do
     it "should include default headers" do
       url = "http://pagerankalert.com/"
-      expected_headers = {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"}
+      expected_headers = {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)", 'Accept-Encoding' => 'identity'}
 
       headers = {}
       expect(headers).to receive(:merge!).with(expected_headers)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 4.3.3
+  version: 4.4.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-26 00:00:00.000000000 Z
+date: 2015-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -237,6 +237,7 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
+- bin/console
 - examples/basic_scraping.rb
 - examples/link_checker.rb
 - examples/spider.rb
@@ -332,7 +333,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: MetaInspector is a ruby gem for web scraping purposes, that returns metadata