metainspector 5.0.2 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a38316bdb0ec3f428421adfa7fe90c61673a26e0
-  data.tar.gz: 46a787a3feaffdb516871dacd8cc72f0009aae36
+  metadata.gz: 0f69de2705313a7048cf7d2067238a38f7e42bfc
+  data.tar.gz: 9b4770b9f1e769b5da1abab6814bc7dc4bb38fa6
 SHA512:
-  metadata.gz: a1168ee9c0530f78f8a6efd6a5f67cf06c0f6f8a3bef1c427e70073056edf71eb6d90c94c84c4d8e97fc14e04924ad6c1f9ef35bf3eaaaaec269c5ebeb955bb5
-  data.tar.gz: ada6c39cf2e0db2d241ef136d47429649899b1bc700c662447e71a57fd752b23c874b9e8c90a9ecb59e8e72430db34a9957769cf01c8e1632bf08aa225526deb
+  metadata.gz: 2f35f03456f48aa4cfd65de50349ac7f67d08ad699e1f72c29251e95468f52457c137beeddcd4cf2afac261a9a6dfdb4d9462b831a907fd75d01bfa6168231e5
+  data.tar.gz: afc08f9b3217b7c6ee0fec83d1fef1f706af14f685a44d081724341e93ef352d2a69645436b375d9c0e08f49a576ab08669539f75b826b51bfbb3203e4f0e6a5

data/README.md

@@ -328,13 +328,13 @@ MetaInspector.new('https://example.com', faraday_options: { ssl: { verify: false
 
 ### HTML Content Only
 
-MetaInspector will try to parse all URLs by default. If you want to raise an exception when trying to parse a non-html URL (one that has a content-type different than text/html), you can state it like this:
+MetaInspector will try to parse all URLs by default. By default, it will raise an exception when trying to parse a non-html URL (one that has a content-type different than text/html). You can disable this behaviour with:
 
 ```ruby
-page = MetaInspector.new('sitevalidator.com', :html_content_only => true)
+page = MetaInspector.new('sitevalidator.com', :html_content_only => false)
 ```
 
-This is useful when using MetaInspector on web spidering. Although on the initial URL you'll probably have an HTML URL, following links you may find yourself trying to parse non-html URLs.
+This option is deprecated since 5.1.0 and will be removed in 5.2.0.
 
 ```ruby
 page = MetaInspector.new('http://example.com/image.png')

data/lib/meta_inspector/document.rb

@@ -8,8 +8,6 @@ module MetaInspector
     # * connection_timeout: defaults to 20 seconds
     # * read_timeout: defaults to 20 seconds
     # * retries: defaults to 3 times
-    # * html_content_type_only: if an exception should be raised if request
-    #   content-type is not text/html. Defaults to false.
     # * allow_redirections: when true, follow HTTP redirects. Defaults to true
     # * document: the html of the url as a string
     # * headers: object containing custom headers for the request
@@ -20,7 +18,16 @@ module MetaInspector
       @connection_timeout = options[:connection_timeout]
       @read_timeout       = options[:read_timeout]
       @retries            = options[:retries]
-      @html_content_only  = options[:html_content_only]
+
+      unless options[:html_content_only].nil?
+        @html_content_only = options[:html_content_only]
+
+        puts <<-EOS
+          DEPRECATION NOTICE: html_content_only is deprecated and turned on by default since 5.1.0,
+          this option will be removed in 5.2.0
+        EOS
+      end
+
       @allow_redirections = options[:allow_redirections]
       @document           = options[:document]
       @download_images    = options[:download_images]
@@ -83,7 +90,7 @@ module MetaInspector
     def defaults
       { :timeout            => 20,
         :retries            => 3,
-        :html_content_only  => false,
+        :html_content_only  => true,
         :headers            => {
                                  'User-Agent'      => default_user_agent,
                                  'Accept-Encoding' => 'identity'
@@ -98,11 +105,11 @@ module MetaInspector
     end
 
     def document
-      @document ||= if html_content_only && content_type != 'text/html'
-                      fail MetaInspector::ParserError.new "The url provided contains #{content_type} content instead of text/html content"
-                    else
-                      @request.read
-                    end
+      @document ||= if html_content_only && !content_type.nil? && content_type != 'text/html'
+        fail MetaInspector::ParserError.new "The url provided contains #{content_type} content instead of text/html content"
+      else
+        @request.read
+      end
     end
   end
 end

data/lib/meta_inspector/request.rb

@@ -2,6 +2,7 @@ require 'faraday'
 require 'faraday_middleware'
 require 'faraday-cookie_jar'
 require 'faraday-http-cache'
+require 'faraday/encoding'
 
 module MetaInspector
 
@@ -62,6 +63,7 @@ module MetaInspector
         end
 
         faraday.headers.merge!(@headers || {})
+        faraday.response :encoding
         faraday.adapter :net_http
       end
 

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = '5.0.2'
+  VERSION = '5.1.0'
 end

data/meta_inspector.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'faraday_middleware', '~> 0.10'
   gem.add_dependency 'faraday-cookie_jar', '~> 0.0'
   gem.add_dependency 'faraday-http-cache', '~> 1.2'
+  gem.add_dependency 'faraday-encoding', '~> 0.0.2'
   gem.add_dependency 'addressable', '~> 2.4'
   gem.add_dependency 'fastimage', '~> 1.8.1'
   gem.add_dependency 'nesty', '~> 1.0'

data/spec/document_spec.rb

@@ -75,11 +75,11 @@ describe MetaInspector::Document do
   end
 
   describe 'exception handling' do
-    it "should parse images when parse_html_content_type_only is not specified" do
+    it "should not parse images when parse_html_content_type_only is not specified" do
       expect do
         image_url = MetaInspector::Document.new('http://pagerankalert.com/image.png')
         image_url.title
-      end.to_not raise_error
+      end.to raise_error
     end
 
     it "should parse images when parse_html_content_type_only is false" do
@@ -137,4 +137,14 @@ describe MetaInspector::Document do
       expect(MetaInspector.new('http://example.com/%EF%BD%9E', normalize_url: false).url).to eq('http://example.com/%EF%BD%9E')
     end
   end
+
+  describe 'page encoding' do
+    it 'should encode title according to the charset' do
+      expect(MetaInspector.new('http://example-rtl.com/').title).to eq('بالفيديو.. "مصطفى بكري" : انتخابات الائتلاف غير نزيهة وموجهة لفوز أشخاص بعينها')
+    end
+
+    it 'should encode description according to the charset' do
+      expect(MetaInspector.new('http://example-rtl.com/').description).to eq('أعلن النائب مصطفى بكري انسحابه من ائتلاف  دعم مصر  بعد اعتراضه على نتيجة الانتخابات الداخلية للائتلاف، وخسارته فيها، وقال إنه سيترشح غدا على منصب الوكيل بالمجلس')
+    end
+  end
 end

data/spec/fixtures/encoding.response

@@ -0,0 +1,27 @@
+HTTP/1.1 200 OK
+Server: nginx admin
+Date: Mon, 01 Feb 2016 16:18:35 GMT
+Content-Type: text/html; charset=windows-1256
+Transfer-Encoding: chunked
+Connection: keep-alive
+Vary: Accept-Encoding
+X-Powered-By: PHP/5.3.29
+Set-Cookie: PHPSESSID=a0ddee5488c7480bbc7b0a50ac472d2a; path=/
+Expires: Thu, 19 Nov 1981 08:52:00 GMT
+Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
+Pragma: no-cache
+
+
+<!doctype html>
+<html class="no-js" lang="ar" dir="rtl">
+<head>
+  <title>��������.. "����� ����" : �������� �������� ��� ����� ������ ���� ����� ������</title>
+  <meta name="description" content="���� ������ ����� ���� ������� �� ������  ��� ���  ��� ������� ��� ����� ���������� �������� �������ݡ ������� ���ǡ ���� ��� ������ ��� ��� ���� ������ �������">
+  <meta name="keywords" content="��������,,,,�����,����,,,,��������,��������,���,�����,������,����,�����,������">
+  <meta property='og:title' content='��������.. "����� ����" : �������� �������� ��� ����� ������ ���� ����� ������' />
+</head>
+<body class="page-front">
+  <h1 class="hide">�����</h1>
+
+</body>
+</html>

data/spec/spec_helper.rb

@@ -107,3 +107,6 @@ FakeWeb.register_uri(:get, "http://unsafe-facebook.com/",   :response => fixture
 # These examples are used to test normalize URLs
 FakeWeb.register_uri(:get, "http://example.com/%EF%BD%9E", :response => fixture_file("example.response"))
 FakeWeb.register_uri(:get, "http://example.com/~", :response => fixture_file("example.response"))
+
+# Example to test correct encoding
+FakeWeb.register_uri(:get, "http://example-rtl.com/", :response => fixture_file("encoding.response"))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 5.0.2
+  version: 5.1.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-24 00:00:00.000000000 Z
+date: 2016-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: faraday-encoding
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -295,6 +309,7 @@ files:
 - spec/fixtures/charset_001.response
 - spec/fixtures/charset_002.response
 - spec/fixtures/empty_page.response
+- spec/fixtures/encoding.response
 - spec/fixtures/example.response
 - spec/fixtures/facebook.com.response
 - spec/fixtures/guardian.co.uk.response