metainspector 5.7.0 → 5.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 860350c6b4704259715f0a0eea047893134d774928b0bf78c9633f5afec480ff
-  data.tar.gz: d0abcb709893e63be252886723f8f202137dae3d02b506173a59b3368dcda7f7
+  metadata.gz: a349b7103e8b214dc3211da2bc46d830e8e88b85fa7bde9b734ff3ba13802be0
+  data.tar.gz: fc4b691504551ee1c0ca6df5f92f5f5dbf5f2ebfa71415cff5259a67aff17c5f
 SHA512:
-  metadata.gz: ab74dde15e1864f4c747be953a512278578ab231bea4637e1fb752234766c5b895b47405bd8572c06fd421cd16170e69c4331fb3462a7177ea4c6c521748e1f9
-  data.tar.gz: fde5cf878c7320d49c82aa430b2fc422044a07168e4b6e6cce53d3b33655b28f6ee811cc535ffca06be46c19403b83e7c453aa942895b9e77c675468fa3fea12
+  metadata.gz: 1a268679f23f0ecfdd4358304999200175c78c2bef37ad1d5acdba641b46512ec6e6dca8d18005b87bc7fbc9b632592ac657818c99d540324685a56912a8985c
+  data.tar.gz: 8f90160c9f923e69157cfc8c2d6b13f386dd394501d37330a6ede47e587d9474adbe9b932df86340fc3c14754eedd59538eef65944801f6567b26189be8db077

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # MetaInpector Changelog
 
+## [Changes in 5.7](https://github.com/jaimeiniesta/metainspector/compare/v5.6.0...v5.7.0)
+
+* Avoids normalizing image URLs. https://github.com/jaimeiniesta/metainspector/pull/241
+* Adds `NonHtmlErrorException` instead of `ParserError` https://github.com/jaimeiniesta/metainspector/pull/248
+
 ## [Changes in 5.6](https://github.com/jaimeiniesta/metainspector/compare/v5.5.0...v5.6.0)
 
 * New feature: `:encoding` option for force encoding of a parsed document.

data/README.md

@@ -85,6 +85,12 @@ page.author              # author of the page from the meta author tag
 page.best_author         # best author of the page, from a selection of candidates
 page.description         # returns the meta description
 page.best_description    # returns the first non-empty description between the following candidates: standard meta description, og:description, twitter:description, the first long paragraph
+page.h1                  # returns h1 text array
+page.h2                  # returns h2 text array
+page.h3                  # returns h3 text array
+page.h4                  # returns h4 text array
+page.h5                  # returns h5 text array
+page.h6                  # returns h6 text array
 ```
 
 ### Links

data/lib/meta_inspector/document.rb

@@ -48,7 +48,7 @@ module MetaInspector
     delegate [:content_type, :response]               => :@request
 
     delegate [:parsed, :title, :best_title, :author, :best_author,
-              :description, :best_description, :links,
+              :h1, :h2, :h3, :h4, :h5, :h6, :description, :best_description, :links,
               :images, :feed, :charset, :meta_tags,
               :meta_tag, :meta, :favicon,
               :head_links, :stylesheets, :canonicals] => :@parser
@@ -66,6 +66,12 @@ module MetaInspector
         'best_author'      => best_author,
         'description'      => description,
         'best_description' => best_description,
+        'h1'               => h1,
+        'h2'               => h2,
+        'h3'               => h3,
+        'h4'               => h4,
+        'h5'               => h5,
+        'h6'               => h6,
         'links'            => links.to_hash,
         'images'           => images.to_a,
         'charset'          => charset,

data/lib/meta_inspector/parser.rb

@@ -26,7 +26,8 @@ module MetaInspector
     delegate [:head_links, :stylesheets, :canonicals, :feed]                               => :@head_links_parser
     delegate [:links, :base_url]                                                           => :@links_parser
     delegate :images                                                                       => :@images_parser
-    delegate [:title, :best_title, :author, :best_author, :description, :best_description] => :@texts_parser
+    delegate [:title, :best_title, :author, :best_author, :description, :best_description, 
+              :h1, :h2, :h3, :h4, :h5, :h6]                                                => :@texts_parser
 
     # Returns the whole parsed document
     def parsed

data/lib/meta_inspector/parsers/texts.rb

@@ -13,6 +13,30 @@ module MetaInspector
         @best_title ||= find_best_title
       end
 
+      def h1
+        @h1 ||= find_heading('h1')
+      end
+
+      def h2
+        @h2 ||= find_heading('h2')
+      end
+
+      def h3
+        @h3 ||= find_heading('h3')
+      end
+      
+      def h4
+        @h4 ||= find_heading('h4')
+      end
+
+      def h5
+        @h5 ||= find_heading('h5')
+      end
+
+      def h6
+        @h6 ||= find_heading('h6')
+      end
+
       # Returns the meta author, if present
       def author
         @author ||= meta['author']
@@ -45,6 +69,10 @@ module MetaInspector
 
       private
 
+      def find_heading(heading)
+        parsed.css(heading).map { |tag| tag.inner_text.strip.gsub(/\s+/, ' ') }.reject(&:empty?)
+      end
+
       # Look for candidates per list of priority
       def find_best_title
         candidates = [

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = '5.7.0'
+  VERSION = '5.8.0'
 end

data/meta_inspector.gemspec

@@ -15,19 +15,19 @@ Gem::Specification.new do |gem|
   gem.version       = MetaInspector::VERSION
 
   gem.add_dependency 'nokogiri', '~> 1.10.4'
-  gem.add_dependency 'faraday', '~> 0.15.3'
-  gem.add_dependency 'faraday_middleware', '~> 0.12.2'
+  gem.add_dependency 'faraday', '~> 0.17.0'
+  gem.add_dependency 'faraday_middleware', '~> 0.13.1'
   gem.add_dependency 'faraday-cookie_jar', '~> 0.0.6'
   gem.add_dependency 'faraday-http-cache', '~> 2.0.0'
   gem.add_dependency 'faraday-encoding', '~> 0.0.5'
-  gem.add_dependency 'addressable', '~> 2.5.2'
-  gem.add_dependency 'fastimage', '~> 2.1.4'
+  gem.add_dependency 'addressable', '~> 2.7.0'
+  gem.add_dependency 'fastimage', '~> 2.1.7'
   gem.add_dependency 'nesty', '~> 1.0.2'
 
-  gem.add_development_dependency 'rspec', '~> 3.8.0'
+  gem.add_development_dependency 'rspec', '~> 3.9.0'
   gem.add_development_dependency 'webmock', '~> 3.7.6'
   gem.add_development_dependency 'awesome_print', '~> 1.8.0'
-  gem.add_development_dependency 'rake', '~> 12.3.1'
+  gem.add_development_dependency 'rake', '~> 13.0.0'
   gem.add_development_dependency 'pry', '~> 0.12.2'
-  gem.add_development_dependency 'rubocop', '~> 0.60.0'
+  gem.add_development_dependency 'rubocop', '~> 0.75.1'
 end

data/spec/document_spec.rb

@@ -44,6 +44,12 @@ describe MetaInspector::Document do
                             "images"          => ["http://pagerankalert.com/images/pagerank_alert.png?1305794559"],
                             "charset"         => "utf-8",
                             "feed"            => "http://feeds.feedburner.com/PageRankAlert",
+                            "h1"              => [],
+                            "h2"              => ["Track your PageRank changes"],
+                            "h3"              => ["WHAT'S YOUR PAGERANK?"],
+                            "h4"              => ["Build your own lists", "Get e-mail alerts", "Track your history"],
+                            "h5"              => [],
+                            "h6"              => [],
                             "content_type"    => "text/html",
                             "meta_tags"       => {
                                                    "name" => {

data/spec/fixtures/headings.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200 OK
+Server: nginx/0.7.67
+Date: Fri, 18 Nov 2011 21:46:46 GMT
+Content-Type: text/html
+Connection: keep-alive
+Last-Modified: Mon, 14 Nov 2011 16:53:18 GMT
+Content-Length: 4987
+X-Varnish: 2000423390
+Age: 0
+Via: 1.1 varnish
+
+<html>
+  <head>
+  </head>
+  <body>
+    <h1>H1</h1>
+    <h2>H2</h2>
+    <h3>H3</h3>
+    <h4>H4</h4>
+    <h5>H5</h5>
+    <h6>H6</h6>
+  </body>
+</html>

data/spec/meta_inspector/texts_spec.rb

@@ -6,6 +6,48 @@ describe MetaInspector do
     expect(page.title).to eq('An example page')
   end
 
+  describe "#h1" do 
+    it "should find h1 content" do
+      page = MetaInspector.new('http://example.com/headings')
+      expect(page.h1.first).to eq('H1')
+    end
+  end
+
+  describe "#h2" do 
+    it "should find h2 content" do
+      page = MetaInspector.new('http://example.com/headings')
+      expect(page.h2.first).to eq('H2')
+    end
+  end
+
+  describe "#h3" do 
+    it "should find h3 content" do
+      page = MetaInspector.new('http://example.com/headings')
+      expect(page.h3.first).to eq('H3')
+    end
+  end
+
+  describe "#h4" do 
+    it "should find h4 content" do
+      page = MetaInspector.new('http://example.com/headings')
+      expect(page.h4.first).to eq('H4')
+    end
+  end
+
+  describe "#h5" do 
+    it "should find h5 content" do
+      page = MetaInspector.new('http://example.com/headings')
+      expect(page.h5.first).to eq('H5')
+    end
+  end
+
+  describe "#h6" do 
+    it "should find h6 content" do
+      page = MetaInspector.new('http://example.com/headings')
+      expect(page.h6.first).to eq('H6')
+    end
+  end
+
   describe '#best_title' do
     it "should find 'head title' when that's the only thing" do
       page = MetaInspector.new('http://example.com/title_in_head')

data/spec/spec_helper.rb

@@ -29,6 +29,7 @@ RSpec.configure do |config|
     stub_request(:get, "http://example.com/desc_in_twitter").to_return(fixture_file("desc_in_twitter.response"))
     stub_request(:get, "http://example.com/empty").to_return(fixture_file("empty_page.response"))
     stub_request(:get, "http://example.com/head_links").to_return(fixture_file("head_links.response"))
+    stub_request(:get, "http://example.com/headings").to_return(fixture_file("headings.response"))
     stub_request(:get, "http://example.com/invalid_byte_seq").to_return(fixture_file("invalid_byte_seq.response"))
     stub_request(:get, "http://example.com/invalid_utf8_byte_seq").to_return(fixture_file("invalid_utf8_byte_seq.response"))
     stub_request(:get, "http://example.com/invalid_href").to_return(fixture_file("invalid_href.response"))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 5.7.0
+  version: 5.8.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-01 00:00:00.000000000 Z
+date: 2019-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -30,28 +30,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.15.3
+        version: 0.17.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.15.3
+        version: 0.17.0
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
 - !ruby/object:Gem::Dependency
   name: faraday-cookie_jar
   requirement: !ruby/object:Gem::Requirement
@@ -100,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.2
+        version: 2.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.2
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
   name: fastimage
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.4
+        version: 2.1.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.4
+        version: 2.1.7
 - !ruby/object:Gem::Dependency
   name: nesty
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.8.0
+        version: 3.9.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.8.0
+        version: 3.9.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -184,14 +184,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.1
+        version: 13.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.1
+        version: 13.0.0
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -212,14 +212,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.60.0
+        version: 0.75.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.60.0
+        version: 0.75.1
 description: MetaInspector lets you scrape a web page and get its links, images, texts,
   meta tags...
 email:
@@ -280,6 +280,7 @@ files:
 - spec/fixtures/facebook.com.response
 - spec/fixtures/guardian.co.uk.response
 - spec/fixtures/head_links.response
+- spec/fixtures/headings.response
 - spec/fixtures/https.facebook.com.response
 - spec/fixtures/international.response
 - spec/fixtures/invalid_byte_seq.response