metainspector 5.7.0 → 5.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 860350c6b4704259715f0a0eea047893134d774928b0bf78c9633f5afec480ff
-  data.tar.gz: d0abcb709893e63be252886723f8f202137dae3d02b506173a59b3368dcda7f7
+  metadata.gz: e684c1133fc1fee2a9ffcd99b3294d22bb1d6d8598c7faf0e5c86a117a0e8663
+  data.tar.gz: b542f91035175aa6304495dd8f5ec79bcf8c0279bd700948f20bc742832b0653
 SHA512:
-  metadata.gz: ab74dde15e1864f4c747be953a512278578ab231bea4637e1fb752234766c5b895b47405bd8572c06fd421cd16170e69c4331fb3462a7177ea4c6c521748e1f9
-  data.tar.gz: fde5cf878c7320d49c82aa430b2fc422044a07168e4b6e6cce53d3b33655b28f6ee811cc535ffca06be46c19403b83e7c453aa942895b9e77c675468fa3fea12
+  metadata.gz: 796aa57288c6873fad48f67cc9c9da36f04d84cd24d66c2825944e42a5655a4d5cbf2a86fddc5592c44afb166a069630d372e3faa6ac4e8964eb8408ee9176aa
+  data.tar.gz: 16dcd111e4197a836a75b357fd035a051247b04f2afc52ff4832a4c63eef0bd986568873f17807dbe1951bcb4b42400d379449ee98fdd2a5afee16b152f439ce

data/.travis.yml

@@ -1,6 +1,5 @@
 script: "bundle exec rspec -b"
 rvm:
-- 2.3.6
-- 2.4.3
-- 2.5.0
-- 2.6.4
+- 2.5.8
+- 2.6.6
+- 2.7.1

data/CHANGELOG.md

@@ -1,5 +1,32 @@
 # MetaInpector Changelog
 
+## Unreleased
+
+* Upgrade to Faraday 1.1.
+
+## [Changes in 5.10.1](https://github.com/jaimeiniesta/metainspector/compare/v5.10.0...v5.10.1)
+
+* Fix for empty base_href. Makes relative links work when base_href is nil but empty ("").
+* Drop support for Ruby 2.4, add support for Ruby 2.7.
+
+## [Changes in 5.10](https://github.com/jaimeiniesta/metainspector/compare/v5.9.0...v5.10.0)
+
+* Upgrade to Faraday 1.0.
+
+## [Changes in 5.9](https://github.com/jaimeiniesta/metainspector/compare/v5.8.0...v5.9.0)
+
+* Added #feeds method to retrieve all feeds of a page.
+* Adds deprecation warning on #feed method.
+
+## [Changes in 5.8](https://github.com/jaimeiniesta/metainspector/compare/v5.7.0...v5.8.0)
+
+* Added h1..h6 support.
+
+## [Changes in 5.7](https://github.com/jaimeiniesta/metainspector/compare/v5.6.0...v5.7.0)
+
+* Avoids normalizing image URLs. https://github.com/jaimeiniesta/metainspector/pull/241
+* Adds `NonHtmlErrorException` instead of `ParserError` https://github.com/jaimeiniesta/metainspector/pull/248
+
 ## [Changes in 5.6](https://github.com/jaimeiniesta/metainspector/compare/v5.5.0...v5.6.0)
 
 * New feature: `:encoding` option for force encoding of a parsed document.

data/README.md

@@ -22,6 +22,8 @@ If you're using it on a Rails application, just add it to your Gemfile and run `
 gem 'metainspector'
 ```
 
+Supported Ruby versions are defined in [`.travis.yml`](.travis.yml).
+
 ## Usage
 
 Initialize a MetaInspector instance for an URL, like this:
@@ -73,7 +75,7 @@ page.root_url            # Root url (scheme + host, like http://sitevalidator.co
 page.head_links          # an array of hashes of all head/links
 page.stylesheets         # an array of hashes of all head/links where rel='stylesheet'
 page.canonicals          # an array of hashes of all head/links where rel='canonical'
-page.feed                # Get rss or atom links in meta data fields as array
+page.feeds               # Get rss or atom links in meta data fields as array of hash in the form { href: "...", title: "...", type: "..." }
 ```
 
 ### Texts
@@ -85,6 +87,12 @@ page.author              # author of the page from the meta author tag
 page.best_author         # best author of the page, from a selection of candidates
 page.description         # returns the meta description
 page.best_description    # returns the first non-empty description between the following candidates: standard meta description, og:description, twitter:description, the first long paragraph
+page.h1                  # returns h1 text array
+page.h2                  # returns h2 text array
+page.h3                  # returns h3 text array
+page.h4                  # returns h4 text array
+page.h5                  # returns h5 text array
+page.h6                  # returns h6 text array
 ```
 
 ### Links

data/lib/meta_inspector/document.rb

@@ -48,8 +48,8 @@ module MetaInspector
     delegate [:content_type, :response]               => :@request
 
     delegate [:parsed, :title, :best_title, :author, :best_author,
-              :description, :best_description, :links,
-              :images, :feed, :charset, :meta_tags,
+              :h1, :h2, :h3, :h4, :h5, :h6, :description, :best_description, :links,
+              :images, :feeds, :feed, :charset, :meta_tags,
               :meta_tag, :meta, :favicon,
               :head_links, :stylesheets, :canonicals] => :@parser
 
@@ -66,10 +66,17 @@ module MetaInspector
         'best_author'      => best_author,
         'description'      => description,
         'best_description' => best_description,
+        'h1'               => h1,
+        'h2'               => h2,
+        'h3'               => h3,
+        'h4'               => h4,
+        'h5'               => h5,
+        'h6'               => h6,
         'links'            => links.to_hash,
         'images'           => images.to_a,
         'charset'          => charset,
         'feed'             => feed,
+        'feeds'            => feeds,
         'content_type'     => content_type,
         'meta_tags'        => meta_tags,
         'favicon'          => images.favicon,

data/lib/meta_inspector/parser.rb

@@ -23,10 +23,11 @@ module MetaInspector
     extend Forwardable
     delegate [:url, :scheme, :host]                                                        => :@document
     delegate [:meta_tags, :meta_tag, :meta, :charset]                                      => :@meta_tag_parser
-    delegate [:head_links, :stylesheets, :canonicals, :feed]                               => :@head_links_parser
+    delegate [:head_links, :stylesheets, :canonicals, :feeds, :feed]                       => :@head_links_parser
     delegate [:links, :base_url]                                                           => :@links_parser
     delegate :images                                                                       => :@images_parser
-    delegate [:title, :best_title, :author, :best_author, :description, :best_description] => :@texts_parser
+    delegate [:title, :best_title, :author, :best_author, :description, :best_description, 
+              :h1, :h2, :h3, :h4, :h5, :h6]                                                => :@texts_parser
 
     # Returns the whole parsed document
     def parsed

data/lib/meta_inspector/parsers/head_links.rb

@@ -3,6 +3,10 @@ module MetaInspector
     class HeadLinksParser < Base
       delegate [:parsed, :base_url] => :@main_parser
 
+      KNOWN_FEED_TYPES = %w[
+        application/rss+xml application/atom+xml application/json
+      ].freeze
+
       def head_links
         @head_links ||= parsed.css('head link').map do |tag|
           Hash[
@@ -24,16 +28,25 @@ module MetaInspector
         @canonicals ||= head_links.select { |hl| hl[:rel] == 'canonical' }
       end
 
-      # Returns the parsed document meta rss link
-      def feed
-        @feed ||= (parsed_feed('rss') || parsed_feed('atom'))
-      end
+      def feeds
+        @feeds ||=
+          parsed.search("//link[@rel='alternate']").map do |link|
+            next if !KNOWN_FEED_TYPES.include?(link["type"]) || link["href"].to_s.strip == ''
 
-      private
+            {
+              title: link["title"],
+              href: URL.absolutify(link["href"], base_url),
+              type: link["type"]
+            }
+          end.compact
+      end
 
-      def parsed_feed(format)
-        feed = parsed.search("//link[@type='application/#{format}+xml']").find{|link| link.attributes["href"] }
-        feed ? URL.absolutify(feed['href'], base_url) : nil
+      def feed
+        warn "DEPRECATION: Use MetaInspector#feeds instead of #feed. The former gives you all feeds and their metadata, the latter will be removed."
+        @feed ||= begin
+          first_feed = feeds.find { |l| /\/(rss|atom)\+xml$/i =~ l[:type] } || {}
+          first_feed[:href]
+        end
       end
     end
   end

data/lib/meta_inspector/parsers/links.rb

@@ -47,7 +47,8 @@ module MetaInspector
       # This can be the one set on a <base> tag,
       # or the url of the document if no <base> tag was found.
       def base_url
-        base_href || url
+        current_base_href = base_href.to_s.strip.empty? ? nil : base_href
+        current_base_href || url
       end
 
       # Returns the value of the href attribute on the <base /> tag, if exists

data/lib/meta_inspector/parsers/texts.rb

@@ -13,6 +13,30 @@ module MetaInspector
         @best_title ||= find_best_title
       end
 
+      def h1
+        @h1 ||= find_heading('h1')
+      end
+
+      def h2
+        @h2 ||= find_heading('h2')
+      end
+
+      def h3
+        @h3 ||= find_heading('h3')
+      end
+      
+      def h4
+        @h4 ||= find_heading('h4')
+      end
+
+      def h5
+        @h5 ||= find_heading('h5')
+      end
+
+      def h6
+        @h6 ||= find_heading('h6')
+      end
+
       # Returns the meta author, if present
       def author
         @author ||= meta['author']
@@ -45,6 +69,10 @@ module MetaInspector
 
       private
 
+      def find_heading(heading)
+        parsed.css(heading).map { |tag| tag.inner_text.strip.gsub(/\s+/, ' ') }.reject(&:empty?)
+      end
+
       # Look for candidates per list of priority
       def find_best_title
         candidates = [

data/lib/meta_inspector/request.rb

@@ -48,7 +48,7 @@ module MetaInspector
       @response ||= fetch
     rescue Faraday::TimeoutError => e
       raise MetaInspector::TimeoutError.new(e)
-    rescue Faraday::Error::ConnectionFailed, Faraday::SSLError, URI::InvalidURIError, FaradayMiddleware::RedirectLimitReached => e
+    rescue Faraday::ConnectionFailed, Faraday::SSLError, URI::InvalidURIError, FaradayMiddleware::RedirectLimitReached => e
       raise MetaInspector::RequestError.new(e)
     end
 

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = '5.7.0'
+  VERSION = '5.11.0'
 end

data/meta_inspector.gemspec

@@ -1,11 +1,11 @@
 require File.expand_path('../lib/meta_inspector/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Jaime Iniesta"]
-  gem.email         = ["jaimeiniesta@gmail.com"]
+  gem.author        = "Jaime Iniesta"
+  gem.email         = "jaimeiniesta@gmail.com"
   gem.description   = %q{MetaInspector lets you scrape a web page and get its links, images, texts, meta tags...}
   gem.summary       = %q{MetaInspector is a ruby gem for web scraping purposes, that returns metadata from a given URL}
-  gem.homepage      = "https://github.com/jaimeiniesta/metainspector"
+  gem.homepage      = "https://github.com/metainspector/metainspector"
   gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split("\n")
@@ -14,20 +14,20 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = MetaInspector::VERSION
 
-  gem.add_dependency 'nokogiri', '~> 1.10.4'
-  gem.add_dependency 'faraday', '~> 0.15.3'
-  gem.add_dependency 'faraday_middleware', '~> 0.12.2'
-  gem.add_dependency 'faraday-cookie_jar', '~> 0.0.6'
-  gem.add_dependency 'faraday-http-cache', '~> 2.0.0'
+  gem.add_dependency 'nokogiri', '~> 1.10.9'
+  gem.add_dependency 'faraday', '~> 1.1.0'
+  gem.add_dependency 'faraday_middleware', '~> 1.0.0'
+  gem.add_dependency 'faraday-cookie_jar', '~> 0.0.7'
+  gem.add_dependency 'faraday-http-cache', '~> 2.2.0'
   gem.add_dependency 'faraday-encoding', '~> 0.0.5'
-  gem.add_dependency 'addressable', '~> 2.5.2'
-  gem.add_dependency 'fastimage', '~> 2.1.4'
+  gem.add_dependency 'addressable', '~> 2.7.0'
+  gem.add_dependency 'fastimage', '~> 2.1.7'
   gem.add_dependency 'nesty', '~> 1.0.2'
 
-  gem.add_development_dependency 'rspec', '~> 3.8.0'
-  gem.add_development_dependency 'webmock', '~> 3.7.6'
+  gem.add_development_dependency 'rspec', '~> 3.9.0'
+  gem.add_development_dependency 'webmock', '~> 3.8.3'
   gem.add_development_dependency 'awesome_print', '~> 1.8.0'
-  gem.add_development_dependency 'rake', '~> 12.3.1'
-  gem.add_development_dependency 'pry', '~> 0.12.2'
-  gem.add_development_dependency 'rubocop', '~> 0.60.0'
+  gem.add_development_dependency 'rake', '~> 13.0.1'
+  gem.add_development_dependency 'pry', '~> 0.13.1'
+  gem.add_development_dependency 'rubocop', '~> 0.82.0'
 end

data/spec/document_spec.rb

@@ -44,6 +44,13 @@ describe MetaInspector::Document do
                             "images"          => ["http://pagerankalert.com/images/pagerank_alert.png?1305794559"],
                             "charset"         => "utf-8",
                             "feed"            => "http://feeds.feedburner.com/PageRankAlert",
+                            "feeds"           => [{href: "http://feeds.feedburner.com/PageRankAlert", title: "PageRankAlert.com blog", type: "application/rss+xml"}],
+                            "h1"              => [],
+                            "h2"              => ["Track your PageRank changes"],
+                            "h3"              => ["WHAT'S YOUR PAGERANK?"],
+                            "h4"              => ["Build your own lists", "Get e-mail alerts", "Track your history"],
+                            "h5"              => [],
+                            "h6"              => [],
                             "content_type"    => "text/html",
                             "meta_tags"       => {
                                                    "name" => {

data/spec/fixtures/feeds.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200
+date: Wed, 08 Jan 2020 23:21:58 GMT
+content-type: text/html; charset=UTF-8
+server: nginx/0.7.67
+
+<!DOCTYPE html>
+<html>
+<head>
+  <title>a page with feeds</title>
+  <link rel="alternate" title="Articles - JSON Feed" type="application/json" href="https://example.org/feed.json" />
+  <link rel="alternate" title="Comments - JSON Feed" type="application/json" href="https://example.org/feed/comments.json" />
+  <link rel="alternate" title="Articles - RSS Feed" type="application/rss+xml" href="https://example.org/feed.rss" />
+  <link rel="alternate" title="Comments - RSS Feed" type="application/rss+xml" href="https://example.org/feed/comments.rss" />
+  <link rel="alternate" title="Articles - Atom Feed" type="application/atom+xml" href="https://example.org/feed.xml" />
+  <link rel="alternate" title="Comments - Atom Feed" type="application/atom+xml" href="https://example.org/feed/comments.xml" />
+
+  <link rel="alternate" title="Invalid Feed" />
+  <link rel="alternate" title="Feed with empty href" type="application/atom+xml" href="" />
+</head>
+<body>
+
+</body>
+</html>

data/spec/fixtures/headings.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200 OK
+Server: nginx/0.7.67
+Date: Fri, 18 Nov 2011 21:46:46 GMT
+Content-Type: text/html
+Connection: keep-alive
+Last-Modified: Mon, 14 Nov 2011 16:53:18 GMT
+Content-Length: 4987
+X-Varnish: 2000423390
+Age: 0
+Via: 1.1 varnish
+
+<html>
+  <head>
+  </head>
+  <body>
+    <h1>H1</h1>
+    <h2>H2</h2>
+    <h3>H3</h3>
+    <h4>H4</h4>
+    <h5>H5</h5>
+    <h6>H6</h6>
+  </body>
+</html>

data/spec/fixtures/relative_links_with_empty_base.response

@@ -0,0 +1,22 @@
+HTTP/1.1 200 OK
+Server: nginx/1.0.5
+Date: Thu, 29 Dec 2011 23:10:13 GMT
+Content-Type: text/html
+Content-Length: 15013
+Last-Modified: Fri, 02 Dec 2011 21:00:49 GMT
+Connection: keep-alive
+Accept-Ranges: bytes
+
+<!DOCTYPE html>
+<html>
+<head>
+  <base href=""/>
+  <meta charset="utf-8" />
+  <title>Relative links</title>
+</head>
+<body>
+  <p>Relative links</p>
+  <a href="about">About</a>
+  <a href="../sitemap">Sitemap</a>
+</body>
+</html>

data/spec/meta_inspector/head_links_spec.rb

@@ -39,7 +39,10 @@ describe MetaInspector do
     context "on page with some broken feed links" do
       let(:page){ MetaInspector.new('http://example.com/broken_head_links') }
       it "tries to find correct one" do
-        expect(page.feed).to eq("http://www.guardian.co.uk/media/techcrunch/rss")
+        expected = [
+          { title: "TechCrunch RSS feed", href: "http://www.guardian.co.uk/media/techcrunch/rss", type: "application/rss+xml" }
+        ]
+        expect(page.feeds).to eq(expected)
       end
     end
   end

data/spec/meta_inspector/links_spec.rb

@@ -145,6 +145,13 @@ describe MetaInspector do
     end
   end
 
+  describe 'Relative links with empty or blank base' do
+    it 'should get the relative links from a document' do
+      m = MetaInspector.new('http://relativewithemptybase.com/company')
+      expect(m.links.internal).to eq(['http://relativewithemptybase.com/about', 'http://relativewithemptybase.com/sitemap'])
+    end
+  end
+
   describe 'Relative links with base' do
     it 'should get the relative links from a document' do
       m = MetaInspector.new('http://relativewithbase.com/company/page2')
@@ -190,20 +197,37 @@ describe MetaInspector do
     end
   end
 
-  describe "Feed" do
-    it "should get rss feed" do
-      @m = MetaInspector.new('http://www.iteh.at')
-      expect(@m.feed).to eq('http://www.iteh.at/de/rss/')
-    end
+  context "Feeds" do
+    let(:meta) { MetaInspector.new('http://feeds.example.com') }
+
+    describe "#feeds" do
+      it "should return all the document's feeds" do
+        expected = [
+          { title: "Articles - JSON Feed", href: "https://example.org/feed.json",          type: "application/json" },
+          { title: "Comments - JSON Feed", href: "https://example.org/feed/comments.json", type: "application/json" },
+          { title: "Articles - RSS Feed",  href: "https://example.org/feed.rss",           type: "application/rss+xml" },
+          { title: "Comments - RSS Feed",  href: "https://example.org/feed/comments.rss",  type: "application/rss+xml" },
+          { title: "Articles - Atom Feed", href: "https://example.org/feed.xml",           type: "application/atom+xml" },
+          { title: "Comments - Atom Feed", href: "https://example.org/feed/comments.xml",  type: "application/atom+xml" }
+        ]
+        expect(meta.feeds).to eq(expected)
+      end
 
-    it "should get atom feed" do
-      @m = MetaInspector.new('http://www.tea-tron.com/jbravo/blog/')
-      expect(@m.feed).to eq('http://www.tea-tron.com/jbravo/blog/feed/')
+      it "should return nothing if no feeds found" do
+        @m = MetaInspector.new('http://www.alazan.com')
+        expect(@m.feeds).to eq([])
+      end
     end
 
-    it "should return nil if no feed found" do
-      @m = MetaInspector.new('http://www.alazan.com')
-      expect(@m.feed).to eq(nil)
+    describe "#feed" do
+      it "should return the first feed's href" do
+        expect(meta.feed).to eq("https://example.org/feed.rss")
+      end
+
+      it "should give a deprecation warning" do
+        warning = "DEPRECATION: Use MetaInspector#feeds instead of #feed. The former gives you all feeds and their metadata, the latter will be removed.\n"
+        expect { meta.feed }.to output(warning).to_stderr
+      end
     end
   end
 end