metainspector 3.3.0 → 4.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72feb77b18ae296efcdd96b30950e2deaefb9689
-  data.tar.gz: b6442b9f256c6b31e18005af2f5955c339b37298
+  metadata.gz: 84153e6fb0df5db8c6e71c3b918c5afb48da2ff0
+  data.tar.gz: c3c98fc6d9488202a114769d0cb38871ba9eb871
 SHA512:
-  metadata.gz: a9b4002024e2466d5538a1e54c39fe336f2b049128249956b1951a23dfb3f1d09cdea7da63b9e602102fe936f8e14b884000a48a1a970ecd05ab58a632aecef5
-  data.tar.gz: 99d8ced9fcf7e03df6316cd5442a54320df1dca1e32ad3a9ee9421638f38f779860994bece0496a048180df49bfbf67580df01ceb4c674d2e66c6df90e0c77f5
+  metadata.gz: 65ba2a93d70893615ccb20a416d5c461e5f93877cdf6a77688131e6f7ad597f48ad48188930267cef9088f17af25fb934cdb219d355187d88f91070bbee77e96
+  data.tar.gz: dbbf14dad512d0ca6bf2dedc1dba7b6c8f93fcd0ed5ce1e3b7a6c0efd3345612555d2825a57db6efc18e727b0f8239794b123adf822c946a306add942185deec

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/meta_inspector/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')             { 'spec' }
+end

data/README.md

@@ -8,9 +8,24 @@ You give it an URL, and it lets you easily get its title, links, images, charset
 
 You can try MetaInspector live at this little demo: [https://metainspectordemo.herokuapp.com](https://metainspectordemo.herokuapp.com)
 
-## Changes in 3.0
+## Changes in 4.0
+
+* The links API has been changed, now instead of `page.links`, `page.internal_links` and `page.external_links` we have:
+
+```ruby
+page.links.raw      # Returns all links found, unprocessed
+page.links.all      # Returns all links found, unrelavitized and absolutified
+page.links.http     # Returns all HTTP links found
+page.links.non_http # Returns all non-HTTP links found
+page.links.internal # Returns all internal HTTP links found
+page.links.external # Returns all external HTTP links found
+```
+
+* The images API has been changed, now instead of `page.image` we have `page.images.best`, and instead of `page.favicon` we have `page.images.favicon`.
 
-This latest release introduces some backwards-incompatible changes, so we've decided to do a major version upgrade:
+* Now `page.image` will return the first image in `page.images` if no OG or Twitter image found, instead of returning `nil`.
+
+## Changes in 3.0
 
 * The redirect API has been changed, now the `:allow_redirections` option will expect only a boolean, which by default is `true`. That is, no more specifying `:safe`, `:unsafe` or `:all`.
 * We've dropped support for Ruby < 2.
@@ -63,18 +78,21 @@ You can see the scraped data like this:
     page.host                # Hostname of the page (like, sitevalidator.com, without the scheme)
     page.root_url            # Root url (scheme + host, like http://sitevalidator.com/)
     page.title               # title of the page, as string
-    page.links               # array of strings, with every link found on the page as an absolute URL
-    page.internal_links      # array of strings, with every internal link found on the page as an absolute URL
-    page.external_links      # array of strings, with every external link found on the page as an absolute URL
+    page.links.raw           # every link found, unprocessed
+    page.links.all           # every link found on the page as an absolute URL
+    page.links.http          # every HTTP link found
+    page.links.non_http      # every non-HTTP link found
+    page.links.internal      # every internal link found on the page as an absolute URL
+    page.links.external      # every external link found on the page as an absolute URL
     page.meta['keywords']    # meta keywords, as string
     page.meta['description'] # meta description, as string
     page.description         # returns the meta description, or the first long paragraph if no meta description is found
-    page.image               # Most relevant image, if defined with the og:image meta tag
-    page.images              # array of strings, with every img found on the page as an absolute URL
+    page.images              # enumerable collection, with every img found on the page as an absolute URL
+    page.images.best         # Most relevant image, if defined with the og:image or twitter:image metatags. Fallback to the first page.images array element
+    page.images.favicon      # absolute URL to the favicon
     page.feed                # Get rss or atom links in meta data fields as array
     page.charset             # UTF-8
     page.content_type        # content-type returned by the server when the url was requested
-    page.favicon             # absolute URL to the favicon
 
 ## Meta tags
 

data/lib/meta_inspector/document.rb

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-
 module MetaInspector
   # A MetaInspector::Document knows about its URL and its contents
   class Document
@@ -35,7 +33,7 @@ module MetaInspector
     extend Forwardable
     def_delegators :@url,     :url, :scheme, :host, :root_url
     def_delegators :@request, :content_type, :response
-    def_delegators :@parser,  :parsed, :respond_to?, :title, :description, :links, :internal_links, :external_links,
+    def_delegators :@parser,  :parsed, :respond_to?, :title, :description, :links,
                               :images, :image, :feed, :charset, :meta_tags, :meta_tag, :meta, :favicon
 
     # Returns all document data as a nested Hash
@@ -43,15 +41,13 @@ module MetaInspector
       {
         'url' => url,
         'title' => title,
-        'links' => links,
-        'internal_links' => internal_links,
-        'external_links' => external_links,
-        'images' => images,
+        'links' => links.to_hash,
+        'images' => images.to_a,
         'charset' => charset,
         'feed' => feed,
         'content_type' => content_type,
         'meta_tags' => meta_tags,
-        'favicon' => favicon,
+        'favicon' => images.favicon,
         'response' => { 'status'  => response.status,
                         'headers' => response.headers }
       }

data/lib/meta_inspector/exception_log.rb

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-
 module MetaInspector
 
   # Stores the exceptions passed to it, warning about them if required

data/lib/meta_inspector/exceptionable.rb

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-
 module MetaInspector
   #
   # This module extracts two common methods for classes that use ExceptionLog

data/lib/meta_inspector/parser.rb

@@ -1,36 +1,30 @@
-# -*- encoding: utf-8 -*-
-
 require 'nokogiri'
 
 module MetaInspector
-  # Parses the document with Nokogiri
+  ##
+  # Parses the document with Nokogiri.
+  #
+  # Delegates the parsing of the different elements to specialized parsers,
+  # passing itself as a reference for coordination purposes
+  #
   class Parser
     include MetaInspector::Exceptionable
 
     def initialize(document, options = {})
-      @document       = document
-      @exception_log  = options[:exception_log]
+      @document        = document
+      @exception_log   = options[:exception_log]
+      @meta_tag_parser = MetaInspector::Parsers::MetaTagsParser.new(self)
+      @links_parser    = MetaInspector::Parsers::LinksParser.new(self)
+      @images_parser   = MetaInspector::Parsers::ImagesParser.new(self)
+      @texts_parser    = MetaInspector::Parsers::TextsParser.new(self)
     end
 
     extend Forwardable
-    def_delegators :@document, :url, :scheme, :host
-
-    def meta_tags
-      {
-        'name'        => meta_tags_by('name'),
-        'http-equiv'  => meta_tags_by('http-equiv'),
-        'property'    => meta_tags_by('property'),
-        'charset'     => [charset_from_meta_charset]
-      }
-    end
-
-    def meta_tag
-      convert_each_array_to_first_element_on meta_tags
-    end
-
-    def meta
-      meta_tag['name'].merge(meta_tag['http-equiv']).merge(meta_tag['property']).merge({'charset' => meta_tag['charset']})
-    end
+    def_delegators :@document,        :url, :scheme, :host
+    def_delegators :@meta_tag_parser, :meta_tags, :meta_tag, :meta, :charset
+    def_delegators :@links_parser,    :links, :feed, :base_url
+    def_delegators :@images_parser,   :images, :image, :favicon
+    def_delegators :@texts_parser,    :title, :description
 
     # Returns the whole parsed document
     def parsed
@@ -38,144 +32,5 @@ module MetaInspector
     rescue Exception => e
       @exception_log << e
     end
-
-    # Returns the parsed document title, from the content of the <title> tag
-    # within the <head> section.
-    # This is not the same as the meta_title tag
-    def title
-      @title ||= parsed.css('head title').inner_text rescue nil
-    end
-
-    # Return favicon url if exist
-    def favicon
-      query = '//link[@rel="icon" or contains(@rel, "shortcut")]'
-      value = parsed.xpath(query)[0].attributes['href'].value
-      @favicon ||= URL.absolutify(value, base_url)
-    rescue
-      nil
-    end
-
-    # A description getter that first checks for a meta description and if not present will
-    # guess by looking at the first paragraph with more than 120 characters
-    def description
-      meta['description'] || secondary_description
-    end
-
-    # Links found on the page, as absolute URLs
-    def links
-      @links ||= parsed_links.map{ |l| URL.absolutify(URL.unrelativize(l, scheme), base_url) }.compact.uniq
-    end
-
-    # Internal links found on the page, as absolute URLs
-    def internal_links
-      @internal_links ||= links.select {|link| URL.new(link).host == host }
-    end
-
-    # External links found on the page, as absolute URLs
-    def external_links
-      @external_links ||= links.select {|link| URL.new(link).host != host }
-    end
-
-    # Images found on the page, as absolute URLs
-    def images
-      @images ||= parsed_images.map{ |i| URL.absolutify(i, base_url) }
-    end
-
-    # Returns the parsed image from Facebook's open graph property tags
-    # Most all major websites now define this property and is usually very relevant
-    # See doc at http://developers.facebook.com/docs/opengraph/
-    # If none found, tries with Twitter image
-    # TODO: if not found, try with images.first
-    def image
-      meta['og:image'] || meta['twitter:image']
-    end
-
-    # Returns the parsed document meta rss link
-    def feed
-      @feed ||= (parsed_feed('rss') || parsed_feed('atom'))
-    end
-
-    # Returns the charset from the meta tags, looking for it in the following order:
-    # <meta charset='utf-8' />
-    # <meta http-equiv="Content-Type" content="text/html; charset=windows-1252" />
-    def charset
-      @charset ||= (charset_from_meta_charset || charset_from_meta_content_type)
-    end
-
-    private
-
-    def meta_tags_by(attribute)
-      hash = {}
-      parsed.css("meta[@#{attribute}]").map do |tag|
-        name    = tag.attributes[attribute].value.downcase rescue nil
-        content = tag.attributes['content'].value rescue nil
-
-        if name && content
-          hash[name] ||= []
-          hash[name] << content
-        end
-      end
-      hash
-    end
-
-    def convert_each_array_to_first_element_on(hash)
-      hash.each_pair do |k, v|
-        hash[k] = if v.is_a?(Hash)
-          convert_each_array_to_first_element_on(v)
-        elsif v.is_a?(Array)
-          v.first
-        else
-          v
-        end
-      end
-    end
-
-    # Look for the first <p> block with 120 characters or more
-    def secondary_description
-      first_long_paragraph = parsed_search('//p[string-length() >= 120]').first
-      first_long_paragraph ? first_long_paragraph.text : ''
-    end
-
-    def parsed_links
-      @parsed_links ||= cleanup_nokogiri_values(parsed_search("//a/@href"))
-    end
-
-    def parsed_images
-      @parsed_images ||= cleanup_nokogiri_values(parsed_search('//img/@src'))
-    end
-
-    def parsed_feed(format)
-      feed = parsed_search("//link[@type='application/#{format}+xml']").first
-      feed ? URL.absolutify(feed.attributes['href'].value, base_url) : nil
-    end
-
-    def charset_from_meta_charset
-      parsed.css("meta[charset]")[0].attributes['charset'].value rescue nil
-    end
-
-    def charset_from_meta_content_type
-      parsed.css("meta[http-equiv='Content-Type']")[0].attributes['content'].value.split(";")[1].split("=")[1] rescue nil
-    end
-
-    # Returns the base url to absolutify relative links. This can be the one set on a <base> tag,
-    # or the url of the document if no <base> tag was found.
-    def base_url
-      base_href || url
-    end
-
-    # Returns the value of the href attribute on the <base /> tag, if it exists
-    def base_href
-      parsed_search('base').first.attributes['href'].value rescue nil
-    end
-
-    # Takes a nokogiri search result, strips the values, rejects the empty ones, and removes duplicates
-    def cleanup_nokogiri_values(results)
-      results.map { |a| a.value.strip }.reject { |s| s.empty? }.uniq
-    end
-
-    # Searches the parsed document for the selector, if the parsed document is searchable
-    def parsed_search(selector)
-      parsed.respond_to?(:search) ? parsed.search(selector) : []
-    end
   end
 end

data/lib/meta_inspector/parsers/base.rb

@@ -0,0 +1,30 @@
+module MetaInspector
+  module Parsers
+    ##
+    # Base class from where the specialized parsers inherit from.
+    #
+    # On initialization a main parser is expected, so the specialized
+    # parsers can request the parsed document to the main parser, and
+    # then perform the searches on it.
+    #
+    # The main parser also serves as a message hub between the specialized
+    # parsers. For example, the ImagesParser needs to know the base_url
+    # in order to absolutify image URLs, so it delegates it to the main parser
+    # which, in turn, delegates it to the LinksParser.
+    #
+    class Base
+      def initialize(main_parser)
+        @main_parser = main_parser
+      end
+
+      extend Forwardable
+
+      private
+
+      # Cleans up nokogiri search results
+      def cleanup(results)
+        results.map { |_| _.value.strip }.reject { |_| _.empty? }.uniq
+      end
+    end
+  end
+end

data/lib/meta_inspector/parsers/images.rb

@@ -0,0 +1,45 @@
+module MetaInspector
+  module Parsers
+    class ImagesParser < Base
+      def_delegators :@main_parser, :parsed, :meta, :base_url
+      def_delegators :images_collection, :length, :size
+
+      include Enumerable
+
+      def images
+        self
+      end
+
+      def each(&block)
+        images_collection.each(&block)
+      end
+
+      # Returns the parsed image from Facebook's open graph property tags
+      # Most all major websites now define this property and is usually very relevant
+      # See doc at http://developers.facebook.com/docs/opengraph/
+      # If none found, tries with Twitter image
+      def best
+        meta['og:image'] || meta['twitter:image'] || images_collection.first
+      end
+
+      # Return favicon url if exist
+      def favicon
+        query = '//link[@rel="icon" or contains(@rel, "shortcut")]'
+        value = parsed.xpath(query)[0].attributes['href'].value
+        @favicon ||= URL.absolutify(value, base_url)
+      rescue
+        nil
+      end
+
+      private
+
+      def images_collection
+        @images_collection ||= parsed_images.map{ |i| URL.absolutify(i, base_url) }
+      end
+
+      def parsed_images
+        @parsed_images ||= cleanup(parsed.search('//img/@src'))
+      end
+    end
+  end
+end

data/lib/meta_inspector/parsers/links.rb

@@ -0,0 +1,69 @@
+module MetaInspector
+  module Parsers
+    class LinksParser < Base
+      def_delegators :@main_parser, :parsed, :url, :scheme, :host
+
+      def links
+        self
+      end
+
+      # Returns all links found, unprocessed
+      def raw
+        @raw ||= cleanup(parsed.search("//a/@href")).compact.uniq
+      end
+
+      # Returns all links found, unrelavitized and absolutified
+      def all
+        @all ||= raw.map { |l| URL.absolutify(URL.unrelativize(l, scheme), base_url) }
+                    .compact.uniq
+      end
+
+      # Returns all HTTP links found
+      def http
+        @http ||= all.select {|l| l =~ /^http(s)?:\/\//i}
+      end
+
+      # Returns all non-HTTP links found
+      def non_http
+        @non_http ||= all.select {|l| l !~ /^http(s)?:\/\//i}
+      end
+
+      # Returns all internal HTTP links found
+      def internal
+        @internal ||= http.select {|link| URL.new(link).host == host }
+      end
+
+      # Returns all external HTTP links found
+      def external
+        @external ||= http.select {|link| URL.new(link).host != host }
+      end
+
+      def to_hash
+        { 'internal' => internal, 'external' => external, 'non_http' => non_http }
+      end
+
+      # Returns the parsed document meta rss link
+      def feed
+        @feed ||= (parsed_feed('rss') || parsed_feed('atom'))
+      end
+
+      # Returns the base url to absolutify relative links. This can be the one set on a <base> tag,
+      # or the url of the document if no <base> tag was found.
+      def base_url
+        base_href || url
+      end
+
+      private
+
+      def parsed_feed(format)
+        feed = parsed.search("//link[@type='application/#{format}+xml']").first
+        feed ? URL.absolutify(feed.attributes['href'].value, base_url) : nil
+      end
+
+      # Returns the value of the href attribute on the <base /> tag, if it exists
+      def base_href
+        parsed.search('base').first.attributes['href'].value rescue nil
+      end
+    end
+  end
+end

data/lib/meta_inspector/parsers/meta_tags.rb

@@ -0,0 +1,72 @@
+module MetaInspector
+  module Parsers
+    class MetaTagsParser < Base
+      def_delegators :@main_parser, :parsed
+
+      def meta_tags
+        {
+          'name'        => meta_tags_by('name'),
+          'http-equiv'  => meta_tags_by('http-equiv'),
+          'property'    => meta_tags_by('property'),
+          'charset'     => [charset_from_meta_charset]
+        }
+      end
+
+      def meta_tag
+        convert_each_array_to_first_element_on meta_tags
+      end
+
+      def meta
+        meta_tag['name']
+          .merge(meta_tag['http-equiv'])
+          .merge(meta_tag['property'])
+          .merge({'charset' => meta_tag['charset']})
+      end
+
+      # Returns the charset from the meta tags, looking for it in the following order:
+      # <meta charset='utf-8' />
+      # <meta http-equiv="Content-Type" content="text/html; charset=windows-1252" />
+      def charset
+        @charset ||= (charset_from_meta_charset || charset_from_meta_content_type)
+      end
+
+      private
+
+      def charset_from_meta_charset
+        parsed.css("meta[charset]")[0].attributes['charset'].value rescue nil
+      end
+
+      def charset_from_meta_content_type
+        parsed.css("meta[http-equiv='Content-Type']")[0]
+          .attributes['content'].value.split(";")[1].split("=")[1] rescue nil
+      end
+
+      def meta_tags_by(attribute)
+        hash = {}
+        parsed.css("meta[@#{attribute}]").map do |tag|
+          name    = tag.attributes[attribute].value.downcase rescue nil
+          content = tag.attributes['content'].value rescue nil
+
+          if name && content
+            hash[name] ||= []
+            hash[name] << content
+          end
+        end
+        hash
+      end
+
+      def convert_each_array_to_first_element_on(hash)
+        hash.each_pair do |k, v|
+          hash[k] = if v.is_a?(Hash)
+            convert_each_array_to_first_element_on(v)
+          elsif v.is_a?(Array)
+            v.first
+          else
+            v
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/meta_inspector/parsers/texts.rb

@@ -0,0 +1,27 @@
+module MetaInspector
+  module Parsers
+    class TextsParser < Base
+      def_delegators :@main_parser, :parsed, :meta
+
+      # Returns the parsed document title, from the content of the <title> tag
+      # within the <head> section.
+      def title
+        @title ||= parsed.css('head title').inner_text rescue nil
+      end
+
+      # A description getter that first checks for a meta description and if not present will
+      # guess by looking at the first paragraph with more than 120 characters
+      def description
+        meta['description'] || secondary_description
+      end
+
+      private
+
+      # Look for the first <p> block with 120 characters or more
+      def secondary_description
+        first_long_paragraph = parsed.search('//p[string-length() >= 120]').first
+        first_long_paragraph ? first_long_paragraph.text : ''
+      end
+    end
+  end
+end

data/lib/meta_inspector/request.rb

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-
 require 'faraday'
 require 'faraday_middleware'
 require 'faraday-cookie_jar'

data/lib/meta_inspector/url.rb

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-
 require 'addressable/uri'
 
 module MetaInspector

data/lib/meta_inspector/version.rb

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-
 module MetaInspector
-  VERSION = "3.3.0"
+  VERSION = "4.0.0.rc1"
 end

data/lib/meta_inspector.rb

@@ -1,11 +1,14 @@
-# -*- encoding: utf-8 -*-
-
 require 'forwardable'
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/exceptionable'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/exception_log'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/request'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/url'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/parser'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/parsers/base'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/parsers/images'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/parsers/links'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/parsers/meta_tags'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/parsers/texts'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/document'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/version'))
 

data/lib/metainspector.rb

@@ -1,3 +1 @@
-# -*- encoding: utf-8 -*-
-
 require File.expand_path(File.join(File.dirname(__FILE__), './meta_inspector'))

data/meta_inspector.gemspec

@@ -1,4 +1,3 @@
-# -*- encoding: utf-8 -*-
 require File.expand_path('../lib/meta_inspector/version', __FILE__)
 
 Gem::Specification.new do |gem|
@@ -27,4 +26,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'awesome_print', '~> 1.2.0'
   gem.add_development_dependency 'rake', '~> 10.1.0'
   gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
 end