metainspector 5.3.1 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef08a1d24e83d3916a355ff153cec55ccdb561a2
-  data.tar.gz: 7ee50a3e9bcc2f23a012df9344c6b528c227d2d6
+  metadata.gz: 273cd0ae91070bbf2937040d522fafad0ccde429
+  data.tar.gz: f0be5b153236da195c05254237ca44a11140f213
 SHA512:
-  metadata.gz: 5f33387482dec0bbf838678fb7409b940fb1e783719b6340f17117ac952e6d645483fa2ad19c2c788417ed26cdad77431c0ee86302f516949f6a53e57591d1e3
-  data.tar.gz: 769057a606eb74d41e5a4143a4c68a923eb19d21f3160f9f1d9fd2b14bd9d3962b0a549bdec4203d5d374d8139473d5622dca003b8799a59d6f0088d6be28179
+  metadata.gz: 1bdf4b3d45fff3711806da5a2cfd27c58e409e4b4d8b849302f1cff6e740ee14ad9e3783eda9622003e5438b65dd9e59ff5a2d507df5f97a473649878950d727
+  data.tar.gz: 7848876a57432a1d208e25d8d10ae78357d3a07d076f9a2109fd53bb3add56827b742245e789ff1db6ad2247465ef054462a6dcc373e06394e34b1a298b744d8

data/.travis.yml

@@ -1,5 +1,5 @@
 rvm:
-- 2.0.0
 - 2.1.8
 - 2.2.4
 - 2.3.1
+- 2.4.0

data/README.md

@@ -83,7 +83,8 @@ page.feed                # Get rss or atom links in meta data fields as array
 ```ruby
 page.title               # title of the page from the head section, as string
 page.best_title          # best title of the page, from a selection of candidates
-page.description         # returns the meta description, or the first long paragraph if no meta description is found
+page.description         # returns the meta description
+page.best_description    # returns the first non-empty description between the following candidates: standard meta description, og:description, twitter:description, the first long paragraph
 ```
 
 ### Links
@@ -437,6 +438,7 @@ You can also come to chat with us on our [Gitter room](https://gitter.im/jaimein
 * [go-metainspector](https://github.com/fern4lvarez/go-metainspector), a port of MetaInspector for Go.
 * [Node-MetaInspector](https://github.com/gabceb/node-metainspector), a port of MetaInspector for Node.
 * [MetaInvestigator](https://github.com/nekova/metainvestigator), a port of MetaInspector for Elixir.
+* [Funkspector] (https://github.com/jaimeiniesta/funkspector), another port of MetaInspector for Elixir.
 
 ## License
 MetaInspector is released under the [MIT license](MIT-LICENSE).

data/lib/meta_inspector/document.rb

@@ -46,7 +46,7 @@ module MetaInspector
     delegate [:content_type, :response]               => :@request
 
     delegate [:parsed, :title, :best_title,
-              :description, :links,
+              :description, :best_description, :links,
               :images, :feed, :charset, :meta_tags,
               :meta_tag, :meta, :favicon,
               :head_links, :stylesheets, :canonicals] => :@parser
@@ -54,22 +54,23 @@ module MetaInspector
     # Returns all document data as a nested Hash
     def to_hash
       {
-        'url'           => url,
-        'scheme'        => scheme,
-        'host'          => host,
-        'root_url'      => root_url,
-        'title'         => title,
-        'best_title'    => best_title,
-        'description'   => description,
-        'links'         => links.to_hash,
-        'images'        => images.to_a,
-        'charset'       => charset,
-        'feed'          => feed,
-        'content_type'  => content_type,
-        'meta_tags'     => meta_tags,
-        'favicon'       => images.favicon,
-        'response'      => { 'status'  => response.status,
-                             'headers' => response.headers }
+        'url'              => url,
+        'scheme'           => scheme,
+        'host'             => host,
+        'root_url'         => root_url,
+        'title'            => title,
+        'best_title'       => best_title,
+        'description'      => description,
+        'best_description' => best_description,
+        'links'            => links.to_hash,
+        'images'           => images.to_a,
+        'charset'          => charset,
+        'feed'             => feed,
+        'content_type'     => content_type,
+        'meta_tags'        => meta_tags,
+        'favicon'          => images.favicon,
+        'response'         => { 'status'  => response.status,
+                                'headers' => response.headers }
       }
     end
 

data/lib/meta_inspector/parser.rb

@@ -21,12 +21,12 @@ module MetaInspector
     end
 
     extend Forwardable
-    delegate [:url, :scheme, :host]                          => :@document
-    delegate [:meta_tags, :meta_tag, :meta, :charset]        => :@meta_tag_parser
-    delegate [:head_links, :stylesheets, :canonicals, :feed] => :@head_links_parser
-    delegate [:links, :base_url]                             => :@links_parser
-    delegate :images                                         => :@images_parser
-    delegate [:title, :best_title, :description]             => :@texts_parser
+    delegate [:url, :scheme, :host]                                 => :@document
+    delegate [:meta_tags, :meta_tag, :meta, :charset]               => :@meta_tag_parser
+    delegate [:head_links, :stylesheets, :canonicals, :feed]        => :@head_links_parser
+    delegate [:links, :base_url]                                    => :@links_parser
+    delegate :images                                                => :@images_parser
+    delegate [:title, :best_title, :description, :best_description] => :@texts_parser
 
     # Returns the whole parsed document
     def parsed

data/lib/meta_inspector/parsers/texts.rb

@@ -14,12 +14,19 @@ module MetaInspector
         @best_title ||= find_best_title
       end
 
-      # A description getter that first checks for a meta description
-      # and if not present will guess by looking at the first paragraph
-      # with more than 120 characters
+      # Returns the meta description, if present
       def description
-        return meta['description'] unless meta['description'].nil? || meta['description'].empty?
-        secondary_description
+        @description ||= meta['description']
+      end
+
+      # A description getter that returns the first non-nill description
+      # from the following candidates:
+      # - the standard meta description
+      # - the og:description meta tag
+      # - the twitter:description meta tag
+      # - the first paragraph with more than 120 characters
+      def best_description
+        @best_description ||= find_best_description
       end
 
       private
@@ -43,6 +50,16 @@ module MetaInspector
         candidates.first
       end
 
+      def find_best_description
+        candidates = [
+          meta['description'],
+          meta['og:description'],
+          meta['twitter:description'],
+          secondary_description
+        ]
+        candidates.find { |x| !x.to_s.empty? }
+      end
+
       # Look for the first <p> block with 120 characters or more
       def secondary_description
         first_long_paragraph = parsed.search('//p[string-length() >= 120]').first

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = '5.3.1'
+  VERSION = '5.4.0'
 end

data/spec/document_spec.rb

@@ -27,6 +27,7 @@ describe MetaInspector::Document do
                             "title"           => "PageRankAlert.com :: Track your PageRank changes & receive alerts",
                             "best_title"      => "PageRankAlert.com :: Track your PageRank changes & receive alerts",
                             "description"     => "Track your PageRank(TM) changes and receive alerts by email",
+                            "best_description"=> "Track your PageRank(TM) changes and receive alerts by email",
                             "favicon"         => "http://pagerankalert.com/src/favicon.ico",
                             "links"           => {
                                                     'internal' => ["http://pagerankalert.com/",

data/spec/fixtures/desc_in_meta.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200 OK
+Age: 13
+Cache-Control: max-age=120
+Content-Type: text/html
+Date: Mon, 06 Jan 2014 12:47:42 GMT
+Expires: Mon, 06 Jan 2014 12:49:28 GMT
+Server: Apache/2.2.14 (Ubuntu)
+Vary: Accept-Encoding
+Via: 1.1 varnish
+X-Powered-By: PHP/5.3.2-1ubuntu4.22
+X-Varnish: 1188792404 1188790413
+Content-Length: 40571
+Connection: keep-alive
+
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:fb="http://www.facebook.com/2008/fbml">
+  <head>
+    <meta name="description" content="the standard description" />
+</head>
+  <body>
+    <p>A sample page with description in standard meta tag</p>
+  </body>
+</html>

data/spec/fixtures/desc_in_og.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200 OK
+Age: 13
+Cache-Control: max-age=120
+Content-Type: text/html
+Date: Mon, 06 Jan 2014 12:47:42 GMT
+Expires: Mon, 06 Jan 2014 12:49:28 GMT
+Server: Apache/2.2.14 (Ubuntu)
+Vary: Accept-Encoding
+Via: 1.1 varnish
+X-Powered-By: PHP/5.3.2-1ubuntu4.22
+X-Varnish: 1188792404 1188790413
+Content-Length: 40571
+Connection: keep-alive
+
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:fb="http://www.facebook.com/2008/fbml">
+  <head>
+    <meta property="og:description" content="the og description" />
+</head>
+  <body>
+    <p>A sample page with description in og meta tag</p>
+  </body>
+</html>

data/spec/fixtures/desc_in_twitter.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200 OK
+Age: 13
+Cache-Control: max-age=120
+Content-Type: text/html
+Date: Mon, 06 Jan 2014 12:47:42 GMT
+Expires: Mon, 06 Jan 2014 12:49:28 GMT
+Server: Apache/2.2.14 (Ubuntu)
+Vary: Accept-Encoding
+Via: 1.1 varnish
+X-Powered-By: PHP/5.3.2-1ubuntu4.22
+X-Varnish: 1188792404 1188790413
+Content-Length: 40571
+Connection: keep-alive
+
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:fb="http://www.facebook.com/2008/fbml">
+  <head>
+    <meta property="twitter:description" content="the twitter description" />
+</head>
+  <body>
+    <p>A sample page with description in og meta tag</p>
+  </body>
+</html>

data/spec/meta_inspector/texts_spec.rb

@@ -47,24 +47,51 @@ describe MetaInspector do
       page = MetaInspector.new('http://www.youtube.com/watch?v=short_title')
       expect(page.best_title).to eq('Angular 2 Forms')
     end
-
   end
 
   describe '#description' do
     it "should find description from meta description" do
-      page = MetaInspector.new('http://www.youtube.com/watch?v=iaGSSrp49uc')
+      page = MetaInspector.new('http://example.com/desc_in_meta')
+
+      expect(page.description).to eq("the standard description")
+    end
+
+    it "should be nil if no meta description" do
+      page = MetaInspector.new('http://example.com/empty')
+
+      expect(page.description).to be(nil)
+    end
+  end
+
+  describe "#best_description" do
+    it "should return the standard description meta tag content if present" do
+      page = MetaInspector.new('http://example.com/desc_in_meta')
 
-      expect(page.description).to eq("This is Youtube")
+      expect(page.best_description).to eq("the standard description")
     end
 
-    it "should find a secondary description if no meta description" do
+    it "should return the og description if standard meta tag is not present" do
+      page = MetaInspector.new('http://example.com/desc_in_og')
+
+      expect(page.best_description).to eq("the og description")
+    end
+
+    it "should return the twitter description if standard and og tag not present" do
+      page = MetaInspector.new('http://example.com/desc_in_twitter')
+
+      expect(page.best_description).to eq("the twitter description")
+    end
+
+    it "should return the secondary description if no meta tag is present" do
       page = MetaInspector.new('http://theonion-no-description.com')
-      expect(page.description).to eq("SAN FRANCISCO—In a move expected to revolutionize the mobile device industry, Apple launched its fastest and most powerful iPhone to date Tuesday, an innovative new model that can only be seen by the company's hippest and most dedicated customers. This is secondary text picked up because of a missing meta description.")
+
+      expect(page.best_description).to eq("SAN FRANCISCO—In a move expected to revolutionize the mobile device industry, Apple launched its fastest and most powerful iPhone to date Tuesday, an innovative new model that can only be seen by the company's hippest and most dedicated customers. This is secondary text picked up because of a missing meta description.")
     end
 
-    it 'should find first paragraph if meta description is empty' do
-      expect(MetaInspector.new('http://example.com/empty-meta-description').description)
-        .to eq("Vivimos en una época en la que el término 'Innovación' esta siendo usado a placer por organizaciones de todo tipo. Hace algunos meses en una reunión de trabajo alguien mencionó:\"La innovación esta siendo usada como todo aquello que las empresas no saben dónde poner\". Mejor no lo pudo haber dicho. Llevamos más de dos años de estar colaborando cercanamente con directores de innovación de decenas de empresas, participando en comisiones industriales enfocadas a la innovación y haciendo conexiones laborales internacionales con expertos en materias de innovación (particularmente innovación abierta), y después de todo, la gran mayoría de las empresas no tienen claro lo que (por lo menos dentro de su organización) es innovación. ")
+    it "should return nil by default" do
+      page = MetaInspector.new('http://example.com/empty')
+
+      expect(page.best_description).to be(nil)
     end
   end
 end

data/spec/spec_helper.rb

@@ -56,6 +56,11 @@ FakeWeb.register_uri(:get, "http://example.com/title_not_present", :response =>
 # best_title now has specific logic for youtube
 FakeWeb.register_uri(:get, "http://www.youtube.com/watch?v=short_title", :response => fixture_file("youtube_short_title.response"))
 
+# Used to test best_description logic
+FakeWeb.register_uri(:get, "http://example.com/desc_in_meta", :response => fixture_file("desc_in_meta.response"))
+FakeWeb.register_uri(:get, "http://example.com/desc_in_og", :response => fixture_file("desc_in_og.response"))
+FakeWeb.register_uri(:get, "http://example.com/desc_in_twitter", :response => fixture_file("desc_in_twitter.response"))
+
 # These are older fixtures
 FakeWeb.register_uri(:get, "http://pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
 FakeWeb.register_uri(:get, "http://pagerankalert-shortcut.com", :response => fixture_file("pagerankalert-shortcut.com.response"))

metadata

@@ -1,153 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 5.3.1
+  version: 5.4.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-19 00:00:00.000000000 Z
+date: 2017-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
 - !ruby/object:Gem::Dependency
   name: faraday-cookie_jar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
 - !ruby/object:Gem::Dependency
   name: faraday-http-cache
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
   name: faraday-encoding
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.3
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
 - !ruby/object:Gem::Dependency
   name: fastimage
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
   name: nesty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
@@ -168,70 +168,70 @@ dependencies:
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 10.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 10.1.0
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: MetaInspector lets you scrape a web page and get its links, images, texts,
@@ -242,10 +242,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec.example
-- .rubocop.yml.example
-- .travis.yml
+- ".gitignore"
+- ".rspec.example"
+- ".rubocop.yml.example"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - MIT-LICENSE
@@ -280,6 +280,9 @@ files:
 - spec/fixtures/charset_000.response
 - spec/fixtures/charset_001.response
 - spec/fixtures/charset_002.response
+- spec/fixtures/desc_in_meta.response
+- spec/fixtures/desc_in_og.response
+- spec/fixtures/desc_in_twitter.response
 - spec/fixtures/empty_metatag_description.response
 - spec/fixtures/empty_page.response
 - spec/fixtures/encoding.response
@@ -344,17 +347,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: MetaInspector is a ruby gem for web scraping purposes, that returns metadata