metainspector 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 694ffa2f1b0080c05335ccc14abe02a874a6562f
-  data.tar.gz: 8ef1ff54d9cf15ab21225bf64f2827033dc3b409
+  metadata.gz: bbcf96088ef49b859442dfd0244bda8b7e4870fb
+  data.tar.gz: 3f87e155f4e1d260f6eff96867b458278a8a450b
 SHA512:
-  metadata.gz: 92ab014d16a8c6ad1332db4dac9e103c1ab114b3f04c823cc2bc140c43ddb038ae3c77d46bb273b6f6651b80bc3a8bde519ae1cc89ca436bcb1510707c03b888
-  data.tar.gz: d9fa07214d680a5af7b1a2d57f1e8481f85dfbe09f76c45978d5fa346c7c7d64614442c305ef4c4a7d3f1dc3613f467a873431abba9c1db60157265249f7dc07
+  metadata.gz: 005a2f07c88b2ca40bcf970ef7eabe2eb0b40d76bb1556a67a0bfbbcc170ab6be1836942dd841d97cd9a110fb25dcd8cd6e3aeb3f164cd2f3dcc020bb7708d27
+  data.tar.gz: 26ef69520abd2564e431a22dd3e6d139263e7163a250478c8a70dd41d3fadbfb22a3828757d645192d763d7ea0e2c5298b53d874a3f999b03ff1d68451ad422b

data/README.md

@@ -8,6 +8,19 @@ You give it an URL, and it lets you easily get its title, links, images, charset
 
 You can try MetaInspector live at this little demo: [https://metainspectordemo.herokuapp.com](https://metainspectordemo.herokuapp.com)
 
+## Changes in 4.2.0
+
+* The images API has been extended, with two new methods:
+
+  * `page.images.owner_suggested` returns the OG or Twitter image, or `nil` if neither are present.
+  * `page.images.largest` returns the largest image found in the page. This uses the HTML height and width attributes as well as the [fastimage](https://github.com/sdsykes/fastimage) gem to return the largest image on the page that has a ratio squarer than 1:10 or 10:1. This usually provides a good alternative to the OG or Twitter images if they are not supplied.
+
+* The criteria for `page.images.best` has changed slightly, we'll now return the largest image instead of the first image if no owner-suggested image is found.
+
+## Changes in 4.1.0
+
+* Introduces the `:normalize_url` option, which allows to disable URL normalization.
+
 ## Changes in 4.0
 
 * The links API has been changed, now instead of `page.links`, `page.internal_links` and `page.external_links` we have:
@@ -339,6 +352,16 @@ While this is generally useful, it can be [tricky](https://github.com/sporkmonge
 
 You can disable URL normalization by passing the `normalize_url: false` option.
 
+### Image downloading
+
+When you ask for the largest image on the page with `page.images.largest`, it will be determined by its height and width attributes on the HTML markup, and also by downloading a small portion of each image using the [fastimage](https://github.com/sdsykes/fastimage) gem. This is really fast as it doesn't download the entire images, normally just the headers of the image files.
+
+If you want to disable this, you can specify it like this:
+
+```ruby
+page = MetaInspector.new('http://example.com', download_images: false)
+```
+
 ## Exception Handling
 
 By default, MetaInspector will raise the exceptions found. We think that this is the safest default: in case the URL you're trying to scrape is unreachable, you should clearly be notified, and treat the exception as needed in your app.

data/lib/meta_inspector/document.rb

@@ -26,19 +26,21 @@ module MetaInspector
       @html_content_only  = options[:html_content_only]
       @allow_redirections = options[:allow_redirections]
       @document           = options[:document]
+      @download_images    = options[:download_images]
       @headers            = options[:headers]
       @warn_level         = options[:warn_level]
       @exception_log      = options[:exception_log] || MetaInspector::ExceptionLog.new(warn_level: warn_level)
       @normalize_url      = options[:normalize_url]
-      @url                = MetaInspector::URL.new(initial_url, exception_log: @exception_log,
-                                                                normalize:     @normalize_url)
-      @request            = MetaInspector::Request.new(@url,  allow_redirections: @allow_redirections,
-                                                              connection_timeout: @connection_timeout,
-                                                              read_timeout:       @read_timeout,
-                                                              retries:            @retries,
-                                                              exception_log:      @exception_log,
-                                                              headers:            @headers) unless @document
-      @parser             = MetaInspector::Parser.new(self,  exception_log:      @exception_log)
+      @url                = MetaInspector::URL.new(initial_url, exception_log:      @exception_log,
+                                                                normalize:          @normalize_url)
+      @request            = MetaInspector::Request.new(@url,    allow_redirections: @allow_redirections,
+                                                                connection_timeout: @connection_timeout,
+                                                                read_timeout:       @read_timeout,
+                                                                retries:            @retries,
+                                                                exception_log:      @exception_log,
+                                                                headers:            @headers) unless @document
+      @parser             = MetaInspector::Parser.new(self,     exception_log:      @exception_log,
+                                                                download_images:    @download_images)
     end
 
     extend Forwardable
@@ -81,7 +83,8 @@ module MetaInspector
         :warn_level         => :raise,
         :headers            => { 'User-Agent' => default_user_agent },
         :allow_redirections => true,
-        :normalize_url      => true }
+        :normalize_url      => true,
+        :download_images    => true }
     end
 
     def default_user_agent

data/lib/meta_inspector/parser.rb

@@ -15,7 +15,8 @@ module MetaInspector
       @exception_log   = options[:exception_log]
       @meta_tag_parser = MetaInspector::Parsers::MetaTagsParser.new(self)
       @links_parser    = MetaInspector::Parsers::LinksParser.new(self)
-      @images_parser   = MetaInspector::Parsers::ImagesParser.new(self)
+      @download_images = options[:download_images]
+      @images_parser   = MetaInspector::Parsers::ImagesParser.new(self, download_images: @download_images)
       @texts_parser    = MetaInspector::Parsers::TextsParser.new(self)
     end
 

data/lib/meta_inspector/parsers/images.rb

@@ -1,3 +1,5 @@
+require 'fastimage'
+
 module MetaInspector
   module Parsers
     class ImagesParser < Base
@@ -6,16 +8,53 @@ module MetaInspector
 
       include Enumerable
 
+      def initialize(main_parser, options = {})
+        @download_images = options[:download_images]
+        super(main_parser)
+      end
+
       def images
         self
       end
 
+      # Returns either the Facebook Open Graph image, twitter suggested image or
+      # the largest image in the image collection
+      def best
+        owner_suggested || largest
+      end
+
       # Returns the parsed image from Facebook's open graph property tags
       # Most major websites now define this property and is usually relevant
       # See doc at http://developers.facebook.com/docs/opengraph/
       # If none found, tries with Twitter image
-      def best
-        meta['og:image'] || meta['twitter:image'] || images_collection.first
+      def owner_suggested
+        meta['og:image'] || meta['twitter:image']
+      end
+
+      # Returns the largest image from the image collection,
+      # filtered for images that are more square than 10:1 or 1:10
+      def largest()
+        @larget_image ||= begin
+          img_nodes = parsed.search('//img')
+          sizes = img_nodes.map { |img_node| [URL.absolutify(img_node['src'], base_url), img_node['width'], img_node['height']] }
+          sizes.uniq! { |url, width, height| url }
+          if @download_images
+            sizes.map! do |url, width, height|
+              width, height = FastImage.size(url) if width.nil? || height.nil?
+              [url, width, height]
+            end
+          else
+            sizes.map! do |url, width, height|
+              width, height = [0, 0] if width.nil? || height.nil?
+              [url, width, height]
+            end
+          end
+          sizes.map! { |url, width, height| [url, width.to_i * height.to_i, width.to_f / height.to_f] }
+          sizes.keep_if { |url, area, ratio| ratio > 0.1 && ratio < 10 }
+          sizes.sort_by! { |url, area, ratio| -area }
+          url, area, ratio = sizes.first
+          url
+        end
       end
 
       # Return favicon url if exist

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = "4.1.0"
+  VERSION = "4.2.0"
 end

data/meta_inspector.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'faraday_middleware', '~> 0.9.1'
   gem.add_dependency 'faraday-cookie_jar', '~> 0.0.6'
   gem.add_dependency 'addressable', '~> 2.3.5'
+  gem.add_dependency 'fastimage'
 
   gem.add_development_dependency 'rspec', '2.14.1'
   gem.add_development_dependency 'fakeweb', '1.3.0'

data/spec/fixtures/largest_image_in_html.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200 OK
+Server: nginx/0.7.67
+Date: Fri, 18 Nov 2011 21:46:46 GMT
+Content-Type: text/html
+Connection: keep-alive
+Last-Modified: Mon, 14 Nov 2011 16:53:18 GMT
+Content-Length: 4987
+X-Varnish: 2000423390
+Age: 0
+Via: 1.1 varnish
+
+<html>
+  <head>
+    <title>An example page</title>
+  </head>
+  <body>
+    <img src="/too_narrow" width="10"  height="100" />
+    <img src="/smaller"    width="10"  height="10"  />
+    <img src="/largest"    width="100" height="100" />
+    <img src="/too_wide"   width="100" height="10"  />
+    <img src="/smallest"   width="1"   height="1"   />
+  </body>
+</html>

data/spec/fixtures/largest_image_using_image_size.response

@@ -0,0 +1,23 @@
+HTTP/1.1 200 OK
+Server: nginx/0.7.67
+Date: Fri, 18 Nov 2011 21:46:46 GMT
+Content-Type: text/html
+Connection: keep-alive
+Last-Modified: Mon, 14 Nov 2011 16:53:18 GMT
+Content-Length: 4987
+X-Varnish: 2000423390
+Age: 0
+Via: 1.1 varnish
+
+<html>
+  <head>
+    <title>An example page</title>
+  </head>
+  <body>
+    <img src="/10x100"   width="10"  height="100" />
+    <img src="/10x10" />
+    <img src="/100x100" />
+    <img src="/100x10"   width="100" height="10"  />
+    <img src="/1x1"      width="1"   height="1"   />
+  </body>
+</html>

data/spec/meta_inspector/images_spec.rb

@@ -70,7 +70,7 @@ describe MetaInspector do
     end
   end
 
-  describe "#image" do
+  describe "images.best" do
     it "should find the og image" do
       page = MetaInspector.new('http://www.theonion.com/articles/apple-claims-new-iphone-only-visible-to-most-loyal,2772/')
 
@@ -84,9 +84,49 @@ describe MetaInspector do
     end
 
     it "should find image when og:image and twitter:image metatags are missing" do
-      page = MetaInspector.new('http://www.alazan.com')
+      page = MetaInspector.new('http://example.com/largest_image_using_image_size')
 
-      page.images.best.should == "http://www.alazan.com/imagenes/logo.jpg"
+      page.images.best.should == "http://example.com/100x100"
+    end
+  end
+
+  describe "images.owner_suggested" do
+    it "should find the og image" do
+      page = MetaInspector.new('http://www.theonion.com/articles/apple-claims-new-iphone-only-visible-to-most-loyal,2772/')
+
+      page.images.owner_suggested.should == "http://o.onionstatic.com/images/articles/article/2772/Apple-Claims-600w-R_jpg_130x110_q85.jpg"
+    end
+
+    it "should find image on youtube" do
+      page = MetaInspector.new('http://www.youtube.com/watch?v=iaGSSrp49uc')
+
+      page.images.owner_suggested.should == "http://i2.ytimg.com/vi/iaGSSrp49uc/mqdefault.jpg"
+    end
+
+    it "should return nil when og:image and twitter:image metatags are missing" do
+      page = MetaInspector.new('http://example.com/largest_image_using_image_size')
+
+      page.images.owner_suggested.should be nil
+    end
+  end
+
+  describe "images.largest" do
+    it "should find the largest image on the page using html sizes" do
+      page = MetaInspector.new('http://example.com/largest_image_in_html')
+
+      page.images.largest.should == "http://example.com/largest"
+    end
+
+    it "should find the largest image on the page using actual image sizes" do
+      page = MetaInspector.new('http://example.com/largest_image_using_image_size')
+
+      page.images.largest.should == "http://example.com/100x100"
+    end
+
+    it "should find the largest image without downloading images" do
+      page = MetaInspector.new('http://example.com/largest_image_using_image_size', download_images: false)
+
+      page.images.largest.should == "http://example.com/1x1"
     end
   end
 

data/spec/spec_helper.rb

@@ -31,6 +31,12 @@ FakeWeb.register_uri(:get, "http://example.com/", :response => fixture_file("exa
 # Used to test response status codes
 FakeWeb.register_uri(:get, "http://example.com/404", :response => fixture_file("404.response"))
 
+# Used to test largest image in page logic
+FakeWeb.register_uri(:get, "http://example.com/largest_image_in_html", :response => fixture_file("largest_image_in_html.response"))
+FakeWeb.register_uri(:get, "http://example.com/largest_image_using_image_size", :response => fixture_file("largest_image_using_image_size.response"))
+FakeWeb.register_uri(:get, "http://example.com/10x10", :response => fixture_file("10x10.jpg.response"))
+FakeWeb.register_uri(:get, "http://example.com/100x100", :response => fixture_file("100x100.jpg.response"))
+
 # These are older fixtures
 FakeWeb.register_uri(:get, "http://pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
 FakeWeb.register_uri(:get, "http://pagerankalert-shortcut.com", :response => fixture_file("pagerankalert-shortcut.com.response"))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-15 00:00:00.000000000 Z
+date: 2015-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.3.5
+- !ruby/object:Gem::Dependency
+  name: fastimage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -243,6 +257,8 @@ files:
 - meta_inspector.gemspec
 - spec/document_spec.rb
 - spec/exception_log_spec.rb
+- spec/fixtures/100x100.jpg.response
+- spec/fixtures/10x10.jpg.response
 - spec/fixtures/404.response
 - spec/fixtures/alazan.com.response
 - spec/fixtures/alazan_websolution.response
@@ -257,6 +273,8 @@ files:
 - spec/fixtures/international.response
 - spec/fixtures/invalid_href.response
 - spec/fixtures/iteh.at.response
+- spec/fixtures/largest_image_in_html.response
+- spec/fixtures/largest_image_using_image_size.response
 - spec/fixtures/malformed_href.response
 - spec/fixtures/markupvalidator_faqs.response
 - spec/fixtures/meta_tags.response