link_oracle 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3be685ce4c88a7b4bb145386e83a7abe75182002
-  data.tar.gz: 8f9f0067936d8d8b330c8d44d2bd302fa18ec10e
+  metadata.gz: 66b779546f8f2c837e044e40d161365baf2ca004
+  data.tar.gz: 96689ad4ed3ac24d666b098c3ea2d4d9c057eb99
 SHA512:
-  metadata.gz: a4b7e2c94808c71a5140aa96b57046f58d6cc4b04be79acb27f7ba7a718034b81ccaa60755d7c0ff391feaca4d88968e349b89fa19620a5f688a395aa1bc4f9d
-  data.tar.gz: ac2fafb2410fc9c2cb0b979f27b0f2f3b2d3e7a55ca24a53b8700b725039b0c4b1b2e8cc9bf4d1f5db7bdaa67d85133151ffadeda95f26d62de457a602ccbdbb
+  metadata.gz: 6649f8b62eba20a02792fea7ac4b7b9018fe73c5a2fb3f8dee7587664cf561b4f7e8b5a54e4fbfa02202e019f7e30a8e414c7d708a7bed751a4cd5a26b3b787c
+  data.tar.gz: f22445e5ea8b082f61ee9c15995d4da57fd05e722f16dfe673e0f114c8f1da6bd9f95412b90025e8154b5c4f8947b9290265842edee4bd0564712ec84e2b60d2

data/lib/link_oracle.rb

@@ -1,6 +1,7 @@
 require 'curb'
 require 'nokogiri'
 require 'uri'
+require 'fastimage'
 
 require 'link_oracle/extractor/base'
 require 'link_oracle/request'

data/lib/link_oracle/Utils/image_url_formatter.rb

@@ -9,7 +9,13 @@ module Utils
 
     def perform
       return unless image_url
-      invalid_url? ? "#{scheme}://#{host}#{encoded_image_url}" : encoded_image_url
+      if host_missing?
+        "#{scheme}://#{host}#{encoded_image_url}"
+      elsif scheme_missing?
+        "http:#{encoded_image_url}"
+      else
+        encoded_image_url
+      end
     end
 
     def encoded_image_url
@@ -32,7 +38,11 @@ module Utils
       @parsed_image_url ||= URI.parse(encoded_image_url)
     end
 
-    def invalid_url?
+    def scheme_missing?
+      parsed_image_url.scheme.to_s.empty?
+    end
+
+    def host_missing?
       !parsed_image_url.host
     end
   end

data/lib/link_oracle/extractor/base.rb

@@ -1,10 +1,11 @@
 class LinkOracle
   module Extractor
     class Base
-      attr_reader :parsed_body, :link_data
+      attr_reader :parsed_body, :url, :link_data
 
-      def initialize(parsed_body)
-        @parsed_body = parsed_body
+      def initialize(parsed_url)
+        @parsed_body = parsed_url[:parsed_data]
+        @url = parsed_url[:url]
         @link_data = LinkData::Data.new
       end
 

data/lib/link_oracle/extractor/body.rb

@@ -1,11 +1,8 @@
 class LinkOracle
   module Extractor
-    class Body
-      attr_reader :parsed_body, :link_data
-
-      def initialize(parsed_body)
-        @parsed_body = parsed_body
-        @link_data = LinkData::Data.new
+    class Body < Base
+      def type
+        :body
       end
 
       def perform
@@ -14,7 +11,6 @@ class LinkOracle
           image_urls: images,
           descriptions: descriptions
         })
-
       end
 
       def titles
@@ -24,9 +20,30 @@ class LinkOracle
       end
 
       def images
-        @images ||= parsed_body.xpath(
-          "//img[@src[contains(.,'://') and not(contains(.,'ads.') or contains(.,'ad.') or contains(.,'?'))]]"
-        ).first(3).compact.map{ |node| node['src'] }
+        @images ||= valid_size_images
+      end
+
+      def parsed_images
+        @parsed_images ||= parsed_body.xpath(
+          "//img[@src[(contains(.,'://') or contains(., '/')) and not(contains(.,'ads.') or contains(.,'ad.') or contains(.,'?') or contains(.,'.gif'))]]"
+        ).map{ |node| node['src'] }
+      end
+
+      def formatted_images
+        parsed_images.map { |image_url| ::Utils::ImageUrlFormatter.new(url, image_url).perform }
+      end
+
+      def valid_size_images
+        formatted_images.select do |image|
+          size = image_size(image)
+          size[0] >= 100 && size[1] >= 100 if size
+        end
+      end
+
+      def image_size(image)
+        ::FastImage.size(image)
+      rescue ::URI::InvalidURIError
+        [0, 0]
       end
 
       def descriptions
@@ -34,4 +51,4 @@ class LinkOracle
       end
     end
   end
-end
+end

data/lib/link_oracle/link_data.rb

@@ -1,10 +1,9 @@
 class LinkOracle
   class LinkData
-    attr_reader :parsed_data, :url
+    attr_reader :parsed_url
 
     def initialize(parsed_url)
-      @parsed_data = parsed_url[:parsed_data]
-      @url = parsed_url[:url]
+      @parsed_url = parsed_url
     end
 
     #TODO: Need to write tests for these
@@ -17,19 +16,19 @@ class LinkOracle
     end
 
     def image_url
-      Utils::ImageUrlFormatter.new(url, og.image_url || meta.image_url || body.image_url).perform
+      og.image_url || meta.image_url || body.image_url
     end
 
     def og
-      @og ||= Extractor::OG.new(parsed_data).perform
+      @og ||= Extractor::OG.new(parsed_url).perform
     end
 
     def meta
-      @meta ||= Extractor::Meta.new(parsed_data).perform
+      @meta ||= Extractor::Meta.new(parsed_url).perform
     end
 
     def body
-      @body ||= Extractor::Body.new(parsed_data).perform
+      @body ||= Extractor::Body.new(parsed_url).perform
     end
   end
 end

data/lib/link_oracle/link_data/data.rb

@@ -12,6 +12,10 @@ class LinkOracle
         image_urls.first
       end
 
+      def format_image(image)
+        Utils::ImageUrlFormatter.new(url, image)
+      end
+
       def title
         titles.first
       end

data/lib/link_oracle/version.rb

@@ -1,3 +1,3 @@
 class LinkOracle
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/link_oracle.gemspec

@@ -7,7 +7,7 @@ require 'link_oracle/version'
 Gem::Specification.new do |spec|
   spec.name          = "link_oracle"
   spec.version       = LinkOracle::VERSION
-  spec.authors       = ["Ian Cooper", 'Fito von Zastrow', 'Kane Baccigalupi', 'Sowjanya Mudunuri']
+  spec.authors       = ["Ian Cooper", 'Fito von Zastrow', 'Kane Baccigalupi', 'Sowjanya Mudunuri', 'Rae Bonfanti', 'Zoe Madden-Wood']
   spec.email         = ["developers@socialchorus.com"]
   spec.description   = %q{Scrapes pages for open graph, meta, and lastly, body preview data}
   spec.summary       = %q{Scrapes pages for open graph, meta, and lastly, body preview data}
@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'nokogiri'
   spec.add_dependency 'curb'
+  spec.add_dependency 'fastimage'
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

data/spec/link_oracle/Utils/image_url_formatter_spec.rb

@@ -2,19 +2,31 @@ require 'spec_helper'
 
 describe Utils::ImageUrlFormatter do
   let(:url) { "http://berkin.com/whatever/else/is/here" }
-  let(:image_url) { "/some/stupid/path" }
   let(:formatted_url) { Utils::ImageUrlFormatter.new(url, image_url).perform }
 
-  context 'scheme is http' do
-    it 'should return the image as a full url using the host as domain' do
-      formatted_url.should == 'http://berkin.com/some/stupid/path'
+  context 'the host is missing from the image url' do
+    let(:image_url) { "/some/stupid/path" }
+
+    context 'scheme is http' do
+      it 'should return the image as a full url using the host as domain' do
+        expect(formatted_url).to eq('http://berkin.com/some/stupid/path')
+      end
+    end
+
+    context 'scheme is https' do
+      let(:url) { "https://berkin.com/whatever/else/is/here" }
+
+      it 'should return the image as a full url using the host as domain' do
+        expect(formatted_url).to eq('https://berkin.com/some/stupid/path')
+      end
     end
   end
 
-  context 'scheme is https' do
-    let(:url) { "https://berkin.com/whatever/else/is/here" }
-    it 'should return the image as a full url using the host as domain' do
-      formatted_url.should == 'https://berkin.com/some/stupid/path'
+  context 'but the host is present, but the scheme is missing' do
+    let(:image_url) { "//berkin.com/some/stupid/path" }
+
+    it 'should return the image as a full url using http as the protocol' do
+      expect(formatted_url).to eq('http://berkin.com/some/stupid/path')
     end
   end
 
@@ -27,7 +39,7 @@ describe Utils::ImageUrlFormatter do
     end
 
     it 'returns nil' do
-      formatted_url.should == nil
+      expect(formatted_url).to be_nil
     end
   end
 end

data/spec/link_oracle/extractor/body_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe LinkOracle::Extractor::Body do
   let(:parsed_body) { ::Nokogiri::HTML.parse(body) }
-  let(:link_data) { LinkOracle::Extractor::Body.new(parsed_body).perform }
+  let(:link_data) { LinkOracle::Extractor::Body.new({ url: 'http://berkin.com', parsed_data: parsed_body}).perform }
 
   let(:body) {
     <<-HTML
@@ -34,6 +34,10 @@ describe LinkOracle::Extractor::Body do
   }
 
   describe 'perform' do
+    before do
+      FastImage.stub(:size).and_return([0, 0])
+    end
+
     context 'there is no suitable stuff in the body' do
       let(:body) {
         "<html>
@@ -58,14 +62,6 @@ describe LinkOracle::Extractor::Body do
         ]
       end
 
-      it 'should populate link_data image_urls' do
-        link_data.image_urls.should == [
-          "http://berkin.com",
-          "http://cherbin.com",
-          "http://flerbin.com"
-        ]
-      end
-
       it 'should populate link_data descriptions' do
         link_data.descriptions.should == [
           "paragraph 1",
@@ -73,6 +69,41 @@ describe LinkOracle::Extractor::Body do
           "paragraph 3"
         ]
       end
+
+      context 'images are a correct size' do
+        before do
+          FastImage.stub(:size).and_return([100, 121])
+        end
+
+        it 'should populate link_data image_urls' do
+          expect(link_data.image_urls).to match_array([
+            "http://berkin.com",
+            "http://cherbin.com",
+            "http://flerbin.com",
+            "http://berkin.com/berkin/cherbin.jpg"
+          ])
+        end
+      end
+
+      context 'images are incorrect size' do
+        it 'should populate link_data image_urls' do
+          expect(link_data.image_urls).to eq([])
+        end
+      end
+
+      context 'some images are correct size and some are not' do
+        it 'should populate link_data image_urls only with the correctly sized images' do
+          FastImage.should_receive(:size).with("http://berkin.com").and_return([50, 60])
+          FastImage.should_receive(:size).with("http://flerbin.com").and_return([60, 55])
+          FastImage.should_receive(:size).with("http://cherbin.com").and_return([160, 155])
+          FastImage.should_receive(:size).with("http://berkin.com/berkin/cherbin.jpg").and_return([160, 155])
+
+          expect(link_data.image_urls).to match_array([
+            "http://cherbin.com",
+            "http://berkin.com/berkin/cherbin.jpg"
+          ])
+        end
+      end
     end
   end
-end
+end

metadata

@@ -1,17 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: link_oracle
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Ian Cooper
 - Fito von Zastrow
 - Kane Baccigalupi
 - Sowjanya Mudunuri
+- Rae Bonfanti
+- Zoe Madden-Wood
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-26 00:00:00.000000000 Z
+date: 2014-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -41,6 +43,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fastimage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -147,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.3.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Scrapes pages for open graph, meta, and lastly, body preview data