preadly-bulbasaur 0.8.2 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8260cd6a3e468ab1139c0096ee1cbb5086fe7ff1
-  data.tar.gz: 146d9e064d05db1eb3c19c7cdc7290c6027fa79a
+  metadata.gz: a87289e0e17c9e9ae9d8f489c3704a14dff5d5ac
+  data.tar.gz: 64d2ec14ed814afe1997c2472678f1af7c3b9166
 SHA512:
-  metadata.gz: 61721cf5b99c6edba03b754fe98b863bd9747ab0bd279d4b3023becfc160309a591d931a2b8b814aab69884c940e026c72d8dc52022769101ef9f79f4da90b49
-  data.tar.gz: f8da5d762adf404102767f71c0c68ea99a3f49fbbb716612f01e95f9fda6ab8a5c982068d35d14232bdb8d033bfdcefab69d628c36a3fbdeb54dab390ab490d7
+  metadata.gz: d74058dc3e6928672b8fb1342e2d7b81db562110cf7bfa491b470838de5f19bd6cbb8ee36e8d929cdb5e4a6e31c6c56b0dda4de758b18efc71214e0a1ae9671a
+  data.tar.gz: 63c9260e6f33a3a114d46c8ea119f7d026a6e5cebd655b31131af8a775a86dd6e63c6bab260bcc3f7fd55aae58eee05a063fd49a925a2e096c98ce908c144347

data/lib/bulbasaur.rb

@@ -11,6 +11,7 @@ require "bulbasaur/removals/remove_attributes"
 require "bulbasaur/replaces/replace_by_tag_image"
 require "bulbasaur/utils/normalize_url"
 require "bulbasaur/utils/normalize_image_sources"
+require "bulbasaur/utils/normalize_image_src_set"
 require "bulbasaur/version"
 
 

data/lib/bulbasaur/removals/remove_attributes.rb

@@ -1,14 +1,16 @@
 module Bulbasaur
   class RemoveAttributes
+    
     def initialize(html, banned_attrs)
       @html = html
       @banned_attrs = banned_attrs
     end
 
     def call
-      parsed_html = Nokogiri::HTML::DocumentFragment.parse @html
+      parsed_html = Nokogiri::HTML::DocumentFragment.parse(@html)
       @banned_attrs.each { |attr| parsed_html.xpath(".//@#{attr}").remove }
       parsed_html.to_s
     end
+
   end
-end
+end

data/lib/bulbasaur/removals/remove_tags.rb

@@ -1,14 +1,28 @@
 module Bulbasaur
   class RemoveTags
-    def initialize(html, banned_tags)
+    
+    SELECTORS_EMPTY_ANALYSE = "div, p"
+    
+    def initialize(html, banned_tags, remove_empty_tags = false)
       @html = html
       @banned_tags = banned_tags
+      @remove_empty_tags = remove_empty_tags
     end
-
+    
     def call
-      parsed_html = Nokogiri::HTML::DocumentFragment.parse @html
+      parsed_html = Nokogiri::HTML::DocumentFragment.parse(@html)
+      remove_empty_tags(parsed_html) if @remove_empty_tags
       @banned_tags.each { |tag| parsed_html.css(tag).remove }
       parsed_html.to_s
     end
+    
+    private
+    
+      def remove_empty_tags(parsed_html)
+        parsed_html.css(SELECTORS_EMPTY_ANALYSE).each do |tag|
+          tag.remove if tag.content.strip.empty? && tag.children.select{ |c| c.name != "text" }.length == 0
+        end
+      end
+
   end
-end
+end

data/lib/bulbasaur/utils/normalize_image_sources.rb

@@ -1,5 +1,6 @@
 module Bulbasaur
   class NormalizeImageSources
+    
     def initialize(html, target_attrs)
       @html = html
       @target_attrs = target_attrs
@@ -8,7 +9,7 @@ module Bulbasaur
     def call
       parsed_html = Nokogiri::HTML::DocumentFragment.parse @html
       parsed_html.css('img').each do |element|
-        check_for_attrs element
+        check_for_attrs(element)
       end
       parsed_html.to_s
     end
@@ -25,12 +26,12 @@ module Bulbasaur
     end
 
     def adjust(element, attr)
-      element.set_attribute 'src', element.xpath(attr).text
-      remove_target_attrs_from element
+      element.set_attribute('src', element.xpath(attr).text)
+      remove_target_attrs_from(element)
     end
 
     def remove_target_attrs_from(element)
       @target_attrs.each { |attr| element.xpath("@#{attr}").remove }
     end
   end
-end
+end

data/lib/bulbasaur/utils/normalize_image_src_set.rb

@@ -0,0 +1,68 @@
+module Bulbasaur
+  class NormalizeImageSrcSet 
+    
+    REGEX_FIND_WIDTH  = /(\d+)w/i
+    REGEX_FIND_HEIGHT = /(\d+)h/i
+    
+    def initialize(html)
+      @html = html
+    end
+    
+    def call
+      parsed_html = Nokogiri::HTML::DocumentFragment.parse(@html)
+      parsed_html.css("img[srcset]").each do |img|
+        image = extract_src_set_attribute(img)
+        img.set_attribute("src", image[:url])
+        img.set_attribute("width", image[:width]) if image[:width]
+        img.set_attribute("height", image[:height]) if image[:height]
+      end
+      parsed_html.to_s
+    end
+    
+    private
+    
+    def extract_src_set_attribute(img)
+      itens = img.get_attribute("srcset").split(",")
+      images = []
+      itens.each do |item|
+        srcset_item = item.split(" ")
+        image_object = {
+          url: extract_url(srcset_item), 
+          width: extract_width(srcset_item), 
+          height: extract_height(srcset_item)
+        }
+        images << image_object
+      end
+      get_better_image(images)
+    end
+    
+    def extract_width(itens)
+      extract_by_regex(itens, REGEX_FIND_WIDTH)
+    end
+    
+    def extract_height(itens)
+      extract_by_regex(itens, REGEX_FIND_HEIGHT)
+    end
+    
+    def extract_url(srcset_item)
+      srcset_item[0]
+    end
+    
+    def extract_by_regex(itens, regex)
+      value = itens.select{ |item| item =~ regex }.first
+      value = value.match(regex).captures.first if value
+      value
+    end
+    
+    def get_better_image(itens)
+      images = Array.new
+      images.concat itens.select{ |img| !img[:width] && !img[:height] }.each{ |img| img[:area] = 0 }
+      images.concat itens.select{ |img| !img[:width] &&  img[:height] }.each{ |img| img[:area] = img[:height] }
+      images.concat itens.select{ |img|  img[:width] && !img[:height] }.each{ |img| img[:area] = img[:width]  }
+      images.concat itens.select{ |img|  img[:width] &&  img[:height] }.each{ |img| img[:area] = img[:width].to_i * img[:height].to_i }  
+      images = images.sort { |a, b| b[:area] <=> a[:area] }
+      images.first
+    end
+
+  end
+end

data/lib/bulbasaur/version.rb

@@ -2,8 +2,8 @@ module Bulbasaur
 
   module Version
     MAJOR = 0
-    MINOR = 8
-    PATCH = 2
+    MINOR = 9
+    PATCH = 0
     STRING = "#{MAJOR}.#{MINOR}.#{PATCH}"
   end
   

data/spec/bulbasaur/removals/remove_tags_spec.rb

@@ -3,11 +3,15 @@ require 'spec_helper'
 RSpec.describe Bulbasaur::RemoveTags do
   
   subject do 
-    described_class.new(html, banned_tags).call
+    described_class.new(html, banned_tags, empty_tags).call
   end
 
   describe '#call' do
     
+    let(:empty_tags) do
+      false
+    end
+    
     let(:html) do
       %[
         <style>
@@ -18,6 +22,12 @@ RSpec.describe Bulbasaur::RemoveTags do
           <input type="text">
         </form>
         <p>hello!</p>
+        <div class="inner top"> 
+          <p></p>
+          <div> </div>
+        </div>
+        <div></div>
+        <p></p>
       ]
     end
 
@@ -26,25 +36,71 @@ RSpec.describe Bulbasaur::RemoveTags do
       let(:banned_tags) do
         []
       end
-
+    
       it 'returns the HTML code as it was before' do
         expect(subject).to eq html
       end
     end
-
+    
     context 'when there are banned tags' do
       
       let(:banned_tags) do
         %w(form style)
       end
-
+    
       it 'returns the HTML code without the banned tags' do
         expect(subject.strip.gsub(/\n/, '').squeeze ' ').to eq %[
           <div style="height: 100px; width: 100px;"></div>
           <p>hello!</p>
+          <div class="inner top"> 
+            <p></p>
+            <div> </div>
+          </div>
+          <div></div>
+          <p></p>
         ].strip.gsub(/\n/, '').squeeze ' '
       end
     end
-  end
+    
+    context 'when the HTML code with the banned tags and defined no empty tags' do
+    
+      let(:html) do
+      %[
+        <style>
+          div { color: green; width: 1024px; }
+        </style>
+        <div style="height: 100px; width: 100px;"></div>
+        <form>
+          <input type="text">
+        </form>
+        <p>hello!</p>
+        <div class="inner top"> 
+          <p></p>
+          <div><img src='test.jpg'></div>
+        </div>
+        <div></div>
+        <p>        </p>
+        <div class="helo">         </div>
+        <p></p>
+      ]
+     end
+    
+      let(:banned_tags) do
+        %w(form style)
+      end
 
+      let(:empty_tags) do
+        true
+      end
+    
+      it 'returns the HTML code without the banned tags and without empty tags' do
+        expect(subject.strip.gsub(/\n/, '').squeeze ' ').to eq %[
+          <p>hello!</p>
+          <div class="inner top"> 
+          <div><img src="test.jpg"></div>
+          </div>
+        ].strip.gsub(/\n/, '').squeeze ' '
+      end
+    end
+  end
 end

data/spec/bulbasaur/utils/normalize_image_srcset_spec.rb

@@ -0,0 +1,122 @@
+require "spec_helper"
+
+RSpec.describe Bulbasaur::NormalizeImageSrcSet do
+
+  subject do
+    described_class.new(html).call
+  end
+
+  describe "#call" do
+    
+    context "When there are no srcset attribute" do
+    
+      let(:html) do
+        <<-HTML
+        <img src="http://somewhere.to/get/a-pixel.gif" data-lazy-src="http://somewhere.to/get/the-real-image.jpg" alt="Image" width="800" height="1200">
+        <img lazy-data="https://place.where/an-image/has/no-src-tag.jpg">
+        HTML
+      end
+    
+      it "Does returns the HTML code as it was before" do
+        expect(subject).to eq(html)
+      end 
+    end
+    
+    context "When there are srcset attribute without size defined" do
+    
+      let(:html) do
+        <<-HTML
+        <img src="http://somewhere.to/get/a-pixel.gif" srcset="http://bulbasaur.com/imageA.jpg" alt="Image" width="800" height="1200">
+        <img srcset="http://bulbasaur.com/imageB.jpg">
+        <img srcset="http://bulbasaur.com/imageC.jpg,http://bulbasaur.com/imageC.jpg,http://bulbasaur.com/imageD.jpg">
+        HTML
+      end
+      
+      let(:html_normalized) do
+        <<-HTML
+        <img src="http://bulbasaur.com/imageA.jpg" srcset="http://bulbasaur.com/imageA.jpg" alt="Image" width="800" height="1200">
+        <img srcset="http://bulbasaur.com/imageB.jpg" src="http://bulbasaur.com/imageB.jpg">
+        <img srcset="http://bulbasaur.com/imageC.jpg,http://bulbasaur.com/imageC.jpg,http://bulbasaur.com/imageD.jpg" src="http://bulbasaur.com/imageC.jpg">
+        HTML
+      end
+    
+      it "Does returns the HTML code with src replaced by srcset attribute" do
+        expect(subject).to eq(html_normalized)
+      end
+    end
+    
+    context "When there are srcset attribute with size defined" do
+    
+      let(:html) do
+        <<-HTML
+        <img src="http://somewhere.to/get/a-pixel.gif" srcset="http://bulbasaur.com/imageA.jpg 100w" alt="Image" width="800" height="1200">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200w">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200h">
+        <img srcset="http://bulbasaur.com/imageB.jpg 20w 30h">
+        <img srcset="http://bulbasaur.com/imageB.jpg 2x">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200W 100H">
+        <img srcset="http://bulbasaur.com/imageC.jpg 100w,http://bulbasaur.com/imageC.jpg 200w,http://bulbasaur.com/imageD.jpg 300w">
+        HTML
+      end
+      
+      let(:html_normalized) do
+        <<-HTML
+        <img src="http://bulbasaur.com/imageA.jpg" srcset="http://bulbasaur.com/imageA.jpg 100w" alt="Image" width="100" height="1200">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200w" src="http://bulbasaur.com/imageB.jpg" width="200">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200h" src="http://bulbasaur.com/imageB.jpg" height="200">
+        <img srcset="http://bulbasaur.com/imageB.jpg 20w 30h" src="http://bulbasaur.com/imageB.jpg" width="20" height="30">
+        <img srcset="http://bulbasaur.com/imageB.jpg 2x" src="http://bulbasaur.com/imageB.jpg">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200W 100H" src="http://bulbasaur.com/imageB.jpg" width="200" height="100">
+        <img srcset="http://bulbasaur.com/imageC.jpg 100w,http://bulbasaur.com/imageC.jpg 200w,http://bulbasaur.com/imageD.jpg 300w" src="http://bulbasaur.com/imageD.jpg" width="300">
+        HTML
+      end
+    
+      it "Does return the HTML code with src and with definitions" do
+        expect(subject).to eq(html_normalized)
+      end
+    end
+    
+    context "When there are no valid params on srcset" do
+      
+      let(:html) do
+        <<-HTML
+        <img srcset="http://bulbasaur.com/imageA.jpg 100w 200H 12h asd hausd  uahsd 12813nusda" alt="Image">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200w 1x 10x">
+        <img srcset="http://bulbasaur.com/imageC.jpg 100w 50h,http://bulbasaur.com/imageC.jpg 600w 500h,http://bulbasaur.com/imageD.jpg 300w 400h">
+        HTML
+      end
+    
+      let(:html_normalized) do
+        <<-HTML
+        <img srcset="http://bulbasaur.com/imageA.jpg 100w 200H 12h asd hausd  uahsd 12813nusda" alt="Image" src="http://bulbasaur.com/imageA.jpg" width="100" height="200">
+        <img srcset="http://bulbasaur.com/imageB.jpg 200w 1x 10x" src="http://bulbasaur.com/imageB.jpg" width="200">
+        <img srcset="http://bulbasaur.com/imageC.jpg 100w 50h,http://bulbasaur.com/imageC.jpg 600w 500h,http://bulbasaur.com/imageD.jpg 300w 400h" src="http://bulbasaur.com/imageC.jpg" width="600" height="500">
+        HTML
+      end
+      
+      it "Does returns the HTML code and ignores invalid params" do
+        expect(subject).to eq(html_normalized)
+      end
+    end
+    
+    context "When there are diferents sizes to the same image" do
+      
+      let(:html) do
+        <<-HTML
+        <img srcset="http://bulbasaur.com/imageC.jpg 100w 50h,http://bulbasaur.com/imageC.jpg 200w 300h,http://bulbasaur.com/imageD.jpg 300w 400h">
+        HTML
+      end
+      
+      let(:html_normalized_) do
+        <<-HTML
+        <img srcset="http://bulbasaur.com/imageC.jpg 100w 50h,http://bulbasaur.com/imageC.jpg 200w 300h,http://bulbasaur.com/imageD.jpg 300w 400h" src="http://bulbasaur.com/imageD.jpg" width="300" height="400">
+        HTML
+      end
+      
+      it "Does returns the HTML code with better image" do
+        expect(subject).to eq(html_normalized_)
+      end
+    end
+    
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: preadly-bulbasaur
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.9.0
 platform: ruby
 authors:
 - Magno Costa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-15 00:00:00.000000000 Z
+date: 2015-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -101,6 +101,7 @@ files:
 - lib/bulbasaur/removals/remove_tags.rb
 - lib/bulbasaur/replaces/replace_by_tag_image.rb
 - lib/bulbasaur/utils/normalize_image_sources.rb
+- lib/bulbasaur/utils/normalize_image_src_set.rb
 - lib/bulbasaur/utils/normalize_url.rb
 - lib/bulbasaur/version.rb
 - spec/bulbasaur/extracts/extract_images_from_all_resources_spec.rb
@@ -114,6 +115,7 @@ files:
 - spec/bulbasaur/removals/remove_tags_spec.rb
 - spec/bulbasaur/replaces/replace_by_tag_image_spec.rb
 - spec/bulbasaur/utils/normalize_image_sources_spec.rb
+- spec/bulbasaur/utils/normalize_image_srcset_spec.rb
 - spec/bulbasaur/utils/normalize_url_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/preadly/bulbasaur
@@ -135,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Bulbasaur is a helper for crawler operations used in Pread.ly
@@ -151,5 +153,6 @@ test_files:
 - spec/bulbasaur/removals/remove_tags_spec.rb
 - spec/bulbasaur/replaces/replace_by_tag_image_spec.rb
 - spec/bulbasaur/utils/normalize_image_sources_spec.rb
+- spec/bulbasaur/utils/normalize_image_srcset_spec.rb
 - spec/bulbasaur/utils/normalize_url_spec.rb
 - spec/spec_helper.rb