distillery 0.2.1 → 0.2.3

data/README.md

@@ -45,6 +45,10 @@ In its cleaning, Distillery will also remove all `<img>` tags from the content e
 Distillery also ships with an executable that allows you to distill documents at the command line:
 
     Usage: distill [options] http://www.example.com/
+
+    options:
+
         -d, --dirty        Do not clean content HTML
+        -i, --images       Keep images in the content HTML
         -v, --version      Print the version
         -h, --help         Print this help message

data/lib/distillery/document.rb

@@ -107,12 +107,12 @@ module Distillery
       top_scoring_elements.each do |element|
 
         element.search("*").each do |node|
-          next if (node.name == 'img' || node.children.css('img').any?) && keep_images
+          next if contains_content_image?(node) && keep_images
           node.remove if has_empty_text?(node)
         end
 
         element.search("*").each do |node|
-          next if node.name == 'img' && keep_images
+          next if contains_content_image?(node) && keep_images
           if UNRELATED_ELEMENTS.include?(node.name) ||
             (node.text.count(',') < 2 && unlikely_to_be_content?(node))
             node.remove
@@ -123,6 +123,10 @@ module Distillery
 
     private
 
+    def contains_content_image?(node)
+      node.name == 'img' || node.children.css('img').length == 1
+    end
+
     def scorable_elements
       search('[data-distillery=scorable]')
     end
@@ -148,9 +152,7 @@ module Distillery
       top_elements = [top_element]
 
       top_element.parent.children.each do |sibling|
-        if scores[sibling.path] > top_score*0.25 && sibling.path != top_element.path
-          top_elements << sibling
-        end
+        top_elements << sibling if related_sibling?(top_element, sibling)
       end
 
       top_elements.each do |element|
@@ -162,6 +164,19 @@ module Distillery
       top_elements
     end
 
+    def related_sibling?(top_element, sibling)
+      score = scores[sibling.path]
+      top_score = scores[top_element.path]
+      identical = identical_attrubutes?(top_element, sibling)
+
+      related = (score > top_score*0.25 && sibling.path != top_element.path) ||
+                (identical && score > top_score*0.05)
+    end
+
+    def identical_attrubutes?(a, b)
+      a['id'] == b['id'] && a['class'] == b['class']
+    end
+
     def scorable_div?(elem)
       elem.name == 'div' &&
         (has_no_block_children?(elem) || has_only_empty_div_children?(elem))

data/lib/distillery/version.rb

@@ -1,3 +1,3 @@
 module Distillery
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

data/spec/acceptance_spec.rb

@@ -115,4 +115,9 @@ distillation_of 'bourbon_balls.html' do
   should_not =~ /I just tried the recipe forCellar Doo/       # Comments
   should_not =~ /FIND A STATION/                              # Header
   should_not =~ /Car Talk/                                    # Footer
+end
+
+distillation_of 'bulgogi.html' do
+  should =~ /early-season barbecue/
+  should =~ /Still, it is American to not fuss/
 end