distillery 0.2.10 → 0.3.0

data/spec/lib/distillery/document_spec.rb

@@ -60,7 +60,7 @@ module Distillery
     end
 
     describe 'remove_unlikely_elements!' do
-      %w[combx comment disqus foot header meta nav rss shoutbox sidebar sponsor].each do |klass|
+      %w[combx comment community disqus extra foot header remark rss shoutbox sidebar sponsor ad-break agegate pagination pager popup].each do |klass|
         it "removes any elements classed .#{klass}, as it is unlikely to be page content" do
           doc = document_of("<div class='#{klass}'>foo</div>", :remove_unlikely_elements!)
           doc.inner_html.should == html_of("")
@@ -117,33 +117,33 @@ module Distillery
 
       it 'gives one point per comma in the text of an element' do
         doc = document_of("<p>foo,bar,baz</p>", :score!)
-        doc.scores['/html/body/p'].should == 4
+        doc.scores['/html/body/p'].should == 68.0
       end
 
       it 'gives one point per chunk of 100 characters, max of 3' do
         doc = document_of("<p>#{'f'*201}</p>", :score!)
-        doc.scores['/html/body/p'].should == 4
+        doc.scores['/html/body/p'].should == 68.0
 
         doc = document_of("<p>#{'f'*1000}</p>", :score!)
-        doc.scores['/html/body/p'].should == 5
+        doc.scores['/html/body/p'].should == 85.0
       end
 
       it 'adds its own points to its parent' do
         doc = document_of("<div><div>foo</div></div>", :score!)
-        doc.scores['/html/body/div/div'].should == 2
-        doc.scores['/html/body/div'].should == 2
+        doc.scores['/html/body/div/div'].should == 30.0
+        doc.scores['/html/body/div'].should == 34.0
       end
 
       it 'adds 1/2 its points to its grandparent' do
         doc = document_of("<div><div><div>foo</div></div></div>", :score!)
-        doc.scores['/html/body/div/div/div'].should == 2
-        doc.scores['/html/body/div/div'].should == 2
-        doc.scores['/html/body/div'].should == 1
+        doc.scores['/html/body/div/div/div'].should == 26
+        doc.scores['/html/body/div/div'].should == 30.0
+        doc.scores['/html/body/div'].should == 17.0
       end
 
       it 'scales the final score by the inverse link density' do
         doc = document_of("<p>foobar<a>baz</a></p>", :score!)
-        doc.scores['/html/body/p'].should == 1.3333333333333335
+        doc.scores['/html/body/p'].should == 22.666666666666668
       end
 
     end
@@ -182,11 +182,6 @@ module Distillery
         doc.search('.remove').should be_empty
       end
 
-      it 'removes elements that have way more li elements and it is not a list' do
-        doc = doc_with_top_scored_html_of("<div class='remove'><div>me<ul>#{'<li>a</li>'*200}</ul></div></div>", :clean_top_scoring_elements!)
-        doc.search('.remove').should be_empty
-      end
-
       it 'removes elements that have more inputs than 1/3 the amount of p tags' do
         doc = doc_with_top_scored_html_of("<div class='remove'><div><input><input><p>f</p><p>f</p><p>f</p></div></div>", :clean_top_scoring_elements!)
         doc.search('.remove').should be_empty
@@ -277,10 +272,11 @@ module Distillery
       end
 
       it 'picks the outtermost element in the event of a tie' do
-        doc = document_of("<div><div class='included'>#{'f,'*10}</div></div>")
-        doc.distill!.should =~ /included/
-        doc.scores['/html/body/div/div'].should == 11
-        doc.scores['/html/body/div'].should == 11
+        doc = document_of("<div><div class='is-included'>#{'f,'*10}</div><div class='also-included'>#{'f,'*10}</div></div>")
+        doc.distill!.should =~ /is-included/
+        doc.distill!.should =~ /also-included/
+        doc.scores['/html/body/div/div[1]'].should == 165.0
+        doc.scores['/html/body/div/div[2]'].should == 165.0
       end
 
       it 'returns sibling elements to the top scoring one that have > 25% of the top scoring element\'s score' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: distillery
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-07 00:00:00.000000000Z
+date: 2011-12-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2161473880 !ruby/object:Gem::Requirement
+  requirement: &2160714940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>'
@@ -21,10 +21,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2161473880
+  version_requirements: *2160714940
 - !ruby/object:Gem::Dependency
   name: slop
-  requirement: &2161473380 !ruby/object:Gem::Requirement
+  requirement: &2160714440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>'
@@ -32,10 +32,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2161473380
+  version_requirements: *2160714440
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2161472920 !ruby/object:Gem::Requirement
+  requirement: &2160713980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>'
@@ -43,10 +43,10 @@ dependencies:
         version: '2.0'
   type: :development
   prerelease: false
-  version_requirements: *2161472920
+  version_requirements: *2160713980
 - !ruby/object:Gem::Dependency
   name: guard
-  requirement: &2161472540 !ruby/object:Gem::Requirement
+  requirement: &2160713600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161472540
+  version_requirements: *2160713600
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2161472080 !ruby/object:Gem::Requirement
+  requirement: &2160713140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161472080
+  version_requirements: *2160713140
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &2161471660 !ruby/object:Gem::Requirement
+  requirement: &2160712720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161471660
+  version_requirements: *2160712720
 - !ruby/object:Gem::Dependency
   name: rb-fsevent
-  requirement: &2161471240 !ruby/object:Gem::Requirement
+  requirement: &2160712300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161471240
+  version_requirements: *2160712300
 - !ruby/object:Gem::Dependency
   name: growl
-  requirement: &2161470820 !ruby/object:Gem::Requirement
+  requirement: &2160711880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,7 +98,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161470820
+  version_requirements: *2160711880
 description: Distillery extracts the "content" portion out of an HTML document. It
   applies heuristics based on element type, location, class/id name and other attributes
   to try and find the content part of the HTML document and return it.
@@ -136,10 +136,13 @@ files:
 - spec/fixtures/ginger_cookies.html
 - spec/fixtures/js_this_keyword.html
 - spec/fixtures/maple_cookies.html
+- spec/fixtures/mothers_brisket.html
 - spec/fixtures/nyt_social_media.html
+- spec/fixtures/oyako.html
 - spec/fixtures/pina_collada_cupcakes.html
 - spec/fixtures/pumpkin_scones.html
 - spec/fixtures/rhubarb.html
+- spec/fixtures/swiss_chard_pie.html
 - spec/fixtures/tofu_bowl.html
 - spec/fixtures/vanilla_pound_cake.html
 - spec/lib/distillery/document_spec.rb
@@ -185,10 +188,13 @@ test_files:
 - spec/fixtures/ginger_cookies.html
 - spec/fixtures/js_this_keyword.html
 - spec/fixtures/maple_cookies.html
+- spec/fixtures/mothers_brisket.html
 - spec/fixtures/nyt_social_media.html
+- spec/fixtures/oyako.html
 - spec/fixtures/pina_collada_cupcakes.html
 - spec/fixtures/pumpkin_scones.html
 - spec/fixtures/rhubarb.html
+- spec/fixtures/swiss_chard_pie.html
 - spec/fixtures/tofu_bowl.html
 - spec/fixtures/vanilla_pound_cake.html
 - spec/lib/distillery/document_spec.rb