webtractor 0.0.1 → 0.0.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZWVkNmUwOWUwYTdjYmMyMDYwZWVjOWNhNWE1MGYyZmJmNGNjMzRkNw==
+    YjAwMDM5NDJhMzg4ODdkNThkZWQ5NmRiNGQzMzM0NDNhNjljMTkyZA==
   data.tar.gz: !binary |-
-    ODlmZWZiNDFjZjg0ZDI4ODRjYjk2ZWVkZDY0YjBkNDMxY2VlMDk3Nw==
+    MzkzOWM4YTgyMDg1YWJlZmJiMmQxZTRlYWQ5ZTZmNDAwYmI2MjYzYw==
 SHA512:
   metadata.gz: !binary |-
-    YTk4NzRiNTkxNDVkZmJhNDkxZDFkNmUwODVkZThmYjc1MDA3MDk2ZjZlMTg1
-    YjBmNzM2NjU5NGU2Y2RjYjkzNzRkZmEwZDcwZTk1NzUyNzVkNThlOTBjMGNi
-    MGJhOWY5YWRiNjhlOTc3OTFhMGMxY2IxMTFhY2QwNDVjZGRlNjI=
+    MDNhNTY1MzI2NDVjMzVmMzRmN2NjYTNiYTYxZDI3NjQ0Y2Y4NjllZmVkYTJh
+    ZmY5NzkyOWQ1YmNkZWRjMmE1OTM0MTBhMTU4NDU0MDA2MDI3NmI0MjEyY2I5
+    Yjg3YjZlYjEzMmE2YTdjM2MzOTM2ZjY0NTBkZjI3ZWU5YmJkNGY=
   data.tar.gz: !binary |-
-    OWNlOTdjMTg2MDA3YzhhYzkwMDE3OTU0NjUzZDllZDY5M2FkY2NjOWZjYjAx
-    ZTk2MDJmODc5OWQ0ZGZjZjIzYzA4YmU4NTQ4MWY3M2E4ZTg3NjY0ODE4ZjM1
-    YTlmMWZiOWIxNzlmZWI2YzY5MzllMDVmZTFhNzJlMzZkZjViODk=
+    YjMzMWRjY2QyNDk2OTBlNjU1OTBmMDlhNzQ1NTdkYWQ5ZWFlNjljOWMxZjg1
+    ZThhZGY0MWU2ODRlOWY2YTg0MGEyNjUzZTRmM2FjZDhmZThlYWM0YjgwZTAw
+    ODRmZWMzOTViZTVmMWRmM2Q5ZGI5YjJmOTA0Mzk3MzczYTg4YmU=

data/README.md

@@ -1,7 +1,7 @@
 # Webtractor
 
 The Webtractor is a ruby library which is able to extract main content
-from webpages like news, blogs, etc. As a result you can just a main
+from webpages like news, blogs, etc. As a result you can just have a main
 content without any boilerplate (menu, footer, comments, etc).
 
 ## Installation
@@ -89,4 +89,4 @@ extractor.add_filter RemoveBolds.new
 
 ## License
 
-This library is distributed under the Bearware license.
+This library is distributed under the Beerware license.

data/lib/webtractor.rb

@@ -16,6 +16,7 @@ require 'webtractor/filters/remove_comments'
 require 'webtractor/filters/remove_noncontent'
 require 'webtractor/filters/remove_menus'
 require 'webtractor/filters/remove_footer'
+require 'webtractor/filters/remove_header'
 require 'webtractor/filters/remove_embeds'
 require 'webtractor/filters/remove_smallest'
 require 'webtractor/filters/remove_empty'

data/lib/webtractor/filters/default_filter.rb

@@ -11,6 +11,7 @@ module Webtractor::Filters
         RemoveNoncontent.new,
         RemoveMenus.new,
         RemoveFooter.new,
+        RemoveHeader.new,
         RemoveEmbeds.new,
         RemoveSmallest.new,
         RemoveEmpty.new,

data/lib/webtractor/filters/remove_embeds.rb

@@ -3,6 +3,7 @@ module Webtractor::Filters
     def process page
       page.css('embed').remove
       page.css('object').remove
+      page.xpath('//*[contains(@class, "video")]').remove
       page
     end
   end

data/lib/webtractor/filters/remove_footer.rb

@@ -2,6 +2,8 @@ module Webtractor::Filters
   class RemoveFooter
     def process page
       page.css('footer').remove
+      page.xpath('//*[contains(@id, "footer")]').remove
+      page.xpath('//*[contains(@class, "footer")]').remove
       page
     end
   end

data/lib/webtractor/filters/remove_header.rb

@@ -0,0 +1,10 @@
+module Webtractor::Filters
+  class RemoveHeader
+    def process page
+      page.css('header').remove
+      page.xpath('//*[contains(@class, "header")]').remove
+      page.xpath('//*[contains(@id, "header")]').remove
+      page
+    end
+  end
+end

data/lib/webtractor/filters/remove_menus.rb

@@ -2,8 +2,12 @@ module Webtractor::Filters
   class RemoveMenus
     def process page
       page.css('nav').remove
+      page.css('#pane').remove
+      page.css('#carousel').remove
+      page.css('#sidebar').remove
       page.css('.pane').remove
       page.css('.carousel').remove
+      page.css('.sidebar').remove
 
       page.css('ul').each do |ul|
         li_count = ul.css('li').count
@@ -17,7 +21,6 @@ module Webtractor::Filters
     end
 
     def explore path, node
-
       path += "/#{node.name}"
 
       node.children.each do |child|
@@ -27,7 +30,8 @@ module Webtractor::Filters
       return if node.name == 'p'
 
       links_count = node.xpath('./a').size
-      if links_count > 0 && links_count.to_f/node.children.count > 0.3
+
+      if links_count > 0 && links_count.to_f/node.children.count >= 0.3
         node.remove
       end
     end

data/lib/webtractor/result.rb

@@ -7,5 +7,9 @@ module Webtractor
       @text = xml.text
       @xml = xml
     end
+
+    def save filename
+      File.write(filename, @xml.to_s)
+    end
   end
 end

data/lib/webtractor/version.rb

@@ -1,3 +1,3 @@
 module Webtractor
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webtractor
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-25 00:00:00.000000000 Z
+date: 2014-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -73,6 +73,7 @@ files:
 - lib/webtractor/filters/remove_empty.rb
 - lib/webtractor/filters/remove_footer.rb
 - lib/webtractor/filters/remove_forms.rb
+- lib/webtractor/filters/remove_header.rb
 - lib/webtractor/filters/remove_images.rb
 - lib/webtractor/filters/remove_menus.rb
 - lib/webtractor/filters/remove_noncontent.rb