webtractor 0.0.2 → 0.0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjAwMDM5NDJhMzg4ODdkNThkZWQ5NmRiNGQzMzM0NDNhNjljMTkyZA==
+    MGJlMDlhZDk0NmI2NTkxZGYwMTBjZjA0MGI1OGI1ODY5N2YzZDMxZQ==
   data.tar.gz: !binary |-
-    MzkzOWM4YTgyMDg1YWJlZmJiMmQxZTRlYWQ5ZTZmNDAwYmI2MjYzYw==
+    OTRmZjhmY2NmZmJlYjQ4ZjU5NTkzZDU4M2E0ZDJiYTQ2MzQ1Y2Y3YQ==
 SHA512:
   metadata.gz: !binary |-
-    MDNhNTY1MzI2NDVjMzVmMzRmN2NjYTNiYTYxZDI3NjQ0Y2Y4NjllZmVkYTJh
-    ZmY5NzkyOWQ1YmNkZWRjMmE1OTM0MTBhMTU4NDU0MDA2MDI3NmI0MjEyY2I5
-    Yjg3YjZlYjEzMmE2YTdjM2MzOTM2ZjY0NTBkZjI3ZWU5YmJkNGY=
+    YzhiMmU4M2YyN2NmNGU3YmYyYjYyMzRiYjFjMjNjYTY3ZTg3ZmQzOTdkMzA4
+    M2MzMDI4MmIzNDA5YjEyYmRhY2Y2ZTZhZTI2ZjczZDYzMTc2NzRiMmMxOTBi
+    OWI0MTVhZWYwZjM3Y2JiOGJkOWRmOWMxYWUwYzEyNWMyYmJmMjI=
   data.tar.gz: !binary |-
-    YjMzMWRjY2QyNDk2OTBlNjU1OTBmMDlhNzQ1NTdkYWQ5ZWFlNjljOWMxZjg1
-    ZThhZGY0MWU2ODRlOWY2YTg0MGEyNjUzZTRmM2FjZDhmZThlYWM0YjgwZTAw
-    ODRmZWMzOTViZTVmMWRmM2Q5ZGI5YjJmOTA0Mzk3MzczYTg4YmU=
+    YTJjYzM5YTI0NTMyNzliOTkxOWEwYTQzYTRjYjVmMDI1N2YyOGM5Njc1ODE1
+    NzBkNWFjMzVkNjE2YjdkOTkwMDc5ZTkxMDJkM2QyOWNmNWRkODY4NTg1OTQy
+    MWE4MWEzNTEzNTc4Y2IwNjA1MWI0NjQ4OWNhOWE1Mzc2ODEwZjU=

data/lib/webtractor.rb

@@ -1,4 +1,5 @@
 require 'open-uri'
+require 'mechanize'
 require 'nokogiri'
 require 'cachy'
 require 'moneta'

data/lib/webtractor/extractor.rb

@@ -3,6 +3,7 @@ module Webtractor
     attr_accessor :filters
 
     def initialize params={}
+      @agent = params[:agent] || Mechanize.new
       @filters = params[:filters] || [Filters::DefaultFilter.new]
       @cache = params[:cache] || false
       @cache_params = params[:cache_params] || {}
@@ -14,15 +15,16 @@ module Webtractor
 
     def extract_from_xml page
       title = page.xpath('//head/title').text
+      body = page.at('body')
       @filters.each do |filter|
-        page = filter.process(page)
+        body = filter.process(body)
       end
-      Result.new(title, page)
+      Result.new(title, body)
     end
 
     def extract_from_url url
       content = Cachy.cache_if(@cache, "webtractor.#{url}", @cache_params) do
-        open(url).read
+        @agent.get(url).content
       end
       extract(content)
     end

data/lib/webtractor/filters/biggest_block.rb

@@ -6,7 +6,7 @@ module Webtractor::Filters
 
     def process page
       @nodes = {}
-      explore(page.name, page.at('body'))
+      explore(page.name, page)
       @nodes = Hash[@nodes.sort.reverse]
 
       max = @nodes.keys[0]

data/lib/webtractor/filters/remove_empty.rb

@@ -1,7 +1,7 @@
 module Webtractor::Filters
   class RemoveEmpty
     def process page
-      explore(page.name, page.at('body'))
+      explore(page.name, page)
       page
     end
 
@@ -12,7 +12,9 @@ module Webtractor::Filters
         explore(path, child)
       end
 
-      node.remove if node.text.nil? || node.text.strip == ''
+      empty = node.text.nil? || node.text.strip == ''
+      hidden = node['class'] && node['class'].include?('hidden')
+      node.remove if empty || hidden
     end
   end
 end

data/lib/webtractor/filters/remove_menus.rb

@@ -15,7 +15,7 @@ module Webtractor::Filters
         ul.remove if a_count >= li_count.to_f/2
       end
 
-      explore(page.name, page.at('body'))
+      explore(page.name, page)
 
       page
     end

data/lib/webtractor/filters/remove_smallest.rb

@@ -1,7 +1,7 @@
 module Webtractor::Filters
   class RemoveSmallest
     def process page
-      explore(page.name, page.at('body'))
+      explore(page.name, page)
       page
     end
 

data/lib/webtractor/version.rb

@@ -1,3 +1,3 @@
 module Webtractor
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webtractor
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Rene Klacan
@@ -10,6 +10,20 @@ bindir: bin
 cert_chain: []
 date: 2014-05-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +66,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: jazz_hands
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: The Webtractor library can extract main content from websites like news,
   blogs, etc without unwanted boilerplate (menus, footer, comments)
 email: