parallel588_polipus 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb50db8ac7efd7f04d3e044b2e1d708ab2434eb3
-  data.tar.gz: fa4158852a74a75603d7a6f0e6af1de9b7940dfc
+  metadata.gz: bfa6988cb2f60027238d9ed8168b758b3e5d2384
+  data.tar.gz: eaf2c566ebea5aaa742f40ab2e2a955f507ccd49
 SHA512:
-  metadata.gz: 8e43cec9505d08bceddfe6f64f5a335a8ee12a37fd5f11d1d1c4f20351df651e656b282e5f884259b0cd1df345975be105fa12dd77de52529d7d62c3387bcf38
-  data.tar.gz: fe3941eaee3fc53104e222cc9761e51bbd2fa8d1eaea02e5c66015fe5a520bc68b43b6537fd0059056988f6674058307bf93e200009ffba045b449a678cff1f7
+  metadata.gz: 0dd405e16b033aef6da59d914349e42965a66625e19a9b749ae5c78e874292e1296d6930ed58a135993361902baf75741fdcfad02f7f72364fbf087e839e4875
+  data.tar.gz: b55f04c79845c6d74e31e3b26ec683c66825b7d16ef4d9e80333ed39c3de2cfc6509382727dbdb261504535486c865124100e2a61e68e511aa7c00366adc3ec6

data/lib/polipus/page.rb

@@ -72,7 +72,7 @@ module Polipus
         u = a['href']
         next if u.nil? || u.empty?
         abs = to_absolute(u) rescue next
-        @links << abs if in_domain?(abs)
+        @links << abs if abs && in_domain?(abs)
       end
       @links.to_a
     end
@@ -82,7 +82,10 @@ module Polipus
     #
     def doc
       return @doc if @doc
-      @doc = Nokogiri::HTML(@body.toutf8, nil, 'utf-8') if @body && html? rescue nil
+      @body ||= ''
+      @body = @body.encode('utf-8', 'binary', :invalid => :replace,
+                           :undef => :replace, :replace => '')
+      @doc = Nokogiri::HTML(@body.toutf8, nil, 'utf-8') if @body && html?
     end
 
     #
@@ -168,10 +171,22 @@ module Polipus
     def to_absolute(link)
       return nil if link.nil?
 
+      valid_link = link.to_s.encode('utf-8', 'binary', :invalid => :replace,
+                                    :undef => :replace, :replace => '')
+
       # remove anchor
-      link = URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/, '')))
+      link =
+        begin
+          URI.encode(URI.decode(valid_link.gsub(/#[a-zA-Z0-9_-]*$/, '')))
+        rescue URI::Error
+          return nil
+        end
 
-      relative = URI(link)
+      relative = begin
+                   URI(link)
+                 rescue URI::Error
+                   return nil
+                 end
       absolute = base ? base.merge(relative) : @url.merge(relative)
 
       absolute.path = '/' if absolute.path.empty?

data/lib/polipus/storage/rethink_store.rb

@@ -12,6 +12,9 @@ module Polipus
         @rethink = options[:conn]
         @table   = options[:table]
 
+        unless @r.db_list.run(@rethink).include?(@rethink.default_db)
+          @r.db_create(@rethink.default_db).run(@rethink)
+        end
         unless @r.table_list.run(@rethink).include?(@table)
           @r.table_create(@table).run(@rethink)
           @r.table(@table).index_create('created_at')

data/lib/polipus/version.rb

@@ -1,5 +1,5 @@
 # encoding: UTF-8
 module Polipus
-  VERSION = '0.4.0'
+  VERSION = '0.4.1'
   HOMEPAGE = 'https://github.com/taganaka/polipus'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel588_polipus
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-03 00:00:00.000000000 Z
+date: 2015-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -345,4 +345,3 @@ signing_key:
 specification_version: 4
 summary: Polipus distributed web-crawler framework
 test_files: []
-has_rdoc: