arb-crawler 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1edbf13a9ded81eb25cf49a3d8f42abf38d86f0f
-  data.tar.gz: fef12e014d16904d3b5c7b99bdbfc30125985b22
+  metadata.gz: 11a51e3d1ff02b3c4c221f655a651a3c96266c93
+  data.tar.gz: a24e36d8a46bce45d9f56deaf2c30c821f5f0254
 SHA512:
-  metadata.gz: 6938b1e9a8b5270dd7068e9d5c541548905c9ce63259d3b32b80b0a924da79e804c03f155915657d1f8d83eb0ecb25c11a9179397cf6fe53361e89dbc57b7def
-  data.tar.gz: f603f483c106a317df770b49bd14ca6d08163ae73208cfb82b23aa58798a6f9647dc05b2ff257a3b4550f2689e11d34b0b695fa028b5fd08dc306bc581bfd428
+  metadata.gz: 63cc9d20c65e70f32efa22ec48bf1c8b7c76fa906a25254c0d7f7dfebd6b1c7135774d102ce6e16d294e9c580f543065fc7906e635690c26dceb60944c420823
+  data.tar.gz: eb551ccc891e231c710ee228995943a150fa69c292ed3126ebb64a79077028c08b2c8d2c71d3d123ba847b57b7b36db5b79d4962c65795bc47ff3440fd3fae99

data/lib/arb/crawler/version.rb

@@ -1,5 +1,5 @@
 module Arb
   module Crawler
-    VERSION = "1.0.0"
+    VERSION = "1.0.1"
   end
 end

data/lib/arb/crawler.rb

@@ -15,6 +15,16 @@ module Arb
         client
       end
 
+      define_method :filter_str do |str, black_list=nil|
+      black_list||=%w{\ / : * ? < > |} << "\n"
+      black_list.each do |i|
+        loop do
+          break unless str.sub!(i,'')
+        end
+      end
+      str
+      end
+
       define_method :filename_of_url do |url|
         url && url[url.rindex('/')+1..-1]
       end
@@ -34,18 +44,24 @@ module Arb
       methods.each do |method|
         ways.each do |way|
           define_method "#{method}_by_#{way}_raw" do |url,css_or_xpath,&blk|
-            ::Nokogiri.parse(client.send(method,url).body).send(way,css_or_xpath).tap do |res|
-              if blk
-                res.each do |e|
-                  blk[e]
+            begin
+              ::Nokogiri.parse(client.send(method,url).body).send(way,css_or_xpath).tap do |res|
+                if blk
+                  res.each do |e|
+                    blk[e]
+                  end
                 end
               end
+            rescue Exception=>e
+              $stderr.puts e
+              nil
             end
           end
 
           define_method "#{method}_by_#{way}" do |url,css_or_xpath,&blk|
             [].tap do |arr|
-              send("#{method}_by_#{way}_raw",url,css_or_xpath).each do |nokogiri_element|
+              raw=send("#{method}_by_#{way}_raw",url,css_or_xpath)
+              raw && raw.each do |nokogiri_element|
                 arr<<Hash.new.tap do |hash|
                   nokogiri_element.attributes.keys.each do |key|
                     hash[key.to_sym]=nokogiri_element.attribute(key).value
@@ -53,7 +69,7 @@ module Arb
                   hash.singleton_class.send :define_method, :text do
                     nokogiri_element.text
                   end
-                  blk[e] if blk
+                  blk[hash] if blk
                 end
               end
             end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arb-crawler
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - arybin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-12 00:00:00.000000000 Z
+date: 2017-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler