aranha-parsers 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bd9b607e666c0b3b4ef0a54f1289891d5f958bbc9d72ad8632e4fc75858fef4
-  data.tar.gz: a462e661c567337abb7184ae5d1bc8555def424742b351e9bc83e9bad7bb88f3
+  metadata.gz: 51a24e93c2727f9cdb6e187eb6f2c15719591c70c0afa5e04be790cc0cdaba7d
+  data.tar.gz: b56ba940af2dadeacca0ca2dd67fce81d28af267b768d80927d157b5a94d8f88
 SHA512:
-  metadata.gz: 85566b737941e252dbdf0f446f926c2cb9421d800c0547f207df3a5350d847bb61e303d9f278b3a8071182291814a64c6213d8f35a575ac93bb793adfeb463d3
-  data.tar.gz: 6f969ca1107f35c3aa41e52288d9e41da1b5f8f3a2efd9c13adf151a8d09e07810f8818d6a05e88e2800f02ed1fcaffc051a5bc1a886ec25d1cb373c79b6b507
+  metadata.gz: 6951921ada5daca5e9a0f6fc70726b447da1308d11334fc8e92639953c6a12c617ded89b969e3561e38d3f4f697a25673c981c945ea79f55fc23d06d580f6726
+  data.tar.gz: 9a201a0e54ae982da2b4f78ba24c2acb188637c370a03e1b9e92cceab27ab0f9991b35ba948de27e6a5564b14a5f6b3ee8591fa4e16691e9c5949677030a84b4

data/lib/aranha/parsers/html/item_list.rb

@@ -7,6 +7,10 @@ module Aranha
     module Html
       class ItemList < Base
         def data
+          items_data
+        end
+
+        def items_data
           count = 0
           @data ||= nokogiri.xpath(items_xpath).map do |m|
             count += 1

data/lib/aranha/parsers/html/node/base.rb

@@ -12,7 +12,13 @@ module Aranha
           end
 
           def parse(node)
-            Hash[fields.map { |f| [f[0], parse_field(node, f[2], f[1])] }]
+            fields.map do |f|
+              begin
+                [f[0], parse_field(node, f[2], f[1])]
+              rescue StandardError => e
+                raise StandardError, "#{e.message}\nFields: #{f}"
+              end
+            end.to_h
           end
 
           private

data/lib/aranha/parsers/html/node/default.rb

@@ -9,12 +9,28 @@ module Aranha
         class Default < ::Aranha::Parsers::Html::Node::Base
           def string_value(node, xpath)
             if node.at_xpath(xpath)
-              node.at_xpath(xpath).text.to_s.tr("\u00A0", ' ').strip
+              sanitize_string(node.at_xpath(xpath).text)
             else
               ''
             end
           end
 
+          def string_recursive_value(node, xpath, required = true)
+            root = node.at_xpath(xpath)
+            if root.blank?
+              return nil unless required
+              raise "No node found (Xpath: #{xpath})"
+            end
+            result = string_recursive(root)
+            return result unless result.blank?
+            return nil unless required
+            raise "String blank (Xpath: #{xpath})"
+          end
+
+          def string_recursive_optional_value(node, xpath)
+            string_recursive_value(node, xpath, false)
+          end
+
           def quoted_value(node, xpath)
             s = string_value(node, xpath)
             return '' unless s
@@ -81,11 +97,25 @@ module Aranha
             s = string_value(node, xpath)
             m = /\d+(?:[\.\,](\d+))?/.match(s)
             if m
-              m[0].sub(',', '.').to_f
+              m[0].delete('.').tr(',', '.').to_f
             elsif required
               raise "Float value not found in \"#{s}\""
             end
           end
+
+          def sanitize_string(obj)
+            obj.to_s.tr("\u00A0", ' ').strip
+          end
+
+          def string_recursive(node)
+            return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
+            s = ''
+            node.children.each do |child|
+              child_s = string_recursive(child)
+              s += ' ' + child_s if child_s.present?
+            end
+            sanitize_string(s)
+          end
         end
       end
     end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'addressable'
-require 'net/http'
+require 'curb'
 
 module Aranha
   module Parsers
@@ -32,29 +32,15 @@ module Aranha
         end
 
         def content
-          content_fetch(url)
+          c = ::Curl::Easy.new(url)
+          raise "Curl perform failed (URL: #{url})" unless c.perform
+          return c.body_str if c.status.to_i == 200
+          raise "Get #{url} returned #{c.status.to_i}"
         end
 
         def serialize
           url
         end
-
-        private
-
-        def content_fetch(uri, limit = 10)
-          raise 'too many HTTP redirects' if limit.zero?
-
-          response = Net::HTTP.get_response(URI(uri))
-
-          case response
-          when Net::HTTPSuccess then
-            response.body
-          when Net::HTTPRedirection then
-            content_fetch(self.class.location_uri(uri, response['location']), limit - 1)
-          else
-            response.value
-          end
-        end
       end
     end
   end

data/lib/aranha/parsers/version.rb

@@ -1,5 +1,5 @@
 module Aranha
   module Parsers
-    VERSION = '0.1.1'.freeze
+    VERSION = '0.2.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-parsers
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Esquilo Azul Company
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-18 00:00:00.000000000 Z
+date: 2019-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.10
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement