aranha-parsers 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb3631929ccd8b029ab5b097554f96f5eb0cc27dcf8767d3a8a97f050ee60b80
-  data.tar.gz: f30f306452ac72997887550051954d290f7e6c321b694236fd9f49bc6fff5d4a
+  metadata.gz: f6ec27959167dfbae56e1fb00bfdbedad5bfe022a0708e493c623ca259df5c5d
+  data.tar.gz: c87db56022bfdf05f3f5cc99f58fbba4f0e73a5e8ddb7683d2569aabbf7bf28a
 SHA512:
-  metadata.gz: 35ebd10ceebadd29ade03ef0c5a75ed024e8d4377b19295861d8ef4c4e3bb4e091df5ff9f74398f523c98e0e3ee65eac1352bedc39eae033000cdc8561c5b749
-  data.tar.gz: 68a7664bef2ffd163cf110f14779feb92305b90cfb6b23e8d254c4cf2ca6871233d64032704790dd30397383cc67fd5a5b5dc9e67a1d866f1365524707b5ebb5
+  metadata.gz: 9116a522005c0a01b650bc889ce1a4c70f7096b478febc2e1b24f6596a4d94c77e750d81e616bd70b6ca557a1c61e56d55fcfe2066ff4f62aea3a9a14237ed01
+  data.tar.gz: 5080f643d1da251f05afb094a1c7ac9c5cbbe140a15f0d05b77ec87591e700b148ba4766615ee1853284d24674e2720149f25e6858b485e0123ebd3add74c4db

data/lib/aranha/parsers/html/node/default/numeric_support.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module NumericSupport
+            def integer_value(node, xpath)
+              r = string_value(node, xpath)
+              return nil if r.blank?
+
+              m = /\d+/.match(r)
+              raise "Integer not found in \"#{r}\"" unless m
+
+              m[0].to_i
+            end
+
+            def integer_optional_value(node, xpath)
+              r = string_value(node, xpath)
+              m = /\d+/.match(r)
+              m ? m[0].to_i : nil
+            end
+
+            def float_value(node, xpath)
+              parse_float(node, xpath, true)
+            end
+
+            def float_optional_value(node, xpath)
+              parse_float(node, xpath, false)
+            end
+
+            def us_decimal_value(node, xpath)
+              parse_us_decimal(node, xpath, true)
+            end
+
+            def us_decimal_optional_value(node, xpath)
+              parse_us_decimal(node, xpath, false)
+            end
+
+            private
+
+            def parse_float(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete('.').tr(',', '.').to_f
+              elsif required
+                raise "Float value not found in \"#{s}\""
+              end
+            end
+
+            def parse_us_decimal(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete(',').to_f
+              elsif required
+                raise "US decimal value not found in \"#{s}\""
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default/string_support.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module StringSupport
+            def quoted_value(node, xpath)
+              s = string_value(node, xpath)
+              return '' unless s
+
+              m = /\"([^\"]+)\"/.match(s)
+              return m[1] if m
+
+              ''
+            end
+
+            def regxep(node, xpath, pattern)
+              s = string_value(node, xpath)
+              m = pattern.match(s)
+              return m if m
+
+              raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
+            end
+
+            def string_value(node, xpath)
+              if node.at_xpath(xpath)
+                sanitize_string(node.at_xpath(xpath).text)
+              else
+                ''
+              end
+            end
+
+            def string_recursive_value(node, xpath, required = true)
+              root = node.at_xpath(xpath)
+              if root.blank?
+                return nil unless required
+
+                raise "No node found (Xpath: #{xpath})"
+              end
+              result = string_recursive(root)
+              return result if result.present?
+              return nil unless required
+
+              raise "String blank (Xpath: #{xpath})"
+            end
+
+            def string_recursive_optional_value(node, xpath)
+              string_recursive_value(node, xpath, false)
+            end
+
+            private
+
+            def sanitize_string(obj)
+              obj.to_s.tr("\u00A0", ' ').strip
+            end
+
+            def string_recursive(node)
+              return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
+
+              s = ''
+              node.children.each do |child|
+                child_s = string_recursive(child)
+                s += ' ' + child_s if child_s.present?
+              end
+              sanitize_string(s)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default.rb

@@ -1,71 +1,14 @@
 # frozen_string_literal: true
 
 require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   module Parsers
     module Html
       module Node
         class Default < ::Aranha::Parsers::Html::Node::Base
-          def string_value(node, xpath)
-            if node.at_xpath(xpath)
-              sanitize_string(node.at_xpath(xpath).text)
-            else
-              ''
-            end
-          end
-
-          def string_recursive_value(node, xpath, required = true)
-            root = node.at_xpath(xpath)
-            if root.blank?
-              return nil unless required
-
-              raise "No node found (Xpath: #{xpath})"
-            end
-            result = string_recursive(root)
-            return result if result.present?
-            return nil unless required
-
-            raise "String blank (Xpath: #{xpath})"
-          end
-
-          def string_recursive_optional_value(node, xpath)
-            string_recursive_value(node, xpath, false)
-          end
-
-          def quoted_value(node, xpath)
-            s = string_value(node, xpath)
-            return '' unless s
-
-            m = /\"([^\"]+)\"/.match(s)
-            return m[1] if m
-
-            ''
-          end
-
-          def integer_value(node, xpath)
-            r = string_value(node, xpath)
-            return nil if r.blank?
-
-            m = /\d+/.match(r)
-            raise "Integer not found in \"#{r}\"" unless m
-
-            m[0].to_i
-          end
-
-          def integer_optional_value(node, xpath)
-            r = string_value(node, xpath)
-            m = /\d+/.match(r)
-            m ? m[0].to_i : nil
-          end
-
-          def float_value(node, xpath)
-            parse_float(node, xpath, true)
-          end
-
-          def float_optional_value(node, xpath)
-            parse_float(node, xpath, false)
-          end
+          require_sub __FILE__, include_modules: true
 
           def array_value(node, xpath)
             r = node.xpath(xpath).map { |n| n.text.strip }
@@ -84,41 +27,6 @@ module Aranha
             m = /(\d+) m/.match(join_value(node, xpath))
             m ? m[1].to_i : nil
           end
-
-          def regxep(node, xpath, pattern)
-            s = string_value(node, xpath)
-            m = pattern.match(s)
-            return m if m
-
-            raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
-          end
-
-          private
-
-          def parse_float(node, xpath, required)
-            s = string_value(node, xpath)
-            m = /\d+(?:[\.\,](\d+))?/.match(s)
-            if m
-              m[0].delete('.').tr(',', '.').to_f
-            elsif required
-              raise "Float value not found in \"#{s}\""
-            end
-          end
-
-          def sanitize_string(obj)
-            obj.to_s.tr("\u00A0", ' ').strip
-          end
-
-          def string_recursive(node)
-            return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
-
-            s = ''
-            node.children.each do |child|
-              child_s = string_recursive(child)
-              s += ' ' + child_s if child_s.present?
-            end
-            sanitize_string(s)
-          end
         end
       end
     end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -3,6 +3,7 @@
 require 'addressable'
 require 'curb'
 require 'aranha/parsers/source_address/fetch_content_error'
+require 'faraday_middleware'
 
 module Aranha
   module Parsers
@@ -38,10 +39,12 @@ module Aranha
         end
 
         def content
-          c = ::Curl::Easy.new(url)
-          c.follow_location = true
-          curl_perform(c)
-          return c.body_str if c.status.to_i == 200
+          conn = ::Faraday.new do |f|
+            f.request :retry # retry transient failures
+            f.response :follow_redirects # follow redirects
+          end
+          c = conn.get(url)
+          return c.body if c.status == 200
 
           raise ::Aranha::Parsers::SourceAddress::FetchContentError,
                 "Get #{url} returned #{c.status.to_i}"
@@ -50,18 +53,6 @@ module Aranha
         def serialize
           url
         end
-
-        private
-
-        def curl_perform(curl)
-          unless curl.perform
-            raise(::Aranha::Parsers::SourceAddress::FetchContentError,
-                  "Curl perform failed (URL: #{url})")
-          end
-          @final_url = curl.url
-        rescue Curl::Err::CurlError => e
-          raise ::Aranha::Parsers::SourceAddress::FetchContentError, "CURL error: #{e.class.name}"
-        end
       end
     end
   end

data/lib/aranha/parsers/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Parsers
-    VERSION = '0.9.0'
+    VERSION = '0.10.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-parsers
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Esquilo Azul Company
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-19 00:00:00.000000000 Z
+date: 2021-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.74'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -156,6 +170,8 @@ files:
 - lib/aranha/parsers/html/node.rb
 - lib/aranha/parsers/html/node/base.rb
 - lib/aranha/parsers/html/node/default.rb
+- lib/aranha/parsers/html/node/default/numeric_support.rb
+- lib/aranha/parsers/html/node/default/string_support.rb
 - lib/aranha/parsers/invalid_state_exception.rb
 - lib/aranha/parsers/json.rb
 - lib/aranha/parsers/json/base.rb