aranha-parsers 0.9.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb3631929ccd8b029ab5b097554f96f5eb0cc27dcf8767d3a8a97f050ee60b80
-  data.tar.gz: f30f306452ac72997887550051954d290f7e6c321b694236fd9f49bc6fff5d4a
+  metadata.gz: 955f0df6b2a59762cbdd597dd17ae6608a880ee74f4547449ad06bbef4cf9f09
+  data.tar.gz: 95e891f8db0aa252f3e385956e783c48e08e42e68b3e6666491356eb2e60618a
 SHA512:
-  metadata.gz: 35ebd10ceebadd29ade03ef0c5a75ed024e8d4377b19295861d8ef4c4e3bb4e091df5ff9f74398f523c98e0e3ee65eac1352bedc39eae033000cdc8561c5b749
-  data.tar.gz: 68a7664bef2ffd163cf110f14779feb92305b90cfb6b23e8d254c4cf2ca6871233d64032704790dd30397383cc67fd5a5b5dc9e67a1d866f1365524707b5ebb5
+  metadata.gz: e5e7266770218b0698edb3111db90d797a24bb3e0f5ee54d5728a5810a87559a22a3a9f3d368bb28fc83fda8c6cbc04c99cab44c50ee25ea76e7550ea4753249
+  data.tar.gz: 66873b24e21f3c3e43a4ae49b82997c6e401a4e500274e2fc76e4ba91b013eae1a4f6b8641880f94896f1dab40bf23c86b24e226c14c2b92afe024d2d18aa230

data/lib/aranha/parsers/base.rb

@@ -37,21 +37,21 @@ module Aranha
       delegate :url, to: :source_address
 
       def content
-        @content ||= begin
-          s = source_address.content
-          log_content(s)
-          s
-        end
+        @content ||= log_content(source_address_content)
       end
 
+      # @return [String]
+      delegate :content, to: :source_address, prefix: true
+
       private
 
+      # @return [String]
       def log_content(content, suffix = '')
         path = log_file(suffix)
 
-        return unless path
+        File.open(path, 'wb') { |file| file.write(content) } if path
 
-        File.open(path, 'wb') { |file| file.write(content) }
+        content
       end
 
       def log_file(suffix)

data/lib/aranha/parsers/html/item_list.rb

@@ -10,11 +10,15 @@ module Aranha
           items_data
         end
 
+        def item_data(item)
+          item
+        end
+
         def items_data
           count = 0
           @data ||= nokogiri.xpath(items_xpath).map do |m|
             count += 1
-            node_parser.parse(m)
+            item_data(node_parser.parse(m))
           end
         rescue StandardError => e
           raise StandardError, "#{e.message} (Count: #{count})"

data/lib/aranha/parsers/html/node/default/numeric_support.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module NumericSupport
+            def integer_value(node, xpath)
+              r = string_value(node, xpath)
+              return nil if r.blank?
+
+              m = /\d+/.match(r)
+              raise "Integer not found in \"#{r}\"" unless m
+
+              m[0].to_i
+            end
+
+            def integer_optional_value(node, xpath)
+              r = string_value(node, xpath)
+              m = /\d+/.match(r)
+              m ? m[0].to_i : nil
+            end
+
+            def float_value(node, xpath)
+              parse_float(node, xpath, true)
+            end
+
+            def float_optional_value(node, xpath)
+              parse_float(node, xpath, false)
+            end
+
+            def us_decimal_value(node, xpath)
+              parse_us_decimal(node, xpath, true)
+            end
+
+            def us_decimal_optional_value(node, xpath)
+              parse_us_decimal(node, xpath, false)
+            end
+
+            private
+
+            def parse_float(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete('.').tr(',', '.').to_f
+              elsif required
+                raise "Float value not found in \"#{s}\""
+              end
+            end
+
+            def parse_us_decimal(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete(',').to_f
+              elsif required
+                raise "US decimal value not found in \"#{s}\""
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default/string_support.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module StringSupport
+            def quoted_value(node, xpath)
+              s = string_value(node, xpath)
+              return '' unless s
+
+              m = /\"([^\"]+)\"/.match(s)
+              return m[1] if m
+
+              ''
+            end
+
+            def regxep(node, xpath, pattern)
+              s = string_value(node, xpath)
+              m = pattern.match(s)
+              return m if m
+
+              raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
+            end
+
+            def string_value(node, xpath)
+              if node.at_xpath(xpath)
+                sanitize_string(node.at_xpath(xpath).text)
+              else
+                ''
+              end
+            end
+
+            def string_recursive_value(node, xpath, required = true)
+              root = node.at_xpath(xpath)
+              if root.blank?
+                return nil unless required
+
+                raise "No node found (Xpath: #{xpath})"
+              end
+              result = string_recursive(root)
+              return result if result.present?
+              return nil unless required
+
+              raise "String blank (Xpath: #{xpath})"
+            end
+
+            def string_recursive_optional_value(node, xpath)
+              string_recursive_value(node, xpath, false)
+            end
+
+            private
+
+            def sanitize_string(obj)
+              obj.to_s.tr("\u00A0", ' ').strip
+            end
+
+            def string_recursive(node)
+              return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
+
+              s = ''
+              node.children.each do |child|
+                child_s = string_recursive(child)
+                s += ' ' + child_s if child_s.present?
+              end
+              sanitize_string(s)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default.rb

@@ -1,71 +1,14 @@
 # frozen_string_literal: true
 
 require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   module Parsers
     module Html
       module Node
         class Default < ::Aranha::Parsers::Html::Node::Base
-          def string_value(node, xpath)
-            if node.at_xpath(xpath)
-              sanitize_string(node.at_xpath(xpath).text)
-            else
-              ''
-            end
-          end
-
-          def string_recursive_value(node, xpath, required = true)
-            root = node.at_xpath(xpath)
-            if root.blank?
-              return nil unless required
-
-              raise "No node found (Xpath: #{xpath})"
-            end
-            result = string_recursive(root)
-            return result if result.present?
-            return nil unless required
-
-            raise "String blank (Xpath: #{xpath})"
-          end
-
-          def string_recursive_optional_value(node, xpath)
-            string_recursive_value(node, xpath, false)
-          end
-
-          def quoted_value(node, xpath)
-            s = string_value(node, xpath)
-            return '' unless s
-
-            m = /\"([^\"]+)\"/.match(s)
-            return m[1] if m
-
-            ''
-          end
-
-          def integer_value(node, xpath)
-            r = string_value(node, xpath)
-            return nil if r.blank?
-
-            m = /\d+/.match(r)
-            raise "Integer not found in \"#{r}\"" unless m
-
-            m[0].to_i
-          end
-
-          def integer_optional_value(node, xpath)
-            r = string_value(node, xpath)
-            m = /\d+/.match(r)
-            m ? m[0].to_i : nil
-          end
-
-          def float_value(node, xpath)
-            parse_float(node, xpath, true)
-          end
-
-          def float_optional_value(node, xpath)
-            parse_float(node, xpath, false)
-          end
+          require_sub __FILE__, include_modules: true
 
           def array_value(node, xpath)
             r = node.xpath(xpath).map { |n| n.text.strip }
@@ -84,41 +27,6 @@ module Aranha
             m = /(\d+) m/.match(join_value(node, xpath))
             m ? m[1].to_i : nil
           end
-
-          def regxep(node, xpath, pattern)
-            s = string_value(node, xpath)
-            m = pattern.match(s)
-            return m if m
-
-            raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
-          end
-
-          private
-
-          def parse_float(node, xpath, required)
-            s = string_value(node, xpath)
-            m = /\d+(?:[\.\,](\d+))?/.match(s)
-            if m
-              m[0].delete('.').tr(',', '.').to_f
-            elsif required
-              raise "Float value not found in \"#{s}\""
-            end
-          end
-
-          def sanitize_string(obj)
-            obj.to_s.tr("\u00A0", ' ').strip
-          end
-
-          def string_recursive(node)
-            return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
-
-            s = ''
-            node.children.each do |child|
-              child_s = string_recursive(child)
-              s += ' ' + child_s if child_s.present?
-            end
-            sanitize_string(s)
-          end
         end
       end
     end

data/lib/aranha/parsers/source_address/hash_http_base.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/source_address/fetch_content_error'
+require 'aranha/parsers/source_address/hash_http_base'
+require 'eac_ruby_utils/core_ext'
+require 'faraday_middleware'
+require 'yaml'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HashHttpBase
+        class << self
+          def http_method
+            const_get 'HTTP_METHOD'
+          end
+
+          def valid_source?(source)
+            source.is_a?(::Hash) &&
+              source.with_indifferent_access[:method].to_s.downcase.strip == http_method.to_s
+          end
+        end
+
+        DEFAULT_BODY = ''
+        DEFAULT_FOLLOW_REDIRECT = true
+        DEFAULT_HEADERS = {}.freeze
+        DEFAULT_PARAMS = {}.freeze
+
+        common_constructor :source do
+          self.source = source.with_indifferent_access
+        end
+        compare_by :source
+
+        def body
+          param(:body, DEFAULT_BODY)
+        end
+
+        def follow_redirect?
+          param(:follow_redirect, DEFAULT_FOLLOW_REDIRECT)
+        end
+
+        def headers
+          param(:headers, DEFAULT_HEADERS)
+        end
+
+        def url
+          source.fetch(:url)
+        end
+
+        def serialize
+          source.to_yaml
+        end
+
+        # @return [Faraday]
+        def faraday_connection
+          ::Faraday.new do |f|
+            f.response :follow_redirects if follow_redirect?
+          end
+        end
+
+        def faraday_request
+          faraday_connection.send(self.class.http_method, url) do |req|
+            headers.if_present { |v| req.headers = v }
+            body.if_present { |v| req.body = v }
+          end
+        end
+
+        def content
+          req = faraday_request
+          return req.body if req.status == 200
+
+          raise ::Aranha::Parsers::SourceAddress::FetchContentError,
+                "Get #{url} returned #{req.status.to_i}"
+        end
+
+        def param(key, default_value)
+          source[key] || params[key] || default_value
+        end
+
+        def params
+          source[:params].if_present(DEFAULT_PARAMS)
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/hash_http_get.rb

@@ -1,24 +1,12 @@
 # frozen_string_literal: true
 
-require 'aranha/parsers/source_address/hash_http_post'
+require 'aranha/parsers/source_address/hash_http_base'
 
 module Aranha
   module Parsers
     class SourceAddress
-      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'get'
-          end
-        end
-
-        def content
-          HTTPClient.new.get_content(
-            source[:url],
-            source[:params]
-          )
-        end
+      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpBase
+        HTTP_METHOD = :get
       end
     end
   end

data/lib/aranha/parsers/source_address/hash_http_post.rb

@@ -1,44 +1,12 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/hash/indifferent_access'
-require 'httpclient'
-require 'yaml'
+require 'aranha/parsers/source_address/hash_http_base'
 
 module Aranha
   module Parsers
     class SourceAddress
-      class HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'post'
-          end
-        end
-
-        attr_reader :source
-
-        def initialize(source)
-          @source = source.with_indifferent_access
-        end
-
-        def ==(other)
-          self.class == other.class && source == other.source
-        end
-
-        def url
-          source.fetch(:url)
-        end
-
-        def serialize
-          source.to_yaml
-        end
-
-        def content
-          HTTPClient.new.post_content(
-            source[:url],
-            source[:params].merge(follow_redirect: true)
-          )
-        end
+      class HashHttpPost < ::Aranha::Parsers::SourceAddress::HashHttpBase
+        HTTP_METHOD = :post
       end
     end
   end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -3,6 +3,7 @@
 require 'addressable'
 require 'curb'
 require 'aranha/parsers/source_address/fetch_content_error'
+require 'faraday_middleware'
 
 module Aranha
   module Parsers
@@ -38,10 +39,12 @@ module Aranha
         end
 
         def content
-          c = ::Curl::Easy.new(url)
-          c.follow_location = true
-          curl_perform(c)
-          return c.body_str if c.status.to_i == 200
+          conn = ::Faraday.new do |f|
+            f.request :retry # retry transient failures
+            f.response :follow_redirects # follow redirects
+          end
+          c = conn.get(url)
+          return c.body if c.status == 200
 
           raise ::Aranha::Parsers::SourceAddress::FetchContentError,
                 "Get #{url} returned #{c.status.to_i}"
@@ -50,18 +53,6 @@ module Aranha
         def serialize
           url
         end
-
-        private
-
-        def curl_perform(curl)
-          unless curl.perform
-            raise(::Aranha::Parsers::SourceAddress::FetchContentError,
-                  "Curl perform failed (URL: #{url})")
-          end
-          @final_url = curl.url
-        rescue Curl::Err::CurlError => e
-          raise ::Aranha::Parsers::SourceAddress::FetchContentError, "CURL error: #{e.class.name}"
-        end
       end
     end
   end

data/lib/aranha/parsers/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Parsers
-    VERSION = '0.9.0'
+    VERSION = '0.12.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-parsers
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Esquilo Azul Company
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-19 00:00:00.000000000 Z
+date: 2022-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -58,34 +58,34 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0.92'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.92.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0.92'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.92.1
 - !ruby/object:Gem::Dependency
-  name: httpclient
+  name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.3
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.3
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -156,6 +156,8 @@ files:
 - lib/aranha/parsers/html/node.rb
 - lib/aranha/parsers/html/node/base.rb
 - lib/aranha/parsers/html/node/default.rb
+- lib/aranha/parsers/html/node/default/numeric_support.rb
+- lib/aranha/parsers/html/node/default/string_support.rb
 - lib/aranha/parsers/invalid_state_exception.rb
 - lib/aranha/parsers/json.rb
 - lib/aranha/parsers/json/base.rb
@@ -167,6 +169,7 @@ files:
 - lib/aranha/parsers/source_address.rb
 - lib/aranha/parsers/source_address/fetch_content_error.rb
 - lib/aranha/parsers/source_address/file.rb
+- lib/aranha/parsers/source_address/hash_http_base.rb
 - lib/aranha/parsers/source_address/hash_http_get.rb
 - lib/aranha/parsers/source_address/hash_http_post.rb
 - lib/aranha/parsers/source_address/http_get.rb