aranha-parsers 0.8.5 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fd66650df384bfd84cc234f0ed901e426f0000a7f11bd8cdcec959d5690efdb
-  data.tar.gz: a0b80baee8f763633753c9fefacd5a0ab528a8047a8f14a876a08f0cd443c6a1
+  metadata.gz: ea4e3ac094b66f5e1b02e6af4f102c752a869e0a234354b0badb3b2d666368c7
+  data.tar.gz: 95de53aed0f9e7157894515f3b3c881c740c6a9d4e2e787d4c5bbb2bb4432294
 SHA512:
-  metadata.gz: f9bd1e8d032eb4f782526e45b549aecdf6951a9c28ba55296293128d4a655a14f8cafda545560d794c5f39a4798c65b96c39e2e5dcea324057a85c5dec492d01
-  data.tar.gz: 577bd16ed893fa9df121de6be1844aefc1f178790871aa89caa9073633e83682059f326de48d8561a7928857d26c08f57bbc655d0205b2a7ef7126dd922e2398
+  metadata.gz: 176470a9a8163f44654485f96285c254ecf8d6661f3b435f7af0bc6ccf076c8081cb0554f7b31d77a751f2ce9caa4592454d7b3b5de4cd522a008c2798ecdc7d
+  data.tar.gz: ebc4b32f65dc83d6ba681a0af28babcd8375f4b4c9585985080fc5bcdb5f7bae653979bf21377f10075862c1ab7fa589cca4c1ce0b254b134f1d8d9e7c1f9d3e

data/lib/aranha/parsers/base.rb

@@ -11,7 +11,9 @@ module Aranha
       class << self
         def from_content(content)
           ::EacRubyUtils::Fs::Temp.on_file do |path|
-            path.write(content)
+            ::File.open(path.to_s, 'w:UTF-8') do |f|
+              f.write content.force_encoding('UTF-8')
+            end
             r = new(path.to_path)
             r.content
             r
@@ -35,21 +37,21 @@ module Aranha
       delegate :url, to: :source_address
 
       def content
-        @content ||= begin
-          s = source_address.content
-          log_content(s)
-          s
-        end
+        @content ||= log_content(source_address_content)
       end
 
+      # @return [String]
+      delegate :content, to: :source_address, prefix: true
+
       private
 
+      # @return [String]
       def log_content(content, suffix = '')
         path = log_file(suffix)
 
-        return unless path
+        File.open(path, 'wb') { |file| file.write(content) } if path
 
-        File.open(path, 'wb') { |file| file.write(content) }
+        content
       end
 
       def log_file(suffix)

data/lib/aranha/parsers/html/node/default/numeric_support.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module NumericSupport
+            def integer_value(node, xpath)
+              r = string_value(node, xpath)
+              return nil if r.blank?
+
+              m = /\d+/.match(r)
+              raise "Integer not found in \"#{r}\"" unless m
+
+              m[0].to_i
+            end
+
+            def integer_optional_value(node, xpath)
+              r = string_value(node, xpath)
+              m = /\d+/.match(r)
+              m ? m[0].to_i : nil
+            end
+
+            def float_value(node, xpath)
+              parse_float(node, xpath, true)
+            end
+
+            def float_optional_value(node, xpath)
+              parse_float(node, xpath, false)
+            end
+
+            def us_decimal_value(node, xpath)
+              parse_us_decimal(node, xpath, true)
+            end
+
+            def us_decimal_optional_value(node, xpath)
+              parse_us_decimal(node, xpath, false)
+            end
+
+            private
+
+            def parse_float(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete('.').tr(',', '.').to_f
+              elsif required
+                raise "Float value not found in \"#{s}\""
+              end
+            end
+
+            def parse_us_decimal(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete(',').to_f
+              elsif required
+                raise "US decimal value not found in \"#{s}\""
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default/string_support.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module StringSupport
+            def quoted_value(node, xpath)
+              s = string_value(node, xpath)
+              return '' unless s
+
+              m = /\"([^\"]+)\"/.match(s)
+              return m[1] if m
+
+              ''
+            end
+
+            def regxep(node, xpath, pattern)
+              s = string_value(node, xpath)
+              m = pattern.match(s)
+              return m if m
+
+              raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
+            end
+
+            def string_value(node, xpath)
+              if node.at_xpath(xpath)
+                sanitize_string(node.at_xpath(xpath).text)
+              else
+                ''
+              end
+            end
+
+            def string_recursive_value(node, xpath, required = true)
+              root = node.at_xpath(xpath)
+              if root.blank?
+                return nil unless required
+
+                raise "No node found (Xpath: #{xpath})"
+              end
+              result = string_recursive(root)
+              return result if result.present?
+              return nil unless required
+
+              raise "String blank (Xpath: #{xpath})"
+            end
+
+            def string_recursive_optional_value(node, xpath)
+              string_recursive_value(node, xpath, false)
+            end
+
+            private
+
+            def sanitize_string(obj)
+              obj.to_s.tr("\u00A0", ' ').strip
+            end
+
+            def string_recursive(node)
+              return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
+
+              s = ''
+              node.children.each do |child|
+                child_s = string_recursive(child)
+                s += ' ' + child_s if child_s.present?
+              end
+              sanitize_string(s)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default.rb

@@ -1,71 +1,14 @@
 # frozen_string_literal: true
 
 require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   module Parsers
     module Html
       module Node
         class Default < ::Aranha::Parsers::Html::Node::Base
-          def string_value(node, xpath)
-            if node.at_xpath(xpath)
-              sanitize_string(node.at_xpath(xpath).text)
-            else
-              ''
-            end
-          end
-
-          def string_recursive_value(node, xpath, required = true)
-            root = node.at_xpath(xpath)
-            if root.blank?
-              return nil unless required
-
-              raise "No node found (Xpath: #{xpath})"
-            end
-            result = string_recursive(root)
-            return result if result.present?
-            return nil unless required
-
-            raise "String blank (Xpath: #{xpath})"
-          end
-
-          def string_recursive_optional_value(node, xpath)
-            string_recursive_value(node, xpath, false)
-          end
-
-          def quoted_value(node, xpath)
-            s = string_value(node, xpath)
-            return '' unless s
-
-            m = /\"([^\"]+)\"/.match(s)
-            return m[1] if m
-
-            ''
-          end
-
-          def integer_value(node, xpath)
-            r = string_value(node, xpath)
-            return nil if r.blank?
-
-            m = /\d+/.match(r)
-            raise "Integer not found in \"#{r}\"" unless m
-
-            m[0].to_i
-          end
-
-          def integer_optional_value(node, xpath)
-            r = string_value(node, xpath)
-            m = /\d+/.match(r)
-            m ? m[0].to_i : nil
-          end
-
-          def float_value(node, xpath)
-            parse_float(node, xpath, true)
-          end
-
-          def float_optional_value(node, xpath)
-            parse_float(node, xpath, false)
-          end
+          require_sub __FILE__, include_modules: true
 
           def array_value(node, xpath)
             r = node.xpath(xpath).map { |n| n.text.strip }
@@ -84,41 +27,6 @@ module Aranha
             m = /(\d+) m/.match(join_value(node, xpath))
             m ? m[1].to_i : nil
           end
-
-          def regxep(node, xpath, pattern)
-            s = string_value(node, xpath)
-            m = pattern.match(s)
-            return m if m
-
-            raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
-          end
-
-          private
-
-          def parse_float(node, xpath, required)
-            s = string_value(node, xpath)
-            m = /\d+(?:[\.\,](\d+))?/.match(s)
-            if m
-              m[0].delete('.').tr(',', '.').to_f
-            elsif required
-              raise "Float value not found in \"#{s}\""
-            end
-          end
-
-          def sanitize_string(obj)
-            obj.to_s.tr("\u00A0", ' ').strip
-          end
-
-          def string_recursive(node)
-            return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
-
-            s = ''
-            node.children.each do |child|
-              child_s = string_recursive(child)
-              s += ' ' + child_s if child_s.present?
-            end
-            sanitize_string(s)
-          end
         end
       end
     end

data/lib/aranha/parsers/json/base.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/base'
+require 'json'
+
+module Aranha
+  module Parsers
+    module Json
+      class Base < ::Aranha::Parsers::Base
+        def data
+          default_data
+        end
+
+        def default_data
+          ::JSON.parse(content)
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/json.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Json
+      require_sub __FILE__
+    end
+  end
+end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -3,6 +3,7 @@
 require 'addressable'
 require 'curb'
 require 'aranha/parsers/source_address/fetch_content_error'
+require 'faraday_middleware'
 
 module Aranha
   module Parsers
@@ -32,11 +33,18 @@ module Aranha
           source
         end
 
+        def final_url
+          content unless @final_url
+          @final_url
+        end
+
         def content
-          c = ::Curl::Easy.new(url)
-          c.follow_location = true
-          curl_perform(c)
-          return c.body_str if c.status.to_i == 200
+          conn = ::Faraday.new do |f|
+            f.request :retry # retry transient failures
+            f.response :follow_redirects # follow redirects
+          end
+          c = conn.get(url)
+          return c.body if c.status == 200
 
           raise ::Aranha::Parsers::SourceAddress::FetchContentError,
                 "Get #{url} returned #{c.status.to_i}"
@@ -45,17 +53,6 @@ module Aranha
         def serialize
           url
         end
-
-        private
-
-        def curl_perform(curl)
-          unless curl.perform
-            raise(::Aranha::Parsers::SourceAddress::FetchContentError,
-                  "Curl perform failed (URL: #{url})")
-          end
-        rescue Curl::Err::CurlError => e
-          raise ::Aranha::Parsers::SourceAddress::FetchContentError, "CURL error: #{e.class.name}"
-        end
       end
     end
   end

data/lib/aranha/parsers/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Parsers
-    VERSION = '0.8.5'
+    VERSION = '0.11.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-parsers
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.11.0
 platform: ruby
 authors:
 - Esquilo Azul Company
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-21 00:00:00.000000000 Z
+date: 2022-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.74'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -156,7 +170,11 @@ files:
 - lib/aranha/parsers/html/node.rb
 - lib/aranha/parsers/html/node/base.rb
 - lib/aranha/parsers/html/node/default.rb
+- lib/aranha/parsers/html/node/default/numeric_support.rb
+- lib/aranha/parsers/html/node/default/string_support.rb
 - lib/aranha/parsers/invalid_state_exception.rb
+- lib/aranha/parsers/json.rb
+- lib/aranha/parsers/json/base.rb
 - lib/aranha/parsers/patches.rb
 - lib/aranha/parsers/patches/ofx_parser.rb
 - lib/aranha/parsers/rspec.rb