aranha-parsers 0.8.3 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee60e5eb7ad164eaac294af5fe274611cc72adb14645c59f12db7fe483dd551f
-  data.tar.gz: 280ba737d37a5cd22d741b92961d3ee15c72b15a43f76604f4c9f2bc92e4731e
+  metadata.gz: f6ec27959167dfbae56e1fb00bfdbedad5bfe022a0708e493c623ca259df5c5d
+  data.tar.gz: c87db56022bfdf05f3f5cc99f58fbba4f0e73a5e8ddb7683d2569aabbf7bf28a
 SHA512:
-  metadata.gz: 743b98993e76af5d18f0c58b4a197b0bf62a18771be883a6b29ce8551c3d236ebaa81acda3bd4200d4c3e8f6ab9f9db434d047bc80ae2ecd040fbf7cf706826f
-  data.tar.gz: 1086dbe158f755c8716dbee97058c5cd6a72673b9769f92e9c11206bc072637131a0d698907ce292ac6fbe864e33d5702abc0aceb044d16b8f034d648199570e
+  metadata.gz: 9116a522005c0a01b650bc889ce1a4c70f7096b478febc2e1b24f6596a4d94c77e750d81e616bd70b6ca557a1c61e56d55fcfe2066ff4f62aea3a9a14237ed01
+  data.tar.gz: 5080f643d1da251f05afb094a1c7ac9c5cbbe140a15f0d05b77ec87591e700b148ba4766615ee1853284d24674e2720149f25e6858b485e0123ebd3add74c4db

data/lib/aranha/parsers/base.rb

@@ -11,7 +11,9 @@ module Aranha
       class << self
         def from_content(content)
           ::EacRubyUtils::Fs::Temp.on_file do |path|
-            path.write(content)
+            ::File.open(path.to_s, 'w:UTF-8') do |f|
+              f.write content.force_encoding('UTF-8')
+            end
             r = new(path.to_path)
             r.content
             r

data/lib/aranha/parsers/html/node/default/numeric_support.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module NumericSupport
+            def integer_value(node, xpath)
+              r = string_value(node, xpath)
+              return nil if r.blank?
+
+              m = /\d+/.match(r)
+              raise "Integer not found in \"#{r}\"" unless m
+
+              m[0].to_i
+            end
+
+            def integer_optional_value(node, xpath)
+              r = string_value(node, xpath)
+              m = /\d+/.match(r)
+              m ? m[0].to_i : nil
+            end
+
+            def float_value(node, xpath)
+              parse_float(node, xpath, true)
+            end
+
+            def float_optional_value(node, xpath)
+              parse_float(node, xpath, false)
+            end
+
+            def us_decimal_value(node, xpath)
+              parse_us_decimal(node, xpath, true)
+            end
+
+            def us_decimal_optional_value(node, xpath)
+              parse_us_decimal(node, xpath, false)
+            end
+
+            private
+
+            def parse_float(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete('.').tr(',', '.').to_f
+              elsif required
+                raise "Float value not found in \"#{s}\""
+              end
+            end
+
+            def parse_us_decimal(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete(',').to_f
+              elsif required
+                raise "US decimal value not found in \"#{s}\""
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default/string_support.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module StringSupport
+            def quoted_value(node, xpath)
+              s = string_value(node, xpath)
+              return '' unless s
+
+              m = /\"([^\"]+)\"/.match(s)
+              return m[1] if m
+
+              ''
+            end
+
+            def regxep(node, xpath, pattern)
+              s = string_value(node, xpath)
+              m = pattern.match(s)
+              return m if m
+
+              raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
+            end
+
+            def string_value(node, xpath)
+              if node.at_xpath(xpath)
+                sanitize_string(node.at_xpath(xpath).text)
+              else
+                ''
+              end
+            end
+
+            def string_recursive_value(node, xpath, required = true)
+              root = node.at_xpath(xpath)
+              if root.blank?
+                return nil unless required
+
+                raise "No node found (Xpath: #{xpath})"
+              end
+              result = string_recursive(root)
+              return result if result.present?
+              return nil unless required
+
+              raise "String blank (Xpath: #{xpath})"
+            end
+
+            def string_recursive_optional_value(node, xpath)
+              string_recursive_value(node, xpath, false)
+            end
+
+            private
+
+            def sanitize_string(obj)
+              obj.to_s.tr("\u00A0", ' ').strip
+            end
+
+            def string_recursive(node)
+              return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
+
+              s = ''
+              node.children.each do |child|
+                child_s = string_recursive(child)
+                s += ' ' + child_s if child_s.present?
+              end
+              sanitize_string(s)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default.rb

@@ -1,71 +1,14 @@
 # frozen_string_literal: true
 
 require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   module Parsers
     module Html
       module Node
         class Default < ::Aranha::Parsers::Html::Node::Base
-          def string_value(node, xpath)
-            if node.at_xpath(xpath)
-              sanitize_string(node.at_xpath(xpath).text)
-            else
-              ''
-            end
-          end
-
-          def string_recursive_value(node, xpath, required = true)
-            root = node.at_xpath(xpath)
-            if root.blank?
-              return nil unless required
-
-              raise "No node found (Xpath: #{xpath})"
-            end
-            result = string_recursive(root)
-            return result if result.present?
-            return nil unless required
-
-            raise "String blank (Xpath: #{xpath})"
-          end
-
-          def string_recursive_optional_value(node, xpath)
-            string_recursive_value(node, xpath, false)
-          end
-
-          def quoted_value(node, xpath)
-            s = string_value(node, xpath)
-            return '' unless s
-
-            m = /\"([^\"]+)\"/.match(s)
-            return m[1] if m
-
-            ''
-          end
-
-          def integer_value(node, xpath)
-            r = string_value(node, xpath)
-            return nil if r.blank?
-
-            m = /\d+/.match(r)
-            raise "Integer not found in \"#{r}\"" unless m
-
-            m[0].to_i
-          end
-
-          def integer_optional_value(node, xpath)
-            r = string_value(node, xpath)
-            m = /\d+/.match(r)
-            m ? m[0].to_i : nil
-          end
-
-          def float_value(node, xpath)
-            parse_float(node, xpath, true)
-          end
-
-          def float_optional_value(node, xpath)
-            parse_float(node, xpath, false)
-          end
+          require_sub __FILE__, include_modules: true
 
           def array_value(node, xpath)
             r = node.xpath(xpath).map { |n| n.text.strip }
@@ -84,41 +27,6 @@ module Aranha
             m = /(\d+) m/.match(join_value(node, xpath))
             m ? m[1].to_i : nil
           end
-
-          def regxep(node, xpath, pattern)
-            s = string_value(node, xpath)
-            m = pattern.match(s)
-            return m if m
-
-            raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
-          end
-
-          private
-
-          def parse_float(node, xpath, required)
-            s = string_value(node, xpath)
-            m = /\d+(?:[\.\,](\d+))?/.match(s)
-            if m
-              m[0].delete('.').tr(',', '.').to_f
-            elsif required
-              raise "Float value not found in \"#{s}\""
-            end
-          end
-
-          def sanitize_string(obj)
-            obj.to_s.tr("\u00A0", ' ').strip
-          end
-
-          def string_recursive(node)
-            return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
-
-            s = ''
-            node.children.each do |child|
-              child_s = string_recursive(child)
-              s += ' ' + child_s if child_s.present?
-            end
-            sanitize_string(s)
-          end
         end
       end
     end

data/lib/aranha/parsers/json/base.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/base'
+require 'json'
+
+module Aranha
+  module Parsers
+    module Json
+      class Base < ::Aranha::Parsers::Base
+        def data
+          default_data
+        end
+
+        def default_data
+          ::JSON.parse(content)
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/json.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Json
+      require_sub __FILE__
+    end
+  end
+end

data/lib/aranha/parsers/rspec/setup.rb

@@ -7,7 +7,7 @@ module Aranha
     module Rspec
       module Setup
         def self.extended(_setup_obj)
-          require 'aranha/parsers/rspec/source_target_fixtures_example'
+          require 'aranha/parsers/rspec/shared_examples/source_target_fixtures'
         end
       end
     end

data/lib/aranha/parsers/source_address/fetch_content_error.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class FetchContentError < ::RuntimeError
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -2,6 +2,8 @@
 
 require 'addressable'
 require 'curb'
+require 'aranha/parsers/source_address/fetch_content_error'
+require 'faraday_middleware'
 
 module Aranha
   module Parsers
@@ -31,13 +33,21 @@ module Aranha
           source
         end
 
+        def final_url
+          content unless @final_url
+          @final_url
+        end
+
         def content
-          c = ::Curl::Easy.new(url)
-          c.follow_location = true
-          raise "Curl perform failed (URL: #{url})" unless c.perform
-          return c.body_str if c.status.to_i == 200
+          conn = ::Faraday.new do |f|
+            f.request :retry # retry transient failures
+            f.response :follow_redirects # follow redirects
+          end
+          c = conn.get(url)
+          return c.body if c.status == 200
 
-          raise "Get #{url} returned #{c.status.to_i}"
+          raise ::Aranha::Parsers::SourceAddress::FetchContentError,
+                "Get #{url} returned #{c.status.to_i}"
         end
 
         def serialize

data/lib/aranha/parsers/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Parsers
-    VERSION = '0.8.3'
+    VERSION = '0.10.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-parsers
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.10.0
 platform: ruby
 authors:
 - Esquilo Azul Company
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.74'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +100,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.8.3
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.4
 - !ruby/object:Gem::Dependency
   name: ofx-parser
   requirement: !ruby/object:Gem::Requirement
@@ -136,13 +170,18 @@ files:
 - lib/aranha/parsers/html/node.rb
 - lib/aranha/parsers/html/node/base.rb
 - lib/aranha/parsers/html/node/default.rb
+- lib/aranha/parsers/html/node/default/numeric_support.rb
+- lib/aranha/parsers/html/node/default/string_support.rb
 - lib/aranha/parsers/invalid_state_exception.rb
+- lib/aranha/parsers/json.rb
+- lib/aranha/parsers/json/base.rb
 - lib/aranha/parsers/patches.rb
 - lib/aranha/parsers/patches/ofx_parser.rb
 - lib/aranha/parsers/rspec.rb
 - lib/aranha/parsers/rspec/setup.rb
-- lib/aranha/parsers/rspec/source_target_fixtures_example.rb
+- lib/aranha/parsers/rspec/shared_examples/source_target_fixtures.rb
 - lib/aranha/parsers/source_address.rb
+- lib/aranha/parsers/source_address/fetch_content_error.rb
 - lib/aranha/parsers/source_address/file.rb
 - lib/aranha/parsers/source_address/hash_http_get.rb
 - lib/aranha/parsers/source_address/hash_http_post.rb