eac_tools 0.37.1 → 0.38.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a06d6cf3c9faa8f37538fc898007ccf3501d0fbff41a883ba7b36a9d8f9a366
-  data.tar.gz: 6bf1bd1dce7b5cedce59963194408964ed49ea4cf85a44609df482e29bddc9d0
+  metadata.gz: 177dcaeff3e81307599207d1fdd3a52e8e88c3461e08a5e4c8319b1a354bd3d5
+  data.tar.gz: 6c3fad0d63f50e97c7cc0fffcc7384c66e6066743874916acd5997237c8b6142
 SHA512:
-  metadata.gz: 55243dbe562ad019575f0014096a024aeb844723597fe4b00dc415c1f234a1424fca4d80498c7c679d9ace63fc0ead2c205b4a224ebf8bbca1f95d9fb96fd92d
-  data.tar.gz: 5b47516144519424c1d3345460f6d36b5fd9d011015a1daa7342b9656d9512e53d0954c59a4545f71395f0a0fcedb74bc352edc7ba6406bf33806fe7c46726e7
+  metadata.gz: e5628867398318c613bd94c4da4dff8e069553ec4bab91b8dfefacd137bf74ed4294047b31455247f4187b39b45b23d898cb318cfca269f03a3c6dcd6458194c
+  data.tar.gz: 4f97c29c0ff189dd0ddc6dbc29d20c4b665d57a6bdc677a98a651427d1ee758fa275c8f29d931f466d6ff981f36ee5f075cf300a45d9486dda125c70f8c5b5f1

data/Gemfile.lock

@@ -1,21 +1,34 @@
 PATH
   remote: .
   specs:
-    eac_tools (0.37.1)
-      avm (~> 0.50)
+    eac_tools (0.38.0)
+      avm (~> 0.52)
       avm-eac_asciidoctor_base0 (~> 0.7, >= 0.7.1)
-      avm-eac_generic_base0 (~> 0.5)
-      avm-eac_latex_base0 (~> 0.2, >= 0.2.1)
-      avm-eac_php_base0 (~> 0.1)
+      avm-eac_generic_base0 (~> 0.7)
+      avm-eac_latex_base0 (~> 0.3)
+      avm-eac_php_base0 (~> 0.3)
       avm-eac_rails_base0 (~> 0.8, >= 0.8.1)
       avm-eac_rails_base1 (~> 0.7)
       avm-eac_redmine_base0 (~> 0.16)
       avm-eac_redmine_plugin_base0 (~> 0.3)
-      avm-eac_ruby_base1 (~> 0.25)
-      avm-eac_webapp_base0 (~> 0.10)
+      avm-eac_ruby_base1 (~> 0.26, >= 0.26.1)
+      avm-eac_webapp_base0 (~> 0.12)
       avm-eac_wordpress_base0 (~> 0.2, >= 0.2.1)
-      avm-tools (~> 0.142, >= 0.142.1)
-      eac_ruby_utils (~> 0.105)
+      avm-tools (~> 0.142, >= 0.142.3)
+      eac_ruby_utils (~> 0.106)
+
+PATH
+  remote: sub/aranha-parsers
+  specs:
+    aranha-parsers (0.14.4)
+      activesupport (>= 4.0.0)
+      addressable (~> 2.8, >= 2.8.1)
+      curb (~> 0.9, >= 0.9.11)
+      eac_ruby_utils (~> 0.106)
+      faraday-gzip (~> 0.1)
+      faraday_middleware (~> 1.2)
+      nokogiri (~> 1.13, >= 1.13.9)
+      ofx-parser (~> 1.1)
 
 PATH
   remote: sub/avm-eac_asciidoctor_base0
@@ -30,23 +43,31 @@ PATH
 PATH
   remote: sub/avm-eac_generic_base0
   specs:
-    avm-eac_generic_base0 (0.5.0)
-      avm (~> 0.28)
-      eac_ruby_utils (~> 0.96)
+    avm-eac_generic_base0 (0.7.0)
+      avm (~> 0.52)
+      eac_ruby_utils (~> 0.105)
 
 PATH
   remote: sub/avm-eac_latex_base0
   specs:
-    avm-eac_latex_base0 (0.2.1)
-      avm (~> 0.45)
-      avm-eac_webapp_base0 (~> 0.9)
-      eac_ruby_utils (~> 0.104)
+    avm-eac_latex_base0 (0.3.0)
+      avm (~> 0.52)
+      avm-eac_webapp_base0 (~> 0.12)
+      eac_ruby_utils (~> 0.106)
 
 PATH
   remote: sub/avm-eac_php_base0
   specs:
-    avm-eac_php_base0 (0.1.0)
-      eac_ruby_utils (~> 0.104)
+    avm-eac_php_base0 (0.3.0)
+      avm-eac_generic_base0 (~> 0.7)
+      eac_ruby_utils (~> 0.106)
+
+PATH
+  remote: sub/avm-eac_python_base0
+  specs:
+    avm-eac_python_base0 (0.2.0)
+      avm-eac_generic_base0 (~> 0.7)
+      eac_ruby_utils (~> 0.106)
 
 PATH
   remote: sub/avm-eac_rails_base0
@@ -87,20 +108,20 @@ PATH
 PATH
   remote: sub/avm-eac_ruby_base1
   specs:
-    avm-eac_ruby_base1 (0.25.0)
-      avm (~> 0.50)
-      avm-eac_generic_base0 (~> 0.5)
-      eac_ruby_utils (~> 0.105)
+    avm-eac_ruby_base1 (0.26.1)
+      avm (~> 0.52)
+      avm-eac_generic_base0 (~> 0.7)
+      eac_ruby_utils (~> 0.106)
 
 PATH
   remote: sub/avm-eac_webapp_base0
   specs:
-    avm-eac_webapp_base0 (0.10.0)
-      avm (~> 0.50)
-      avm-eac_generic_base0 (~> 0.5)
+    avm-eac_webapp_base0 (0.12.0)
+      avm (~> 0.52)
+      avm-eac_generic_base0 (~> 0.7)
       avm-eac_postgresql_base0 (~> 0.2)
       avm-eac_ubuntu_base0 (~> 0.3)
-      eac_ruby_utils (~> 0.105)
+      eac_ruby_utils (~> 0.106)
 
 PATH
   remote: sub/avm-eac_wordpress_base0
@@ -113,11 +134,11 @@ PATH
 PATH
   remote: sub/avm-files
   specs:
-    avm-files (0.4.1)
-      avm (~> 0.43)
+    avm-files (0.6.0)
+      avm (~> 0.52)
       eac_cli (~> 0.28)
       eac_fs (~> 0.12, >= 0.12.3)
-      eac_ruby_utils (~> 0.103)
+      eac_ruby_utils (~> 0.106)
       eac_templates (~> 0.3, >= 0.3.2)
 
 PATH
@@ -133,12 +154,12 @@ PATH
 PATH
   remote: sub/avm-tools
   specs:
-    avm-tools (0.142.1)
-      aranha-parsers (~> 0.14, >= 0.14.3)
-      avm (~> 0.50)
-      avm-eac_ruby_base1 (~> 0.25)
+    avm-tools (0.142.3)
+      aranha-parsers (~> 0.14, >= 0.14.4)
+      avm (~> 0.52)
+      avm-eac_ruby_base1 (~> 0.26, >= 0.26.1)
       avm-eac_ubuntu_base0 (~> 0.3)
-      avm-files (~> 0.4, >= 0.4.1)
+      avm-files (~> 0.6)
       avm-git (~> 0.8)
       clipboard (~> 1.3, >= 1.3.6)
       curb (~> 0.9, >= 0.9.11)
@@ -151,11 +172,13 @@ PATH
 PATH
   remote: sub/avm
   specs:
-    avm (0.50.0)
+    avm (0.52.0)
+      aranha-parsers (~> 0.14, >= 0.14.4)
       eac_cli (~> 0.28)
       eac_config (~> 0.11, >= 0.11.1)
       eac_docker (~> 0.5)
-      eac_git (~> 0.12, >= 0.12.3)
+      eac_fs (~> 0.12, >= 0.12.3)
+      eac_git (~> 0.13)
       eac_ruby_utils (~> 0.105)
       eac_templates (~> 0.3, >= 0.3.2)
       filesize (~> 0.2)
@@ -211,7 +234,7 @@ PATH
 PATH
   remote: sub/eac_ruby_utils
   specs:
-    eac_ruby_utils (0.105.0)
+    eac_ruby_utils (0.106.0)
       activesupport (>= 4, < 7)
       addressable (~> 2.8, >= 2.8.1)
       bundler
@@ -229,15 +252,6 @@ GEM
       zeitwerk (~> 2.3)
     addressable (2.8.1)
       public_suffix (>= 2.0.2, < 6.0)
-    aranha-parsers (0.14.3)
-      activesupport (>= 4.0.0)
-      addressable (~> 2.8, >= 2.8.1)
-      curb (~> 0.9, >= 0.9.11)
-      eac_ruby_utils (~> 0.102, >= 0.102.1)
-      faraday-gzip (~> 0.1)
-      faraday_middleware (~> 1.2)
-      nokogiri (~> 1.13, >= 1.13.8)
-      ofx-parser (~> 1.1)
     asciidoctor (2.0.18)
     ast (2.4.2)
     avm-eac_postgresql_base0 (0.2.0)
@@ -306,7 +320,7 @@ GEM
     minitest (5.16.3)
     multipart-post (2.2.3)
     net-ssh (4.2.0)
-    nokogiri (1.13.8-x86_64-linux)
+    nokogiri (1.13.9-x86_64-linux)
       racc (~> 1.4)
     ofx-parser (1.1.0)
       hpricot (>= 0.6)
@@ -362,11 +376,13 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  aranha-parsers!
   avm!
   avm-eac_asciidoctor_base0!
   avm-eac_generic_base0!
   avm-eac_latex_base0!
   avm-eac_php_base0!
+  avm-eac_python_base0!
   avm-eac_rails_base0!
   avm-eac_rails_base1!
   avm-eac_redmine_base0!

data/lib/eac_tools/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EacTools
-  VERSION = '0.37.1'
+  VERSION = '0.38.0'
 end

data/sub/aranha-parsers/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec
+
+local_gemfile = ::File.join(::File.dirname(__FILE__), 'Gemfile.local')
+eval_gemfile local_gemfile if ::File.exist?(local_gemfile)

data/sub/aranha-parsers/aranha-parsers.gemspec

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+
+require 'aranha/parsers/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'aranha-parsers'
+  s.version     = ::Aranha::Parsers::VERSION
+  s.authors     = ['Esquilo Azul Company']
+  s.summary     = 'Parsers\' utilities for Ruby.'
+
+  s.files = Dir['{lib}/**/*']
+
+  s.add_dependency 'activesupport', '>= 4.0.0'
+  s.add_dependency 'addressable', '~> 2.8', '>= 2.8.1'
+  s.add_dependency 'curb', '~> 0.9', '>= 0.9.11'
+  s.add_dependency 'eac_ruby_utils', '~> 0.106'
+  s.add_dependency 'faraday-gzip', '~> 0.1'
+  s.add_dependency 'faraday_middleware', '~> 1.2'
+  s.add_dependency 'nokogiri', '~> 1.13', '>= 1.13.9'
+  s.add_dependency 'ofx-parser', '~> 1.1'
+
+  s.add_development_dependency 'eac_ruby_gem_support', '~> 0.5.1'
+end

data/sub/aranha-parsers/lib/aranha/parsers/base.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'fileutils'
+require 'aranha/parsers/source_address'
+require 'eac_ruby_utils/fs/temp'
+
+module Aranha
+  module Parsers
+    class Base
+      class << self
+        def from_content(content)
+          ::EacRubyUtils::Fs::Temp.on_file do |path|
+            ::File.open(path.to_s, 'w:UTF-8') do |f|
+              f.write content.dup.force_encoding('UTF-8')
+            end
+            r = new(path.to_path)
+            r.content
+            r
+          end
+        end
+
+        def parse_content(content)
+          from_content(content).data
+        end
+      end
+
+      LOG_DIR_ENVVAR = 'ARANHA_PARSERS_LOG_DIR'
+
+      attr_reader :source_address
+
+      def initialize(url)
+        @source_address = ::Aranha::Parsers::SourceAddress.new(url)
+        log_content(source_address.serialize, '-source-address')
+      end
+
+      delegate :url, to: :source_address
+
+      def content
+        @content ||= log_content(source_address_content)
+      end
+
+      # @return [String]
+      delegate :content, to: :source_address, prefix: true
+
+      private
+
+      # @return [String]
+      def log_content(content, suffix = '')
+        path = log_file(suffix)
+
+        File.open(path, 'wb') { |file| file.write(content) } if path
+
+        content
+      end
+
+      def log_file(suffix)
+        dir = log_parsers_dir
+        return nil unless dir
+
+        f = ::File.join(dir, "#{self.class.name.parameterize}#{suffix}.log")
+        FileUtils.mkdir_p(File.dirname(f))
+        f
+      end
+
+      def log_parsers_dir
+        return ENV[LOG_DIR_ENVVAR] if ENV[LOG_DIR_ENVVAR]
+        return ::Rails.root.join('log', 'parsers') if rails_root_exist?
+
+        nil
+      end
+
+      def rails_root_exist?
+        ::Rails.root
+        true
+      rescue NameError
+        false
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/base.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'aranha/parsers/base'
+require 'aranha/parsers/html/node/default'
+
+module Aranha
+  module Parsers
+    module Html
+      class Base < ::Aranha::Parsers::Base
+        class << self
+          def fields
+            @fields ||= []
+            @fields.dup
+          end
+
+          def field(name, type, xpath)
+            @fields ||= []
+            @fields << Field.new(name, type, xpath)
+          end
+
+          Field = Struct.new(:name, :type, :xpath)
+        end
+
+        def nokogiri
+          @nokogiri ||= Nokogiri::HTML(content, &:noblanks)
+        end
+
+        protected
+
+        def node_parser_class
+          ::Aranha::Parsers::Html::Node::Default
+        end
+
+        private
+
+        def node_parser
+          @node_parser ||= node_parser_class.new(fields)
+        end
+
+        def fields
+          self.class.fields.map { |f| [f.name, f.type, f.xpath] }
+        end
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/item.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/base'
+
+module Aranha
+  module Parsers
+    module Html
+      class Item < Base
+        def data
+          @data ||= node_parser.parse(item_node)
+        end
+
+        def item_node
+          @item_node ||= begin
+            r = item_xpath ? nokogiri.at_xpath(item_xpath) : nokogiri
+            raise "Item node not found (Item xpath: #{item_xpath})" unless r
+
+            r
+          end
+        end
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/item_list.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/base'
+
+module Aranha
+  module Parsers
+    module Html
+      class ItemList < Base
+        def data
+          items_data
+        end
+
+        def item_data(item)
+          item
+        end
+
+        def items_data
+          count = 0
+          @data ||= nokogiri.xpath(items_xpath).map do |m|
+            count += 1
+            item_data(node_parser.parse(m))
+          end
+        rescue StandardError => e
+          raise StandardError, "#{e.message} (Count: #{count})"
+        end
+
+        def items_xpath
+          raise "Class #{self.class} has no method \"#{__method__}\". Implement it"
+        end
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/node/base.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Base
+          attr_reader :fields
+
+          def initialize(fields)
+            @fields = fields
+          end
+
+          def parse(node)
+            fields.map do |f|
+              [f[0], parse_field(node, f[2], f[1])]
+            rescue StandardError => e
+              raise StandardError, "#{e.message}\nFields: #{f}"
+            end.to_h
+          end
+
+          private
+
+          def parse_field(node, xpath, parser_method)
+            value_method = "#{parser_method}_value"
+            return send(value_method, node, xpath) if respond_to?(value_method)
+
+            raise "Method \"#{value_method}\" not found in #{self.class}"
+          end
+        end
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/node/default/numeric_support.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module NumericSupport
+            def integer_value(node, xpath)
+              r = string_value(node, xpath)
+              return nil if r.blank?
+
+              m = /\d+/.match(r)
+              raise "Integer not found in \"#{r}\"" unless m
+
+              m[0].to_i
+            end
+
+            def integer_optional_value(node, xpath)
+              r = string_value(node, xpath)
+              m = /\d+/.match(r)
+              m ? m[0].to_i : nil
+            end
+
+            def float_value(node, xpath)
+              parse_float(node, xpath, true)
+            end
+
+            def float_optional_value(node, xpath)
+              parse_float(node, xpath, false)
+            end
+
+            def us_decimal_value(node, xpath)
+              parse_us_decimal(node, xpath, true)
+            end
+
+            def us_decimal_optional_value(node, xpath)
+              parse_us_decimal(node, xpath, false)
+            end
+
+            private
+
+            def parse_float(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete('.').tr(',', '.').to_f
+              elsif required
+                raise "Float value not found in \"#{s}\""
+              end
+            end
+
+            def parse_us_decimal(node, xpath, required)
+              s = string_value(node, xpath)
+              m = /\d+(?:[\.\,](\d+))?/.match(s)
+              if m
+                m[0].delete(',').to_f
+              elsif required
+                raise "US decimal value not found in \"#{s}\""
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/node/default/string_support.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          module StringSupport
+            def quoted_value(node, xpath)
+              s = string_value(node, xpath)
+              return '' unless s
+
+              m = /\"([^\"]+)\"/.match(s)
+              return m[1] if m
+
+              ''
+            end
+
+            def regxep(node, xpath, pattern)
+              s = string_value(node, xpath)
+              m = pattern.match(s)
+              return m if m
+
+              raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
+            end
+
+            def string_value(node, xpath)
+              if node.at_xpath(xpath)
+                sanitize_string(node.at_xpath(xpath).text)
+              else
+                ''
+              end
+            end
+
+            def string_recursive_value(node, xpath, required = true)
+              root = node.at_xpath(xpath)
+              if root.blank?
+                return nil unless required
+
+                raise "No node found (Xpath: #{xpath})"
+              end
+              result = string_recursive(root)
+              return result if result.present?
+              return nil unless required
+
+              raise "String blank (Xpath: #{xpath})"
+            end
+
+            def string_recursive_optional_value(node, xpath)
+              string_recursive_value(node, xpath, false)
+            end
+
+            private
+
+            def sanitize_string(obj)
+              obj.to_s.tr("\u00A0", ' ').strip
+            end
+
+            def string_recursive(node)
+              return sanitize_string(node.text) if node.is_a?(::Nokogiri::XML::Text)
+
+              s = ''
+              node.children.each do |child|
+                child_s = string_recursive(child)
+                s += ' ' + child_s if child_s.present?
+              end
+              sanitize_string(s)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/node/default.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          require_sub __FILE__, include_modules: true
+
+          def array_value(node, xpath)
+            r = node.xpath(xpath).map { |n| n.text.strip }
+            r.join('|')
+          end
+
+          def join_value(node, xpath)
+            m = ''
+            node.xpath(xpath).each do |n|
+              m << n.text.strip
+            end
+            m
+          end
+
+          def duration_value(node, xpath)
+            m = /(\d+) m/.match(join_value(node, xpath))
+            m ? m[1].to_i : nil
+          end
+        end
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html/node.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/require_sub'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        ::EacRubyUtils.require_sub __FILE__
+      end
+    end
+  end
+end

data/sub/aranha-parsers/lib/aranha/parsers/html.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/require_sub'
+
+module Aranha
+  module Parsers
+    module Html
+      ::EacRubyUtils.require_sub __FILE__
+    end
+  end
+end