eco-helpers 3.0.18 → 3.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47c28bbf13f50c0f3f895c40c0ad8532d13233e0e2b81faea474bbeb0a1c2308
-  data.tar.gz: e13091192acd044b667cd20ebe7bcaf5c17da30c1ef83266fefa9ffa90fb8348
+  metadata.gz: 0f6637f51c2d892eb93c881cca7613fba034cbb00ff2676966172c2277f583ce
+  data.tar.gz: 68703625905aa5ed2c00223a628de16612714a2c2cdd69a1c0a284c8c6096332
 SHA512:
-  metadata.gz: 02fe482f0c75e0926fe147e1ab3f4b0a0d1b826990bae3ad8ea97fb0aa53a800e1d208fb62e209877c0594b4f32792096967ec1f0528be8c81c3cd8c404c5131
-  data.tar.gz: 4f9c12554b25c81243eac348f106392a092853c87204922d25cd1f25d287abcc23461df2b0a6a19b3dd3e4624f625e49ce5920ed04000ab6641f26312a38cb0f
+  metadata.gz: 89bbd57468fd7de3412240e3e44f61122aee9982896401494c4ee88c5e2dec97563e421cac2da0794c4f68ff90206422bc52b2868ee1e7ef3422fce50f3ec73d
+  data.tar.gz: 82703dafe2ccecf25994567926c80e7bd40fe0af70853cce664dbcbfc12b56b0b9ad72308ccf2789a386e053baadce2a8c29095130aa57dc560228d99c27267e

data/CHANGELOG.md

@@ -2,10 +2,29 @@
 
 All notable changes to this project will be documented in this file.
 
-## [3.0.18] - 2024-10-xx
+## [3.0.19] - 2024-10-xx
 
 ### Added
 
+- Input entries `:json` parser
+- **Cases**:
+  - `json-to-csv`
+  - `split-json`
+  - `sort-csv`
+  - `group-csv`
+
+### Changed
+
+- Refactored input file parsers (`csv`, `xls`)
+- Upgraded gems:
+  - `ecoportal-api`
+  - `ecoportal-api-v2`
+  - `ecoportal-api-graphql`
+
+### Fixed
+
+## [3.0.18] - 2024-10-28
+
 ### Changed
 
 - upgrade `ecoportal-api` gem
@@ -15,8 +34,6 @@ All notable changes to this project will be documented in this file.
   - **added** auto-swap endpoint from `job` to `batch` on TimeOut errors.
   - **retries** also on `Ecoportal::API::Errors::StartTimeOut`
 
-### Fixed
-
 ## [3.0.17] - 2024-10-18
 
 ### Added

data/eco-helpers.gemspec

@@ -40,9 +40,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'bcrypt_pbkdf',  '~> 1.0'
   spec.add_dependency 'docx',          '>= 0.8.0', '< 0.9'
   spec.add_dependency 'dotenv',        '~> 3'
-  spec.add_dependency 'ecoportal-api',         '~> 0.10', '>= 0.10.6'
-  spec.add_dependency 'ecoportal-api-graphql', '~> 0.4',  '>= 0.4.2'
-  spec.add_dependency 'ecoportal-api-v2',      '~> 2.0',  '>= 2.0.11'
+  spec.add_dependency 'ecoportal-api',         '~> 0.10', '>= 0.10.7'
+  spec.add_dependency 'ecoportal-api-graphql', '~> 0.4',  '>= 0.4.3'
+  spec.add_dependency 'ecoportal-api-v2',      '~> 2.0',  '>= 2.0.12'
   spec.add_dependency 'ed25519',       '~> 1.2'
   spec.add_dependency 'fast_excel',    '>= 0.5.0',  '< 0.6'
   spec.add_dependency 'fuzzy_match',   '>= 2.1.0',  '< 2.2'

data/lib/eco/api/common/loaders/parser.rb

@@ -21,6 +21,7 @@ module Eco
               return []    if miss.empty?
               return attrs if match.empty?
               return miss  if type == :all
+
               []
             end
           end
@@ -38,6 +39,7 @@ module Eco
                 msg << "#{self.class}, is linked to"
                 return @attribute || (raise msg)
               end
+
               name value
               @attribute = value
             end
@@ -62,6 +64,7 @@ module Eco
             def parsing_phase(phase = nil)
               @parsing_phase ||= :internal
               return @parsing_phase unless phase
+
               @parsing_phase = phase
             end
 
@@ -71,6 +74,7 @@ module Eco
             def serializing_phase(phase = nil)
               @serializing_phase ||= :person
               return @serializing_phase unless phase
+
               @serializing_phase = phase
             end
 
@@ -86,6 +90,7 @@ module Eco
             # Helper to build the `active_when` condition.
             def active_when_all(*attrs)
               @active_when_attrs = RequiredAttrs.new(attribute, :all, attrs)
+
               @active_when = proc do |source_data|
                 keys = data_keys(source_data)
                 attrs.all? {|key| keys.include?(key)}
@@ -149,6 +154,10 @@ module Eco
 
           private
 
+          def options
+            ASSETS.cli.options
+          end
+
           def _define_parser(attr_parser)
             if (active_when = self.class.active_when)
               attr_parser.def_parser(
@@ -166,6 +175,7 @@ module Eco
 
           def _define_serializer(attr_parser)
             return unless respond_to?(:serializer, true)
+
             attr_parser.def_serializer(
               self.class.serializing_phase,
               &method(:serializer)

data/lib/eco/api/common/people/default_parsers/csv_parser.rb

@@ -1,235 +1,48 @@
 class Eco::API::Common::People::DefaultParsers::CSVParser < Eco::API::Common::Loaders::Parser
   attribute :csv
 
+  include Eco::API::Common::People::DefaultParsers::Helpers::ExpectedHeaders
+  include Eco::API::Common::People::DefaultParsers::Helpers::NullParsing
+
   def parser(data, deps)
     Eco::CSV.parse(data, headers: true, skip_blanks: true).tap do |table|
-      require_headers!(table)
-      check_headers(table) if deps[:check_headers] && check_headers?
-    end.each_with_object([]) do |row, arr_hash|
-      row_hash = row.headers.uniq.each_with_object({}) do |attr, hash|
+      require_headers!(table.headers)
+
+      next unless deps[:check_headers]
+      next unless check_headers?
+
+      check_headers!(
+        table.headers,
+        order_check: options.dig(:input, :header_check, :order)
+      )
+    end.each_with_object([]) do |item, arr_hash|
+      item_hash = item.headers.uniq.each_with_object({}) do |attr, hash|
         next if attr.to_s.strip.empty?
-        hash[attr.strip] = parse_string(row[attr])
+
+        hash[attr.strip] = parse_null(item[attr])
       end
-      arr_hash.push(row_hash)
+
+      arr_hash.push(item_hash)
     end
   end
 
   def serializer(array_hash, _deps)
     arr_rows = []
+
     unless array_hash.empty?
       header = array_hash.first.keys
+
       arr_rows = array_hash.map do |csv_row|
         CSV::Row.new(header, csv_row.values_at(*header))
       end
     end
+
     CSV::Table.new(arr_rows).to_csv
   end
 
   private
 
-  def abort(msg)
-    super(msg, raising: false)
-  end
-
-  def require_headers!(table)
-    headers = table.headers
-
-    abort("Missing headers in CSV") unless headers&.any?
-
-    empty = []
-    headers.each_with_index do |header, idx|
-      empty << idx if header.to_s.strip.empty?
-    end
-
-    abort("Empty headers in column(s): #{empty.join(', ')}") if empty.any?
-
-    true
-  end
-
   def check_headers?
     !options.dig(:input, :header_check, :skip)
   end
-
-  def options
-    ASSETS.cli.options
-  end
-
-  def parse_string(value)
-    return nil if value.to_s.empty?
-    return nil if null?(value)
-    value
-  end
-
-  def null?(value)
-    return true unless value
-
-    str = value.strip.upcase
-    ["NULL"].any? {|token| str == token}
-  end
-
-  def check_headers(table) # rubocop:disable Metrics/AbcSize
-    headers  = table.headers
-    unmatch  = []
-    unmatch  = unmatched_headers(headers) if options.dig(:input, :header_check, :order)
-    missing  = missing_headers(headers)
-    unknown  = unknown_headers(headers)
-    criteria = [unknown, missing[:direct], missing[:indirect], unmatch]
-    return if criteria.all?(&:empty?)
-
-    msg = "Detected possible HEADER ISSUES !!!\n"
-
-    # requires exact match
-    unless unmatch.empty?
-      msg << "CSV headers do NOT exactly match the expected:\n"
-      msg << "  * Expected: #{known_headers}\n"
-      expected, given = unmatch.first
-      msg << "  * First unmatch => Given: '#{given}' where expected '#{expected}'\n"
-      missed = known_headers - headers
-      unless missed.empty?
-        msg << "  * Missing headers:\n"
-        msg << "    - #{missed.join("\n    - ")}\n"
-      end
-    end
-
-    msg << "Missing or Wrong HEADER names in the CSV file:\n"
-    msg << "  * UNKNOWN (or not used?): #{unknown}\n" unless unknown.empty?
-    msg << "  * MISSING HEADER: #{missing[:direct]}\n" unless missing[:direct].empty?
-
-    unless (data = missing[:indirect]).empty?
-      msg << "  * MISSING INDIRECTLY:\n"
-      data.each do |ext, info|
-        msg <<  "    - '#{ext}' => "
-        msg << (info[:attrs] || {}).map do |status, attrs|
-          if status == :inactive
-            "makes inactive: #{attrs}"
-          elsif status == :active
-            "there could be missing info in: #{attrs}"
-          end
-        end.compact.join("; ")
-        msg << "\n"
-      end
-    end
-
-    log(:warn) { msg }
-
-    msg = "There were issues identified on the CSV header names. Aborting..."
-    abort(msg) if options.dig(:input, :header_check, :must_be_valid)
-
-    sleep(2)
-  end
-
-  def unmatched_headers(headers)
-    known_headers.zip(headers).reject do |(expected, given)|
-      expected == given
-    end
-  end
-
-  def unknown_headers(headers)
-    (headers - known_headers) - all_internal_attrs
-  end
-
-  def missing_headers(headers) # rubocop:disable Metrics/AbcSize
-    int_head = internal_present_or_active(headers)
-    external = headers.select do |e|
-      i = fields_mapper.to_internal(e)
-      int_head.include?(i)
-    end
-
-    ext_present = known_headers_present(int_head) | external
-    ext_miss    = known_headers - ext_present
-
-    {
-      direct:   [],
-      indirect: {}
-    }.tap do |missing|
-      ext_miss.each do |ext|
-        next unless (int = fields_mapper.to_internal(ext))
-
-        missing[:direct] << ext if all_internal_attrs.include?(int)
-        related_attrs_requirements = required_attrs.values.select do |req|
-          dep       = req.dependant?(int)
-          affects   = dep && !int_head.include?(int)
-          in_header = int_head.include?(req.attr)
-          affects || (dep && !in_header)
-        end
-
-        next if related_attrs_requirements.empty?
-
-        data = missing[:indirect][ext] = {}
-        data[:int]   = int
-        data[:attrs] = {}
-
-        related_attrs_requirements.each_with_object(data[:attrs]) do |req, attrs|
-          status = req.active?(*int_head) ? :active : :inactive
-          attrs[status] ||= []
-          attrs[status] << req.attr
-        end
-      end
-    end
-  end
-
-  def known_headers_present(headers_internal)
-    known_headers.select do |ext|
-      int = fields_mapper.to_internal(ext)
-      headers_internal.include?(int)
-    end
-  end
-
-  # Scopes what internal attrs appear in headers as they are
-  def internal_present_or_active(headers, inactive_requirements = {}) # rubocop:disable Metrics/AbcSize
-    # internal attrs that are not being mapped
-    int_all     = all_internal_attrs.reject {|i| fields_mapper.external?(i)}
-    hint        = headers & int_all
-    hext        = headers - hint
-    int_present = hint + hext.map {|e| fields_mapper.to_internal(e)}.compact
-
-    update_inactive = proc do
-      inactive_requirements.dup.each do |attr, req|
-        next unless req.active?(*int_present)
-
-        inactive_requirements.delete(attr)
-        int_present << attr
-        update_inactive.call
-      end
-    end
-
-    required_attrs.each_value do |req|
-      next if int_present.include?(req)
-
-      if req.active?(*int_present)
-        inactive_requirements.delete(req.attr)
-        int_present << req.attr
-        update_inactive.call
-      else
-        inactive_requirements[req.attr] = req
-      end
-    end
-
-    int_present
-  end
-
-  # The csv header names as expected
-  def known_headers
-    @known_headers ||= fields_mapper.list(:external).compact.uniq
-  end
-
-  def fields_mapper
-    session.fields_mapper
-  end
-
-  def required_attrs
-    @required_attrs ||= person_parser.required_attrs.to_h {|ra| [ra.attr, ra]}
-  end
-
-  def all_internal_attrs
-    @all_internal_attrs ||= [].tap do |int_attrs|
-      known_int_attrs  = person_parser.all_attrs(include_defined_parsers: true)
-      known_int_attrs |= fields_mapper.list(:internal).compact
-      int_attrs.concat(known_int_attrs)
-    end
-  end
-
-  def person_parser
-    session.entry_factory.person_parser
-  end
 end

data/lib/eco/api/common/people/default_parsers/helpers/expected_headers.rb

@@ -0,0 +1,206 @@
+module Eco::API::Common::People
+  class DefaultParsers
+    module Helpers
+      module ExpectedHeaders
+        include Eco::Language::AuxiliarLogger
+
+        private
+
+        def require_headers!(raw_headers)
+          abort("Missing headers in CSV") unless raw_headers&.any?
+
+          empty = []
+          raw_headers.each_with_index do |header, idx|
+            empty << idx if header.to_s.strip.empty?
+          end
+
+          abort("Empty headers in column(s): #{empty.join(', ')}") if empty.any?
+
+          true
+        end
+
+        def check_headers!(raw_headers, order_check: false) # rubocop:disable Metrics/AbcSize
+          unmatch  = []
+          unmatch  = unmatched_headers(raw_headers) if order_check
+          missing  = missing_headers(raw_headers)
+          unknown  = unknown_headers(raw_headers)
+
+          criteria = [unknown, missing[:direct], missing[:indirect], unmatch]
+          return if criteria.all?(&:empty?)
+
+          msg = "Detected possible HEADER / FIELD  ISSUES !!!\n"
+
+          # requires exact match
+          unless unmatch.empty?
+            msg << "File headers/fields do NOT exactly match the expected:\n"
+            msg << "  * Expected: #{expected_headers}\n"
+
+            expected, given = unmatch.first
+            msg << "  * First unmatch => Given: '#{given}' where expected '#{expected}'\n"
+
+            missed = expected_headers - raw_headers
+
+            unless missed.empty?
+              msg << "  * Missing headers/fields:\n"
+              msg << "    - #{missed.join("\n    - ")}\n"
+            end
+          end
+
+          msg << "Missing or Wrong HEADER names in the file:\n"
+          msg << "  * UNKNOWN (or not used?): #{unknown}\n"  unless unknown.empty?
+          msg << "  * MISSING HEADER/FIELD: #{missing[:direct]}\n" unless missing[:direct].empty?
+
+          unless (data = missing[:indirect]).empty?
+            msg << "  * MISSING INDIRECTLY:\n"
+
+            data.each do |ext, info|
+              msg <<  "    - '#{ext}' => "
+              msg << (info[:attrs] || {}).map do |status, attrs|
+                if status == :inactive
+                  "makes inactive: #{attrs}"
+                elsif status == :active
+                  "there could be missing info in: #{attrs}"
+                end
+              end.compact.join("; ")
+
+              msg << "\n"
+            end
+          end
+
+          log(:warn) { msg }
+
+          msg = "There were issues identified on the file header/field names. Aborting..."
+          abort(msg) if options.dig(:input, :header_check, :must_be_valid)
+
+          sleep(2)
+        end
+
+
+        # @return [Hash] with missing `:direct` and `:indirect` attrs, where
+        #   - `:direct`   [Array] refers to direct attrs
+        #   - `:indirect` [Hash]  refers to indirect attrs that are `:active` or `:inactive`.
+        def missing_headers(raw_headers) # rubocop:disable Metrics/AbcSize
+          int_head = internal_present_or_active(raw_headers)
+
+          external = raw_headers.select do |e|
+            i = fields_mapper.to_internal(e)
+            int_head.include?(i)
+          end
+
+          ext_present = present_internal_expected_headers(int_head) | external
+          ext_miss    = expected_headers - ext_present
+
+          {
+            direct:   [],
+            indirect: {}
+          }.tap do |missing|
+            ext_miss.each do |ext|
+              next unless (int = fields_mapper.to_internal(ext))
+
+              missing[:direct] << ext if all_internal_attrs.include?(int)
+
+              related_attrs_requirements = required_attrs.values.select do |req|
+                dep       = req.dependant?(int)
+                affects   = dep && !int_head.include?(int)
+                in_header = int_head.include?(req.attr)
+                affects || (dep && !in_header)
+              end
+
+              next if related_attrs_requirements.empty?
+
+              data = missing[:indirect][ext] = {}
+              data[:int]   = int
+              data[:attrs] = {}
+
+              related_attrs_requirements.each_with_object(data[:attrs]) do |req, attrs|
+                status = req.active?(*int_head) ? :active : :inactive
+                attrs[status] ||= []
+                attrs[status] << req.attr
+              end
+            end
+          end
+        end
+
+
+        # The input file header names as expected
+        def expected_headers
+          @expected_headers ||= fields_mapper.list(:external).compact.uniq
+        end
+
+        def present_internal_expected_headers(internal_headers)
+          expected_headers.select do |ext|
+            int = fields_mapper.to_internal(ext)
+            internal_headers.include?(int)
+          end
+        end
+
+        def unmatched_headers(raw_headers)
+          expected_headers.zip(raw_headers).reject do |(expected, given)|
+            expected == given
+          end
+        end
+
+        def unknown_headers(raw_headers)
+          (raw_headers - expected_headers) - all_internal_attrs
+        end
+
+        def fields_mapper
+          session.fields_mapper
+        end
+
+        def required_attrs
+          @required_attrs ||= person_parser.required_attrs.to_h {|ra| [ra.attr, ra]}
+        end
+
+        def all_internal_attrs
+          @all_internal_attrs ||= [].tap do |int_attrs|
+            known_int_attrs  = person_parser.all_attrs(include_defined_parsers: true)
+            known_int_attrs |= fields_mapper.list(:internal).compact
+            int_attrs.concat(known_int_attrs)
+          end
+        end
+
+        # Scopes what internal attrs appear in headers as they are
+        def internal_present_or_active(raw_headers, inactive_requirements = {}) # rubocop:disable Metrics/AbcSize
+          # internal attrs that are not being mapped
+          int_all     = all_internal_attrs.reject {|i| fields_mapper.external?(i)}
+          hint        = raw_headers & int_all
+          hext        = raw_headers - hint
+          int_present = hint + hext.map {|e| fields_mapper.to_internal(e)}.compact
+
+          update_inactive = proc do
+            inactive_requirements.dup.each do |attr, req|
+              next unless req.active?(*int_present)
+
+              inactive_requirements.delete(attr)
+              int_present << attr
+              update_inactive.call
+            end
+          end
+
+          required_attrs.each_value do |req|
+            next if int_present.include?(req)
+
+            if req.active?(*int_present)
+              inactive_requirements.delete(req.attr)
+              int_present << req.attr
+              update_inactive.call
+            else
+              inactive_requirements[req.attr] = req
+            end
+          end
+
+          int_present
+        end
+
+        def person_parser
+          session.entry_factory.person_parser
+        end
+
+        def abort(msg)
+          super(msg, raising: false) if defined?(super)
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/people/default_parsers/helpers/null_parsing.rb

@@ -0,0 +1,36 @@
+module Eco::API::Common::People
+  class DefaultParsers
+    module Helpers
+      module NullParsing
+        private
+
+        def parse_null(value)
+          return                           if null?(value)
+          return parse_null_on_hash(value) if value.is_a?(Hash)
+          return value                     unless value.is_a?(Array)
+
+          value.map {|val| parse_null(val)}
+        end
+
+        def parse_null_on_hash(value)
+          return value unless value.is_a?(Hash)
+
+          value.dup do |out|
+            value.each do |key, val|
+              next out.delete(key) unless (out[key] = parse_null(val))
+            end
+          end
+        end
+
+        def null?(value)
+          return true  if value.nil?
+          return false unless value.is_a?(String)
+          return true  if value.strip.to_s.empty?
+
+          str = value.strip.upcase
+          ['NULL'].any? {|token| str == token}
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/people/default_parsers/helpers.rb

@@ -0,0 +1,15 @@
+require_relative 'helpers/expected_headers'
+require_relative 'helpers/null_parsing'
+
+module Eco
+  module API
+    module Common
+      module People
+        class DefaultParsers
+          module Helpers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/people/default_parsers/json_parser.rb

@@ -0,0 +1,56 @@
+class Eco::API::Common::People::DefaultParsers::JsonParser < Eco::API::Common::Loaders::Parser
+  attribute :json
+
+  include Eco::API::Common::People::DefaultParsers::Helpers::ExpectedHeaders
+  include Eco::API::Common::People::DefaultParsers::Helpers::NullParsing
+
+  def parser(filename, deps)
+    parse_json_file(filename).tap do |data|
+      # Don't enable header checks for now
+      next
+
+      puts 'Identifying unified json object keys...'
+      raw_headers = data.each_with_object([]) do |item, head|
+        head.concat(item.keys - head)
+      end
+
+      require_headers!(raw_headers)
+
+      next unless deps[:check_headers]
+      next unless check_headers?
+
+      check_headers!(
+        data,
+        order_check: false
+      )
+    end.each_with_object([]) do |item, arr_hash|
+      item_hash = item.keys.each_with_object({}) do |attr, hash|
+        next if attr.to_s.strip.empty?
+
+        hash[attr.strip] = parse_null(item[attr])
+      end
+
+      arr_hash.push(item_hash)
+    end
+  end
+
+  def serializer(array_hash, _deps)
+    array_hash.to_json
+  end
+
+  private
+
+  def check_headers?
+    !options.dig(:input, :header_check, :skip)
+  end
+
+  def parse_json_file(filename)
+    fd = File.open(filename)
+    JSON.load fd # rubocop:disable Security/JSONLoad
+  rescue JSON::ParserError => err
+    log(:error) { "Parsing error on file '#{filename}'" }
+    raise err
+  ensure
+    fd&.close
+  end
+end