eco-helpers 3.0.18 → 3.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47c28bbf13f50c0f3f895c40c0ad8532d13233e0e2b81faea474bbeb0a1c2308
-  data.tar.gz: e13091192acd044b667cd20ebe7bcaf5c17da30c1ef83266fefa9ffa90fb8348
+  metadata.gz: 96eaee3dd7897d77b22857cf8f812ce6d0639af324565c54f649c1e8068b7fb4
+  data.tar.gz: c9af686fe8f01d45eb3ea6a0dc89de276b789dfd6e3f64b515e17107b233b2d3
 SHA512:
-  metadata.gz: 02fe482f0c75e0926fe147e1ab3f4b0a0d1b826990bae3ad8ea97fb0aa53a800e1d208fb62e209877c0594b4f32792096967ec1f0528be8c81c3cd8c404c5131
-  data.tar.gz: 4f9c12554b25c81243eac348f106392a092853c87204922d25cd1f25d287abcc23461df2b0a6a19b3dd3e4624f625e49ce5920ed04000ab6641f26312a38cb0f
+  metadata.gz: 66bdc8b7c98b2aa0e60e205ac64b47a302e2cd8dbe1efd1720ad36c04a4f4c1a70b2cbb4d4978c67a17bd0f86893d34b47ff809f95a3e06952c002b872872553
+  data.tar.gz: f205d190dc159c0218bc1cafa52f139a6faaa133a01190ae1b189d2b8b6ef2470f49b634169effcf2021023803ec58f1793b8ce4c019db9d1dd025df7bbf5493

data/.gitignore

@@ -5,6 +5,7 @@ Gemfile.lock
 *.gem
 /.bundle
 /.vscode
+.solargraph.yml
 /vendor/bundle
 /spec/reports/
 /tmp/

data/CHANGELOG.md

@@ -2,10 +2,43 @@
 
 All notable changes to this project will be documented in this file.
 
-## [3.0.18] - 2024-10-xx
+## [3.0.20] - 2024-12-xx
 
 ### Added
 
+- Mailer: to specify non-default `space` in the subject, when used.
+- `Eco::API::Common::Loaders::Config`
+  - Can call config with params
+- `Eco::API::Common::Loaders::Session`
+  - Which switches the configuration to that of the specific environment
+
+### Changed
+
+- On failure, `Eco::API::Session::Job#summary` to give precise estimates on what is pending to be run.
+
+### Fixed
+
+## [3.0.19] - 2024-11-21
+
+### Added
+
+- Input entries `:json` parser
+- **Cases**:
+  - `json-to-csv`
+  - `split-json`
+  - `sort-csv`
+  - `group-csv`
+
+### Changed
+
+- Refactored input file parsers (`csv`, `xls`)
+- Upgraded gems:
+  - `ecoportal-api`
+  - `ecoportal-api-v2`
+  - `ecoportal-api-graphql`
+
+## [3.0.18] - 2024-10-28
+
 ### Changed
 
 - upgrade `ecoportal-api` gem
@@ -15,8 +48,6 @@ All notable changes to this project will be documented in this file.
   - **added** auto-swap endpoint from `job` to `batch` on TimeOut errors.
   - **retries** also on `Ecoportal::API::Errors::StartTimeOut`
 
-### Fixed
-
 ## [3.0.17] - 2024-10-18
 
 ### Added

data/eco-helpers.gemspec

@@ -40,9 +40,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'bcrypt_pbkdf',  '~> 1.0'
   spec.add_dependency 'docx',          '>= 0.8.0', '< 0.9'
   spec.add_dependency 'dotenv',        '~> 3'
-  spec.add_dependency 'ecoportal-api',         '~> 0.10', '>= 0.10.6'
-  spec.add_dependency 'ecoportal-api-graphql', '~> 0.4',  '>= 0.4.2'
-  spec.add_dependency 'ecoportal-api-v2',      '~> 2.0',  '>= 2.0.11'
+  spec.add_dependency 'ecoportal-api',         '~> 0.10', '>= 0.10.7'
+  spec.add_dependency 'ecoportal-api-graphql', '~> 0.4',  '>= 0.4.3'
+  spec.add_dependency 'ecoportal-api-v2',      '~> 2.0',  '>= 2.0.12'
   spec.add_dependency 'ed25519',       '~> 1.2'
   spec.add_dependency 'fast_excel',    '>= 0.5.0',  '< 0.6'
   spec.add_dependency 'fuzzy_match',   '>= 2.1.0',  '< 2.2'

data/lib/eco/api/common/loaders/config/session.rb

@@ -0,0 +1,12 @@
+class Eco::API::Common::Loaders::Session < Eco::API::Common::Loaders::Config
+  class << self
+    # This is a config tied to the current environment session.
+    def config
+      return @config if instance_variable_defined?(:@config)
+
+      @config = super(key: super.apis.active_root_name)
+    end
+  end
+
+  delegate_missing_to :workflow
+end

data/lib/eco/api/common/loaders/config/workflow/mailer.rb

@@ -18,9 +18,8 @@ class Eco::API::Common::Loaders::Workflow::Mailer < Eco::API::Common::Loaders::W
       next if     session.config.dry_run?
       next unless session.config.run_mode_remote?
 
-      # temporary contingency
-      maybe_error_pages_or_tree_updates = other_case?(io) && error?
-      next unless some_update?(io) || maybe_error_pages_or_tree_updates
+      # temporary contingency: maybe_error_or_tree_updates?
+      next unless some_update?(io) || maybe_error_pages_or_tree_updates?(io)
 
       subject = base_subject
 
@@ -47,7 +46,15 @@ class Eco::API::Common::Loaders::Workflow::Mailer < Eco::API::Common::Loaders::W
     end
 
     def base_subject
-      "#{org_name} (#{active_enviro}) at #{Time.now.iso8601}"
+      "#{org_name} (#{active_enviro_desc}) at #{Time.now.iso8601}"
+    end
+
+    def active_enviro_desc
+      space_desc = config.active_enviro_space.to_s.strip
+      space_desc = ''                          if space_desc == 'default'
+      space_desc = " - space: '#{space_desc}'" unless space_desc.empty?
+
+      "#{active_enviro}#{space_desc}"
     end
 
     def some_update?(io)
@@ -66,6 +73,11 @@ class Eco::API::Common::Loaders::Workflow::Mailer < Eco::API::Common::Loaders::W
       !!error
     end
 
+    # @note at some stage, tree cases should have their own case type
+    def maybe_error_pages_or_tree_updates?(io)
+      other_case?(io) && error?
+    end
+
     def errors_n_warnings(io)
       [error_message, log_err_n_warn(io)].join("\n")
     end
@@ -73,6 +85,7 @@ class Eco::API::Common::Loaders::Workflow::Mailer < Eco::API::Common::Loaders::W
     def log_err_n_warn(io)
       warn_errors = io.logger.cache.logs(level: %i[error warn])
       return if warn_errors.empty?
+
       "ALL WARNINGS & ERRORS:\n#{warn_errors.join}\n"
     end
 

data/lib/eco/api/common/loaders/config.rb

@@ -1,17 +1,24 @@
 class Eco::API::Common::Loaders::Config
   extend Eco::API::Common::ClassHelpers
   extend Eco::Language::Methods::DelegateMissing
+
   inheritable_class_vars :delegate_missing_to
 
   class << self
     # To create samples of configurations
     def config_block(&block)
       return @config_block unless block_given?
+
+      unless @config_block.nil?
+        msg = "Reconfiguring config_block on #{self}"
+        session.log(:warn) { msg }
+      end
+
       @config_block = block
     end
 
-    def config
-      ASSETS.config
+    def config(...)
+      ASSETS.config(...)
     end
 
     def cli
@@ -27,3 +34,4 @@ class Eco::API::Common::Loaders::Config
 end
 
 require_relative 'config/workflow'
+require_relative 'config/session'

data/lib/eco/api/common/loaders/parser.rb

@@ -21,6 +21,7 @@ module Eco
               return []    if miss.empty?
               return attrs if match.empty?
               return miss  if type == :all
+
               []
             end
           end
@@ -38,6 +39,7 @@ module Eco
                 msg << "#{self.class}, is linked to"
                 return @attribute || (raise msg)
               end
+
               name value
               @attribute = value
             end
@@ -62,6 +64,7 @@ module Eco
             def parsing_phase(phase = nil)
               @parsing_phase ||= :internal
               return @parsing_phase unless phase
+
               @parsing_phase = phase
             end
 
@@ -71,6 +74,7 @@ module Eco
             def serializing_phase(phase = nil)
               @serializing_phase ||= :person
               return @serializing_phase unless phase
+
               @serializing_phase = phase
             end
 
@@ -86,6 +90,7 @@ module Eco
             # Helper to build the `active_when` condition.
             def active_when_all(*attrs)
               @active_when_attrs = RequiredAttrs.new(attribute, :all, attrs)
+
               @active_when = proc do |source_data|
                 keys = data_keys(source_data)
                 attrs.all? {|key| keys.include?(key)}
@@ -149,6 +154,10 @@ module Eco
 
           private
 
+          def options
+            ASSETS.cli.options
+          end
+
           def _define_parser(attr_parser)
             if (active_when = self.class.active_when)
               attr_parser.def_parser(
@@ -166,6 +175,7 @@ module Eco
 
           def _define_serializer(attr_parser)
             return unless respond_to?(:serializer, true)
+
             attr_parser.def_serializer(
               self.class.serializing_phase,
               &method(:serializer)

data/lib/eco/api/common/people/default_parsers/csv_parser.rb

@@ -1,235 +1,48 @@
 class Eco::API::Common::People::DefaultParsers::CSVParser < Eco::API::Common::Loaders::Parser
   attribute :csv
 
+  include Eco::API::Common::People::DefaultParsers::Helpers::ExpectedHeaders
+  include Eco::API::Common::People::DefaultParsers::Helpers::NullParsing
+
   def parser(data, deps)
     Eco::CSV.parse(data, headers: true, skip_blanks: true).tap do |table|
-      require_headers!(table)
-      check_headers(table) if deps[:check_headers] && check_headers?
-    end.each_with_object([]) do |row, arr_hash|
-      row_hash = row.headers.uniq.each_with_object({}) do |attr, hash|
+      require_headers!(table.headers)
+
+      next unless deps[:check_headers]
+      next unless check_headers?
+
+      check_headers!(
+        table.headers,
+        order_check: options.dig(:input, :header_check, :order)
+      )
+    end.each_with_object([]) do |item, arr_hash|
+      item_hash = item.headers.uniq.each_with_object({}) do |attr, hash|
         next if attr.to_s.strip.empty?
-        hash[attr.strip] = parse_string(row[attr])
+
+        hash[attr.strip] = parse_null(item[attr])
       end
-      arr_hash.push(row_hash)
+
+      arr_hash.push(item_hash)
     end
   end
 
   def serializer(array_hash, _deps)
     arr_rows = []
+
     unless array_hash.empty?
       header = array_hash.first.keys
+
       arr_rows = array_hash.map do |csv_row|
         CSV::Row.new(header, csv_row.values_at(*header))
       end
     end
+
     CSV::Table.new(arr_rows).to_csv
   end
 
   private
 
-  def abort(msg)
-    super(msg, raising: false)
-  end
-
-  def require_headers!(table)
-    headers = table.headers
-
-    abort("Missing headers in CSV") unless headers&.any?
-
-    empty = []
-    headers.each_with_index do |header, idx|
-      empty << idx if header.to_s.strip.empty?
-    end
-
-    abort("Empty headers in column(s): #{empty.join(', ')}") if empty.any?
-
-    true
-  end
-
   def check_headers?
     !options.dig(:input, :header_check, :skip)
   end
-
-  def options
-    ASSETS.cli.options
-  end
-
-  def parse_string(value)
-    return nil if value.to_s.empty?
-    return nil if null?(value)
-    value
-  end
-
-  def null?(value)
-    return true unless value
-
-    str = value.strip.upcase
-    ["NULL"].any? {|token| str == token}
-  end
-
-  def check_headers(table) # rubocop:disable Metrics/AbcSize
-    headers  = table.headers
-    unmatch  = []
-    unmatch  = unmatched_headers(headers) if options.dig(:input, :header_check, :order)
-    missing  = missing_headers(headers)
-    unknown  = unknown_headers(headers)
-    criteria = [unknown, missing[:direct], missing[:indirect], unmatch]
-    return if criteria.all?(&:empty?)
-
-    msg = "Detected possible HEADER ISSUES !!!\n"
-
-    # requires exact match
-    unless unmatch.empty?
-      msg << "CSV headers do NOT exactly match the expected:\n"
-      msg << "  * Expected: #{known_headers}\n"
-      expected, given = unmatch.first
-      msg << "  * First unmatch => Given: '#{given}' where expected '#{expected}'\n"
-      missed = known_headers - headers
-      unless missed.empty?
-        msg << "  * Missing headers:\n"
-        msg << "    - #{missed.join("\n    - ")}\n"
-      end
-    end
-
-    msg << "Missing or Wrong HEADER names in the CSV file:\n"
-    msg << "  * UNKNOWN (or not used?): #{unknown}\n" unless unknown.empty?
-    msg << "  * MISSING HEADER: #{missing[:direct]}\n" unless missing[:direct].empty?
-
-    unless (data = missing[:indirect]).empty?
-      msg << "  * MISSING INDIRECTLY:\n"
-      data.each do |ext, info|
-        msg <<  "    - '#{ext}' => "
-        msg << (info[:attrs] || {}).map do |status, attrs|
-          if status == :inactive
-            "makes inactive: #{attrs}"
-          elsif status == :active
-            "there could be missing info in: #{attrs}"
-          end
-        end.compact.join("; ")
-        msg << "\n"
-      end
-    end
-
-    log(:warn) { msg }
-
-    msg = "There were issues identified on the CSV header names. Aborting..."
-    abort(msg) if options.dig(:input, :header_check, :must_be_valid)
-
-    sleep(2)
-  end
-
-  def unmatched_headers(headers)
-    known_headers.zip(headers).reject do |(expected, given)|
-      expected == given
-    end
-  end
-
-  def unknown_headers(headers)
-    (headers - known_headers) - all_internal_attrs
-  end
-
-  def missing_headers(headers) # rubocop:disable Metrics/AbcSize
-    int_head = internal_present_or_active(headers)
-    external = headers.select do |e|
-      i = fields_mapper.to_internal(e)
-      int_head.include?(i)
-    end
-
-    ext_present = known_headers_present(int_head) | external
-    ext_miss    = known_headers - ext_present
-
-    {
-      direct:   [],
-      indirect: {}
-    }.tap do |missing|
-      ext_miss.each do |ext|
-        next unless (int = fields_mapper.to_internal(ext))
-
-        missing[:direct] << ext if all_internal_attrs.include?(int)
-        related_attrs_requirements = required_attrs.values.select do |req|
-          dep       = req.dependant?(int)
-          affects   = dep && !int_head.include?(int)
-          in_header = int_head.include?(req.attr)
-          affects || (dep && !in_header)
-        end
-
-        next if related_attrs_requirements.empty?
-
-        data = missing[:indirect][ext] = {}
-        data[:int]   = int
-        data[:attrs] = {}
-
-        related_attrs_requirements.each_with_object(data[:attrs]) do |req, attrs|
-          status = req.active?(*int_head) ? :active : :inactive
-          attrs[status] ||= []
-          attrs[status] << req.attr
-        end
-      end
-    end
-  end
-
-  def known_headers_present(headers_internal)
-    known_headers.select do |ext|
-      int = fields_mapper.to_internal(ext)
-      headers_internal.include?(int)
-    end
-  end
-
-  # Scopes what internal attrs appear in headers as they are
-  def internal_present_or_active(headers, inactive_requirements = {}) # rubocop:disable Metrics/AbcSize
-    # internal attrs that are not being mapped
-    int_all     = all_internal_attrs.reject {|i| fields_mapper.external?(i)}
-    hint        = headers & int_all
-    hext        = headers - hint
-    int_present = hint + hext.map {|e| fields_mapper.to_internal(e)}.compact
-
-    update_inactive = proc do
-      inactive_requirements.dup.each do |attr, req|
-        next unless req.active?(*int_present)
-
-        inactive_requirements.delete(attr)
-        int_present << attr
-        update_inactive.call
-      end
-    end
-
-    required_attrs.each_value do |req|
-      next if int_present.include?(req)
-
-      if req.active?(*int_present)
-        inactive_requirements.delete(req.attr)
-        int_present << req.attr
-        update_inactive.call
-      else
-        inactive_requirements[req.attr] = req
-      end
-    end
-
-    int_present
-  end
-
-  # The csv header names as expected
-  def known_headers
-    @known_headers ||= fields_mapper.list(:external).compact.uniq
-  end
-
-  def fields_mapper
-    session.fields_mapper
-  end
-
-  def required_attrs
-    @required_attrs ||= person_parser.required_attrs.to_h {|ra| [ra.attr, ra]}
-  end
-
-  def all_internal_attrs
-    @all_internal_attrs ||= [].tap do |int_attrs|
-      known_int_attrs  = person_parser.all_attrs(include_defined_parsers: true)
-      known_int_attrs |= fields_mapper.list(:internal).compact
-      int_attrs.concat(known_int_attrs)
-    end
-  end
-
-  def person_parser
-    session.entry_factory.person_parser
-  end
 end