eco-helpers 3.0.18 → 3.0.20

data/lib/eco/api/session/batch/launcher.rb

@@ -71,7 +71,7 @@ module Eco
             tap_status(status: status, enviro: enviro, queue: data, method: method) do |overall_status|
               pending_for_server_error = data.to_a[0..]
 
-              batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |job_mode, per_page|
+              batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |as_job_mode, per_page|
                 iteration  = 0
                 done       = 0
                 iterations = (data.length.to_f / per_page).ceil
@@ -79,7 +79,7 @@ module Eco
                 start_time = Time.now
 
                 data.each_slice(per_page) do |slice|
-                  iteration  += 1
+                  iteration += 1
 
                   msg  = "starting batch '#{method}' iteration #{iteration}/#{iterations}, "
                   msg << "with #{slice.length} entries of #{data.length} -- #{done} done"
@@ -89,7 +89,7 @@ module Eco
                   start_slice = Time.now
 
                   offer_retry_on(*RETRY_ON, retries_left: TIMEOUT_RETRIES) do
-                    people_api.batch(job_mode: job_mode) do |batch|
+                    people_api.batch(job_mode: as_job_mode) do |batch|
                       slice.each do |person|
                         batch.public_send(method, person) do |response|
                           faltal("Request with no response") unless response

data/lib/eco/api/session/config/api.rb

@@ -135,6 +135,7 @@ module Eco
             self.class.description(self)
           end
 
+          # @todo: deletage to `apis.one_off?`
           def one_off?
             name.is_a?(Symbol)
           end

data/lib/eco/api/session/config/apis/one_off.rb

@@ -4,14 +4,14 @@ module Eco
       class Config
         class Apis
           module OneOff
-            private
-
             def one_off?
-              @is_one_off ||=
+              @is_one_off ||= # rubocop:disable Naming/MemoizedInstanceVariableName
                 SCR.get_arg('-api-key') ||
                 SCR.get_arg('-one-off')
             end
 
+            private
+
             def one_off_key
               return @one_off_key if instance_variable_defined?(:@one_off_key)
 
@@ -48,10 +48,10 @@ module Eco
               return @one_off_org if instance_variable_defined?(:@one_off_org)
 
               msg = "You should specify -org NAME when using -api-key or -one-off"
-              raise msg unless org = SCR.get_arg('-org', with_param: true)
+              raise msg unless (org = SCR.get_arg('-org', with_param: true))
 
               str_org = "#{org.downcase.split(/[^a-z]+/).join('_')}_#{one_off_enviro.gsub('.', '_')}"
-              @one_off_org  ||= str_org.to_sym
+              @one_off_org ||= str_org.to_sym
             end
 
             def one_off_enviro
@@ -83,7 +83,7 @@ module Eco
 
               true
             rescue StandardError => err
-              puts "#{err}"
+              puts err.to_s
               false
             end
           end

data/lib/eco/api/session/config/workflow.rb

@@ -141,7 +141,8 @@ module Eco
           # @yieldreturn [Eco::API::UseCases::BaseIO] the `io` input/output object carried througout all the _workflow_
           # @return [Eco::API::Session::Config::Workflow] the current stage object (to ease chainig).
           def rescue(&block)
-            return @rescue unless block
+            return @rescue unless block_given?
+
             @rescue = block
             self
           end
@@ -150,7 +151,8 @@ module Eco
 
           # Called on `SystemExit` exception
           def exit_handle(&block)
-            return @exit_handle unless block
+            return @exit_handle unless block_given?
+
             @exit_handle = block
             self
           end
@@ -171,6 +173,7 @@ module Eco
           # @return [Eco::API::Session::Config::Workflow] the current stage object (to ease chainig).
           def before(key = nil, &block)
             raise ArgumentError, "A block should be given." unless block_given?
+
             if key
               stage(key).before(&block)
             else
@@ -195,6 +198,7 @@ module Eco
           # @return [Eco::API::Session::Config::Workflow] the current stage object (to ease chainig).
           def after(key = nil, &block)
             raise ArgumentError, "A block should be given." unless block_given?
+
             if key
               stage(key).after(&block)
             else
@@ -267,6 +271,7 @@ module Eco
                 io.evaluate(self, io, &c)
               end
             end
+
             io
           end
 
@@ -276,6 +281,7 @@ module Eco
                 io.evaluate(self, io, &c)
               end
             end
+
             io
           end
 
@@ -305,6 +311,7 @@ module Eco
                 io.evaluate(self, io, &@on)
               end
             end
+
             io
           ensure
             @pending = false
@@ -341,7 +348,11 @@ module Eco
 
           def stage(key)
             self.class.validate_stage(key)
-            @stages[key] ||= self.class.workflow_class(key).new(key, _parent: self, config: config)
+            @stages[key] ||= self.class.workflow_class(key).new(
+              key,
+              _parent: self,
+              config:  config
+            )
           end
 
           # helper to treat trigger the exit and rescue handlers
@@ -354,6 +365,7 @@ module Eco
             io = io_result(io: io) do
               io.evaluate(err, io, &exit_handle)
             end
+
             exit err.status
           rescue Interrupt => _int
             raise
@@ -362,6 +374,7 @@ module Eco
             io = io_result(io: io) do
               io.evaluate(err, io, &self.rescue)
             end
+
             raise
           end
         end

data/lib/eco/api/session.rb

@@ -70,6 +70,7 @@ module Eco
       )
         if live && api?(version: :graphql)
           return live_tree(include_archived: include_archived, **kargs, &block) unless merge
+
           live_trees(include_archived: include_archived, **kargs, &block).inject(&:merge)
         else
           config.tagtree(recache: recache)
@@ -118,10 +119,12 @@ module Eco
       # @return [Eco::Data::Mapper] the mappings between the internal and external attribute/property names.
       def fields_mapper
         return @fields_mapper if instance_variable_defined?(:@fields_mapper)
+
         mappings = []
         if (map_file = config.people.fields_mapper)
           mappings = map_file ? file_manager.load_json(map_file) : []
         end
+
         @fields_mapper = Eco::Data::Mapper.new(mappings)
       end
 
@@ -132,7 +135,9 @@ module Eco
       #  If `schema` is `nil` or not provided it uses the currently associated to the `session`
       def entry_factory(schema: nil)
         schema = to_schema(schema) || self.schema
+
         return @entry_factories[schema&.id] if @entry_factories.key?(schema&.id)
+
         unless @entry_factories.empty?
           @entry_factories[schema&.id] = @entry_factories.values.first.newFactory(schema: schema)
           return @entry_factories[schema&.id]
@@ -164,9 +169,9 @@ module Eco
       # @param phase [Symbol] the phase when this parser should be active.
       # @return [Object] the parsed attribute.
       def parse_attribute(attr, source, phase = :internal, deps: {})
-        unless (parsers = entry_factory.person_parser)
-          raise "There are no parsers defined"
-        end
+        msg = "There are no parsers defined"
+        raise msg unless (parsers = entry_factory.person_parser)
+
         parsers.parse(attr, source, phase, deps: deps)
       end
 
@@ -388,18 +393,19 @@ module Eco
 
       # from schema `id` or `name` to a PersonSchema object
       def to_schema(value)
-        return nil unless value
         sch = nil
+        return unless value
+
         case value
         when String
-          unless (sch = schemas.schema(value))
-            fatal "The schema with id or name '#{value}' does not exist."
-          end
+          msg = "The schema with id or name '#{value}' does not exist."
+          fatal msg unless (sch = schemas.schema(value))
         when Ecoportal::API::V1::PersonSchema
           sch = value
         else
           fatal "Required String or Ecoportal::API::V1::PersonSchema. Given: #{value}"
         end
+
         sch
       end
     end

data/lib/eco/api/usecases/default/locations/tagtree_extract_case.rb

@@ -195,6 +195,7 @@ class Eco::API::UseCases::Default::Locations::TagtreeExtract < Eco::API::UseCase
 
   def excel(filename)
     require 'fast_excel'
+
     FastExcel.open(filename, constant_memory: true).tap do |workbook|
       yield(workbook)
       workbook.close

data/lib/eco/api/usecases/default/locations/tagtree_upload_case.rb

@@ -27,6 +27,7 @@ class Eco::API::UseCases::Default::Locations::TagtreeUpload < Eco::API::UseCases
         comms << insert_command(tree, pid: pid) unless top_id?(tree.id)
         pid    = tree.id
       end
+
       tree.nodes.map do |node|
         insert_commands(node, pid: pid)
       end.flatten(1).tap do |subs|
@@ -54,6 +55,7 @@ class Eco::API::UseCases::Default::Locations::TagtreeUpload < Eco::API::UseCases
 
   def top_id?(node_id = nil)
     return top_id.is_a?(String) if node_id.nil?
+
     node_id == top_id
   end
 

data/lib/eco/api/usecases/default/utils/cli/group_csv_cli.rb

@@ -0,0 +1,26 @@
+class Eco::API::UseCases::Default::Utils::GroupCsv
+  class Cli < Eco::API::UseCases::Cli
+    str_desc  = 'Groups the csv rows by a pivot field. '
+    str_desc << 'It assumes the sorting field is sorted '
+    str_desc << '(same values should be consecutive)'
+
+    desc str_desc
+
+    callback do |_session, options, _usecase|
+      if (file = SCR.get_file(cli_name, required: true, should_exist: true))
+        options.deep_merge!(input: {file: {name: file}})
+      end
+    end
+
+    add_option("-start-at", "Get only the last N-start_at rows") do |options|
+      count = SCR.get_arg("-start-at", with_param: true)
+      options.deep_merge!(input: {file: {start_at: count}})
+    end
+
+    add_option('-by', 'The column that should be used to group') do |options|
+      if (file = SCR.get_arg("-by", with_param: true))
+        options.deep_merge!(input: {group_by_field: file})
+      end
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/cli/json_to_csv_cli.rb

@@ -0,0 +1,10 @@
+class Eco::API::UseCases::Default::Utils::JsonToCsv
+  class Cli < Eco::API::UseCases::Cli
+    desc "Transforms an input JSON file into a CSV one."
+
+    callback do |_sess, options, _case|
+      file = SCR.get_file(cli_name, required: true, should_exist: true)
+      options.deep_merge!(source: {file: file})
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/cli/sort_csv_cli.rb

@@ -0,0 +1,17 @@
+class Eco::API::UseCases::Default::Utils::SortCsv
+  class Cli < Eco::API::UseCases::Cli
+    desc 'Sorts the CSV by column -by'
+
+    callback do |_session, options, _usecase|
+      if (file = SCR.get_file(cli_name, required: true, should_exist: true))
+        options.deep_merge!(input: {file: file})
+      end
+    end
+
+    add_option('-by', 'The column that should be used to sorting') do |options|
+      if (file = SCR.get_arg("-by", with_param: true))
+        options.deep_merge!(input: {sort_by: file})
+      end
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/cli/split_json_cli.rb

@@ -0,0 +1,15 @@
+class Eco::API::UseCases::Default::Utils::SplitJson
+  class Cli < Eco::API::UseCases::Cli
+    desc 'Splits a json input file into multiple files'
+
+    callback do |_sess, options, _case|
+      file = SCR.get_file(cli_name, required: true, should_exist: true)
+      options.deep_merge!(source: {file: file})
+    end
+
+    add_option("-max-items", "The max count of items of the output files") do |options|
+      count = SCR.get_arg("-max-items", with_param: true)
+      options.deep_merge!(output: {file: {max_items: count}})
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/group_csv_case.rb

@@ -0,0 +1,213 @@
+# This script assumes that for the `GROUP_BY_FIELD` rows are consecutive.
+# @note you might run first the `sort-csv` case.
+# @note you must inherit from this case and define the constants.
+#
+#      GROUP_BY_FIELD = 'target_csv_field'.freeze
+#      GROUPED_FIELDS = [
+#        'joined_field_1',
+#        'joined_field_2',
+#        'joined_field_3',
+#      ].freeze
+#
+class Eco::API::UseCases::Default::Utils::GroupCsv < Eco::API::Custom::UseCase
+  name 'group-csv'
+  type :other
+
+  require_relative 'cli/group_csv_cli'
+
+  def main(*_args)
+    if simulate?
+      count = Eco::CSV.count(input_file)
+      log(:info) { "CSV '#{input_file}' has #{count} rows." }
+    else
+      generate_file
+    end
+  end
+
+  private
+
+  def generate_file # rubocop:disable Metrics/AbcSize
+    row_count = 0
+    in_index = nil
+
+    CSV.open(output_filename, 'wb') do |out_csv|
+      first = true
+
+      puts "\n"
+
+      streamed_input.for_each(start_at_idx: start_at) do |row, idx|
+        if first
+          first = false
+          headers!(row)
+          out_csv << headers
+          require_group_by_field!(row, file: input_file)
+        end
+
+        in_index = idx
+        next unless !block_given? || yield(row, idx)
+
+        next unless pivotable?(row, idx)
+        next unless (last_group = pivot_row(row))
+
+        row_count += 1
+
+        if (row_count % 500).zero?
+          print "... Done #{row_count} rows          \r"
+          $stdout.flush
+        end
+
+        out_csv << last_group.values_at(*headers)
+      end
+
+      # finalize
+      if (lrow = pivot_row)
+        row_count += 1
+        out_csv   << lrow.values_at(*headers)
+      end
+    ensure
+      msg  = "Generated file '#{output_filename}' "
+      msg << "with #{row_count} rows (out of #{in_index})."
+
+      log(:info) { msg } unless simulate?
+    end
+  end
+
+  # It tracks the current grouped row
+  # @return [Nil, Hash] the last grouped row when `row` doesn't belong
+  #   or `nil` otherwise
+  def pivot_row(row = nil)
+    @group ||= {}
+    return @group unless row
+
+    pivot_value = row[group_by_field]
+
+    unless (last_pivot = @group[group_by_field])
+      last_pivot = @group[group_by_field] = pivot_value
+    end
+
+    last   = @group
+    @group = {group_by_field => pivot_value} unless pivot_value == last_pivot
+
+    headers_rest.each do |field|
+      curr_values   = row[field].to_s.split('|').compact.uniq
+      pivot_values  = @group[field].to_s.split('|').compact.uniq
+      @group[field] = (pivot_values | curr_values).join('|')
+    end
+
+    last unless last == @group
+  end
+
+  attr_reader :group
+  attr_reader :headers, :headers_rest
+
+  def headers!(row)
+    return if headers?
+
+    @headers_rest  = grouped_fields & row.headers
+    @headers_rest -= [group_by_field]
+    @headers       = [group_by_field, *headers_rest]
+  end
+
+  def headers?
+    instance_variable_defined?(:@headers)
+  end
+
+  def pivotable?(row, idx)
+    return true unless row[group_by_field].to_s.strip.empty?
+
+    msg  = "Row #{idx} doesn't have value for pivot field '#{group_by_field}'"
+    msg << ". Skipping (discared) ..."
+    log(:warn) { msg }
+    false
+  end
+
+  def streamed_input
+    @streamed_input ||= Eco::CSV::Stream.new(input_file)
+  end
+
+  def input_file
+    options.dig(:input, :file, :name)
+  end
+
+  def start_at
+    return nil unless (num = options.dig(:input, :file, :start_at))
+
+    num = num.to_i
+    num = nil if num.zero?
+    num
+  end
+
+  def output_filename
+    return nil unless input_name
+
+    File.join(input_dir, "#{input_name}_grouped#{input_ext}")
+  end
+
+  def input_name
+    @input_name ||= File.basename(input_basename, input_ext)
+  end
+
+  def input_ext
+    @input_ext ||= input_basename.split('.')[1..].join('.').then do |name|
+      ".#{name}"
+    end
+  end
+
+  def input_basename
+    @input_basename ||= File.basename(input_full_filename)
+  end
+
+  def input_dir
+    @input_dir = File.dirname(input_full_filename)
+  end
+
+  def input_full_filename
+    @input_full_filename ||= File.expand_path(input_file)
+  end
+
+  def require_group_by_field!(row, file:)
+    return true if row.key?(group_by_field)
+
+    msg = "Pivot field '#{group_by_field}' missing in header of file '#{file}'"
+    log(:error) { msg }
+    raise msg
+  end
+
+  def group_by_field
+    return @group_by_field if instance_variable_defined?(:@group_by_field)
+
+    return (@group_by_field = opts_group_by) if opts_group_by
+
+    unless self.class.const_defined?(:GROUP_BY_FIELD)
+      msg = "(#{self.class}) You must define GROUP_BY_FIELD constant"
+      log(:error) { msg }
+      raise msg
+    end
+
+    @group_by_field = self.class::GROUP_BY_FIELD
+  end
+
+  def grouped_fields
+    return @grouped_fields if instance_variable_defined?(:@grouped_fields)
+
+    unless self.class.const_defined?(:GROUPED_FIELDS)
+      msg = "(#{self.class}) You must define GROUPED_FIELDS constant"
+      log(:error) { msg }
+      raise msg
+    end
+
+    @grouped_fields ||= [self.class::GROUPED_FIELDS].flatten.compact.tap do |flds|
+      next unless flds.empty?
+
+      log(:warn) {
+        msg  = "There were no fields to be grouped/joined. "
+        msg << "This is equivalent to launch a unique operation."
+        msg
+      }
+    end
+  end
+
+  def opts_group_by
+    options.dig(:input, :group_by_field)
+  end
+end

data/lib/eco/api/usecases/default/utils/json_to_csv_case.rb

@@ -0,0 +1,71 @@
+class Eco::API::UseCases::Default::Utils::JsonToCsv < Eco::API::Common::Loaders::UseCase
+  require_relative 'cli/json_to_csv_cli'
+
+  name 'json-to-csv'
+  type :other
+
+  def main(*_args)
+    return if simulate?
+
+    CSV.open(out_filename, 'w') do |csv|
+      csv << all_keys
+      data.each do |item|
+        csv << item.values_at(*all_keys)
+      end
+    ensure
+      log(:info) {
+        "Generated output file: '#{File.expand_path(out_filename)}'."
+      }
+    end
+  end
+
+  private
+
+  def all_keys
+    @all_keys ||= data.each_with_object([]) do |item, head|
+      head.concat(item.keys - head)
+    end
+  end
+
+  def data
+    @data ||= parse_json_file.tap do |dt|
+      ensure_array!(dt)
+
+      log(:info) {
+        "Loaded #{dt.count} items (from file '#{File.basename(input_file)}')"
+      }
+
+      exit 0 if simulate?
+    end
+  end
+
+  def out_filename
+    @out_filename ||= ''.then do
+      input_basename = File.basename(input_file)
+      base_name      = File.basename(input_basename, '.json')
+      "#{base_name}.csv"
+    end
+  end
+
+  def input_file
+    options.dig(:source, :file)
+  end
+
+  def ensure_array!(data)
+    return if data.is_a?(Array)
+
+    msg = "Expecting JSON file to contain an Array. Given: #{data.class}"
+    log(:error) { msg }
+    raise msg
+  end
+
+  def parse_json_file(filename = input_file)
+    fd = File.open(filename)
+    JSON.load fd # rubocop:disable Security/JSONLoad
+  rescue JSON::ParserError => err
+    log(:error) { "Parsing error on file '#{filename}'" }
+    raise err
+  ensure
+    fd&.close
+  end
+end