eco-helpers 3.0.18 → 3.0.19

data/lib/eco/api/common/people/default_parsers/xls_parser.rb

@@ -2,12 +2,9 @@ class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Lo
   attribute :xls
 
   attr_accessor :already_required
-  attr_reader   :file
 
-  def parser(file, _deps)
-    @file = file
-    rows.tap do |rws|
-      @file = nil
+  def parser(filename, _deps)
+    rows(file: filename).tap do |rws|
       rws.each do |row|
         to_string!(row)
       end
@@ -22,13 +19,14 @@ class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Lo
 
   def to_string!(row)
     row.transform_values! do |val|
-      next nil unless val
+      next     unless val
       next val if val.is_a?(String)
+
       val.to_s
     end
   end
 
-  def headers
+  def expected_headers
     log(:warn) {
       "Headers detection is using your fields_map.json file (native behaviour)"
     }
@@ -39,30 +37,31 @@ class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Lo
     0
   end
 
-  def workbook
+  def workbook(file)
     require_reading_libs!
     Roo::Spreadsheet.open(file)
   end
 
-  def spreadheet(name_or_index = sheet_name)
-    workbook.sheet(name_or_index)
+  def spreadheet(name_or_index = sheet_name, file:)
+    workbook(file).sheet(name_or_index)
   end
 
-  def rows(target = headers)
-    spreadheet.parse(header_search: target, clean: true)
+  def rows(target = expected_headers, file:)
+    spreadheet(file: file).parse(header_search: target, clean: true)
   rescue Roo::HeaderRowNotFoundError => e
     missing = JSON.parse(e.message)
 
     log(:warn) {
-      "The input file is missing these headers: #{missing}"
+      "The input file is missing these expected headers: #{missing}"
     }
 
     present = target - missing
-    rows(present)
+    rows(present, file: file)
   end
 
   def require_reading_libs!
     return if already_required
+
     require 'roo'
     require 'roo-xls'
     self.already_required = true

data/lib/eco/api/common/people/default_parsers.rb

@@ -12,6 +12,7 @@ module Eco
   end
 end
 
+require_relative 'default_parsers/helpers'
 require_relative 'default_parsers/select_parser'
 require_relative 'default_parsers/boolean_parser'
 require_relative 'default_parsers/numeric_parser'
@@ -22,4 +23,5 @@ require_relative 'default_parsers/freemium_parser'
 require_relative 'default_parsers/policy_groups_parser'
 require_relative 'default_parsers/login_providers_parser'
 require_relative 'default_parsers/csv_parser'
+require_relative 'default_parsers/json_parser'
 require_relative 'default_parsers/xls_parser'

data/lib/eco/api/common/people/entry_factory.rb

@@ -28,6 +28,7 @@ module Eco
           #   to translate external names into internal ones and _vice versa_.
           def initialize(e, schema:, person_parser: nil, default_parser: nil, attr_map: nil)
             super(e)
+
             msg = "Constructor needs a PersonSchema. Given: #{schema.class}"
             fatal msg unless schema.is_a?(Ecoportal::API::V1::PersonSchema)
 
@@ -133,9 +134,10 @@ module Eco
                 out.concat(curr)
               end
             end
-            # Get content only when it's not :xls
+
+            # Get content only when it's not :xls, nor :json
             # note: even if content was provided, file takes precedence
-            if (format != :xls) && file # rubocop:disable Style/IfUnlessModifier
+            if get_content?(format) && file # rubocop:disable Style/IfUnlessModifier
               content = get_file_content(file, encoding: encoding)
             end
 
@@ -166,8 +168,10 @@ module Eco
               end
             end.tap do |out_array|
               start_from_two = (format == :csv) || format == :xls
-              out_array.each_with_index do |entry_hash, i|
-                entry_hash["idx"] = start_from_two ? i + 2 : i + 1
+              first_idx      = start_from_two ? 2 : 1
+
+              out_array.each.with_index(first_idx) do |entry_hash, idx|
+                entry_hash["idx"]         = idx
                 entry_hash["source_file"] = file
               end
             end
@@ -222,6 +226,13 @@ module Eco
 
           private
 
+          def get_content?(format)
+            return false if format == :xls
+            return false if format == :json
+
+            true
+          end
+
           def abort(message)
             log(:error) { message }
             exit(1)

data/lib/eco/api/session/batch/launcher/status_handling.rb

@@ -6,14 +6,16 @@ module Eco
           module StatusHandling
             private
 
-            def tap_status(enviro:, queue:, method:, status: nil, &block)
+            def tap_status(enviro:, queue:, method:, status: nil)
               status ||= Eco::API::Session::Batch::Status.new(
                 enviro,
                 queue:  queue,
                 method: method
               )
 
-              status.tap(&block)
+              status.tap do
+                yield(status) if block_given?
+              end
             end
           end
         end

data/lib/eco/api/session/batch/launcher.rb

@@ -71,7 +71,7 @@ module Eco
             tap_status(status: status, enviro: enviro, queue: data, method: method) do |overall_status|
               pending_for_server_error = data.to_a[0..]
 
-              batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |job_mode, per_page|
+              batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |as_job_mode, per_page|
                 iteration  = 0
                 done       = 0
                 iterations = (data.length.to_f / per_page).ceil
@@ -79,7 +79,7 @@ module Eco
                 start_time = Time.now
 
                 data.each_slice(per_page) do |slice|
-                  iteration  += 1
+                  iteration += 1
 
                   msg  = "starting batch '#{method}' iteration #{iteration}/#{iterations}, "
                   msg << "with #{slice.length} entries of #{data.length} -- #{done} done"
@@ -89,7 +89,7 @@ module Eco
                   start_slice = Time.now
 
                   offer_retry_on(*RETRY_ON, retries_left: TIMEOUT_RETRIES) do
-                    people_api.batch(job_mode: job_mode) do |batch|
+                    people_api.batch(job_mode: as_job_mode) do |batch|
                       slice.each do |person|
                         batch.public_send(method, person) do |response|
                           faltal("Request with no response") unless response

data/lib/eco/api/session.rb

@@ -132,7 +132,9 @@ module Eco
       #  If `schema` is `nil` or not provided it uses the currently associated to the `session`
       def entry_factory(schema: nil)
         schema = to_schema(schema) || self.schema
+
         return @entry_factories[schema&.id] if @entry_factories.key?(schema&.id)
+
         unless @entry_factories.empty?
           @entry_factories[schema&.id] = @entry_factories.values.first.newFactory(schema: schema)
           return @entry_factories[schema&.id]

data/lib/eco/api/usecases/default/utils/cli/group_csv_cli.rb

@@ -0,0 +1,26 @@
+class Eco::API::UseCases::Default::Utils::GroupCsv
+  class Cli < Eco::API::UseCases::Cli
+    str_desc  = 'Groups the csv rows by a pivot field. '
+    str_desc << 'It assumes the sorting field is sorted '
+    str_desc << '(same values should be consecutive)'
+
+    desc str_desc
+
+    callback do |_session, options, _usecase|
+      if (file = SCR.get_file(cli_name, required: true, should_exist: true))
+        options.deep_merge!(input: {file: {name: file}})
+      end
+    end
+
+    add_option("-start-at", "Get only the last N-start_at rows") do |options|
+      count = SCR.get_arg("-start-at", with_param: true)
+      options.deep_merge!(input: {file: {start_at: count}})
+    end
+
+    add_option('-by', 'The column that should be used to group') do |options|
+      if (file = SCR.get_arg("-by", with_param: true))
+        options.deep_merge!(input: {group_by_field: file})
+      end
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/cli/json_to_csv_cli.rb

@@ -0,0 +1,10 @@
+class Eco::API::UseCases::Default::Utils::JsonToCsv
+  class Cli < Eco::API::UseCases::Cli
+    desc "Transforms an input JSON file into a CSV one."
+
+    callback do |_sess, options, _case|
+      file = SCR.get_file(cli_name, required: true, should_exist: true)
+      options.deep_merge!(source: {file: file})
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/cli/sort_csv_cli.rb

@@ -0,0 +1,17 @@
+class Eco::API::UseCases::Default::Utils::SortCsv
+  class Cli < Eco::API::UseCases::Cli
+    desc 'Sorts the CSV by column -by'
+
+    callback do |_session, options, _usecase|
+      if (file = SCR.get_file(cli_name, required: true, should_exist: true))
+        options.deep_merge!(input: {file: file})
+      end
+    end
+
+    add_option('-by', 'The column that should be used to sorting') do |options|
+      if (file = SCR.get_arg("-by", with_param: true))
+        options.deep_merge!(input: {sort_by: file})
+      end
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/cli/split_json_cli.rb

@@ -0,0 +1,15 @@
+class Eco::API::UseCases::Default::Utils::SplitJson
+  class Cli < Eco::API::UseCases::Cli
+    desc 'Splits a json input file into multiple files'
+
+    callback do |_sess, options, _case|
+      file = SCR.get_file(cli_name, required: true, should_exist: true)
+      options.deep_merge!(source: {file: file})
+    end
+
+    add_option("-max-items", "The max count of items of the output files") do |options|
+      count = SCR.get_arg("-max-items", with_param: true)
+      options.deep_merge!(output: {file: {max_items: count}})
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/group_csv_case.rb

@@ -0,0 +1,213 @@
+# This script assumes that for the `GROUP_BY_FIELD` rows are consecutive.
+# @note you might run first the `sort-csv` case.
+# @note you must inherit from this case and define the constants.
+#
+#      GROUP_BY_FIELD = 'target_csv_field'.freeze
+#      GROUPED_FIELDS = [
+#        'joined_field_1',
+#        'joined_field_2',
+#        'joined_field_3',
+#      ].freeze
+#
+class Eco::API::UseCases::Default::Utils::GroupCsv < Eco::API::Custom::UseCase
+  name 'group-csv'
+  type :other
+
+  require_relative 'cli/group_csv_cli'
+
+  def main(*_args)
+    if simulate?
+      count = Eco::CSV.count(input_file)
+      log(:info) { "CSV '#{input_file}' has #{count} rows." }
+    else
+      generate_file
+    end
+  end
+
+  private
+
+  def generate_file # rubocop:disable Metrics/AbcSize
+    row_count = 0
+    in_index = nil
+
+    CSV.open(output_filename, 'wb') do |out_csv|
+      first = true
+
+      puts "\n"
+
+      streamed_input.for_each(start_at_idx: start_at) do |row, idx|
+        if first
+          first = false
+          headers!(row)
+          out_csv << headers
+          require_group_by_field!(row, file: input_file)
+        end
+
+        in_index = idx
+        next unless !block_given? || yield(row, idx)
+
+        next unless pivotable?(row, idx)
+        next unless (last_group = pivot_row(row))
+
+        row_count += 1
+
+        if (row_count % 500).zero?
+          print "... Done #{row_count} rows          \r"
+          $stdout.flush
+        end
+
+        out_csv << last_group.values_at(*headers)
+      end
+
+      # finalize
+      if (lrow = pivot_row)
+        row_count += 1
+        out_csv   << lrow.values_at(*headers)
+      end
+    ensure
+      msg  = "Generated file '#{output_filename}' "
+      msg << "with #{row_count} rows (out of #{in_index})."
+
+      log(:info) { msg } unless simulate?
+    end
+  end
+
+  # It tracks the current grouped row
+  # @return [Nil, Hash] the last grouped row when `row` doesn't belong
+  #   or `nil` otherwise
+  def pivot_row(row = nil)
+    @group ||= {}
+    return @group unless row
+
+    pivot_value = row[group_by_field]
+
+    unless (last_pivot = @group[group_by_field])
+      last_pivot = @group[group_by_field] = pivot_value
+    end
+
+    last   = @group
+    @group = {group_by_field => pivot_value} unless pivot_value == last_pivot
+
+    headers_rest.each do |field|
+      curr_values   = row[field].to_s.split('|').compact.uniq
+      pivot_values  = @group[field].to_s.split('|').compact.uniq
+      @group[field] = (pivot_values | curr_values).join('|')
+    end
+
+    last unless last == @group
+  end
+
+  attr_reader :group
+  attr_reader :headers, :headers_rest
+
+  def headers!(row)
+    return if headers?
+
+    @headers_rest  = grouped_fields & row.headers
+    @headers_rest -= [group_by_field]
+    @headers       = [group_by_field, *headers_rest]
+  end
+
+  def headers?
+    instance_variable_defined?(:@headers)
+  end
+
+  def pivotable?(row, idx)
+    return true unless row[group_by_field].to_s.strip.empty?
+
+    msg  = "Row #{idx} doesn't have value for pivot field '#{group_by_field}'"
+    msg << ". Skipping (discared) ..."
+    log(:warn) { msg }
+    false
+  end
+
+  def streamed_input
+    @streamed_input ||= Eco::CSV::Stream.new(input_file)
+  end
+
+  def input_file
+    options.dig(:input, :file, :name)
+  end
+
+  def start_at
+    return nil unless (num = options.dig(:input, :file, :start_at))
+
+    num = num.to_i
+    num = nil if num.zero?
+    num
+  end
+
+  def output_filename
+    return nil unless input_name
+
+    File.join(input_dir, "#{input_name}_grouped#{input_ext}")
+  end
+
+  def input_name
+    @input_name ||= File.basename(input_basename, input_ext)
+  end
+
+  def input_ext
+    @input_ext ||= input_basename.split('.')[1..].join('.').then do |name|
+      ".#{name}"
+    end
+  end
+
+  def input_basename
+    @input_basename ||= File.basename(input_full_filename)
+  end
+
+  def input_dir
+    @input_dir = File.dirname(input_full_filename)
+  end
+
+  def input_full_filename
+    @input_full_filename ||= File.expand_path(input_file)
+  end
+
+  def require_group_by_field!(row, file:)
+    return true if row.key?(group_by_field)
+
+    msg = "Pivot field '#{group_by_field}' missing in header of file '#{file}'"
+    log(:error) { msg }
+    raise msg
+  end
+
+  def group_by_field
+    return @group_by_field if instance_variable_defined?(:@group_by_field)
+
+    return (@group_by_field = opts_group_by) if opts_group_by
+
+    unless self.class.const_defined?(:GROUP_BY_FIELD)
+      msg = "(#{self.class}) You must define GROUP_BY_FIELD constant"
+      log(:error) { msg }
+      raise msg
+    end
+
+    @group_by_field = self.class::GROUP_BY_FIELD
+  end
+
+  def grouped_fields
+    return @grouped_fields if instance_variable_defined?(:@grouped_fields)
+
+    unless self.class.const_defined?(:GROUPED_FIELDS)
+      msg = "(#{self.class}) You must define GROUPED_FIELDS constant"
+      log(:error) { msg }
+      raise msg
+    end
+
+    @grouped_fields ||= [self.class::GROUPED_FIELDS].flatten.compact.tap do |flds|
+      next unless flds.empty?
+
+      log(:warn) {
+        msg  = "There were no fields to be grouped/joined. "
+        msg << "This is equivalent to launch a unique operation."
+        msg
+      }
+    end
+  end
+
+  def opts_group_by
+    options.dig(:input, :group_by_field)
+  end
+end

data/lib/eco/api/usecases/default/utils/json_to_csv_case.rb

@@ -0,0 +1,71 @@
+class Eco::API::UseCases::Default::Utils::JsonToCsv < Eco::API::Common::Loaders::UseCase
+  require_relative 'cli/json_to_csv_cli'
+
+  name 'json-to-csv'
+  type :other
+
+  def main(*_args)
+    return if simulate?
+
+    CSV.open(out_filename, 'w') do |csv|
+      csv << all_keys
+      data.each do |item|
+        csv << item.values_at(*all_keys)
+      end
+    ensure
+      log(:info) {
+        "Generated output file: '#{File.expand_path(out_filename)}'."
+      }
+    end
+  end
+
+  private
+
+  def all_keys
+    @all_keys ||= data.each_with_object([]) do |item, head|
+      head.concat(item.keys - head)
+    end
+  end
+
+  def data
+    @data ||= parse_json_file.tap do |dt|
+      ensure_array!(dt)
+
+      log(:info) {
+        "Loaded #{dt.count} items (from file '#{File.basename(input_file)}')"
+      }
+
+      exit 0 if simulate?
+    end
+  end
+
+  def out_filename
+    @out_filename ||= ''.then do
+      input_basename = File.basename(input_file)
+      base_name      = File.basename(input_basename, '.json')
+      "#{base_name}.csv"
+    end
+  end
+
+  def input_file
+    options.dig(:source, :file)
+  end
+
+  def ensure_array!(data)
+    return if data.is_a?(Array)
+
+    msg = "Expecting JSON file to contain an Array. Given: #{data.class}"
+    log(:error) { msg }
+    raise msg
+  end
+
+  def parse_json_file(filename = input_file)
+    fd = File.open(filename)
+    JSON.load fd # rubocop:disable Security/JSONLoad
+  rescue JSON::ParserError => err
+    log(:error) { "Parsing error on file '#{filename}'" }
+    raise err
+  ensure
+    fd&.close
+  end
+end

data/lib/eco/api/usecases/default/utils/sort_csv_case.rb

@@ -0,0 +1,127 @@
+class Eco::API::UseCases::Default::Utils::SortCsv < Eco::API::Custom::UseCase
+  name 'sort-csv'
+  type :other
+
+  require_relative 'cli/sort_csv_cli'
+
+  def main(*_args)
+    if simulate?
+      count = Eco::CSV.count(input_file)
+      log(:info) { "CSV '#{input_file}' has #{count} rows." }
+    else
+      group_input_rows
+      generate_file
+    end
+  end
+
+  private
+
+  attr_reader :headers, :headers_rest
+
+  def group_input_rows
+    idx   = 0
+    first = true
+
+    Eco::CSV.foreach(input_file, headers: true, skip_blanks: true) do |row|
+      idx += 1
+
+      if first
+        first           = false
+        @output_headers = row.headers
+        require_sort_field!(row, file: input_file)
+      end
+
+      pivot_value = row[sort_field]
+      (row_groups[pivot_value] ||= []) << row
+
+      if (idx % 500).zero?
+        print "... Tracked #{idx} rows          \r"
+        $stdout.flush
+      end
+    end
+  ensure
+    log(:info) { "Tracked #{idx} rows"}
+  end
+
+  def generate_file
+    idx = 0
+
+    CSV.open(output_filename, 'wb') do |csv|
+      csv << @output_headers
+
+      row_groups.keys.sort.each do |key|
+        row_groups[key].each do |row|
+          csv << row.values_at(*@output_headers)
+
+          idx += 1
+          if (idx % 500).zero?
+            print "... Sorted #{idx} rows          \r"
+            $stdout.flush
+          end
+        end
+      end
+    end
+  ensure
+    msg = "Generated file '#{output_filename}' with #{idx} rows."
+    log(:info) { msg } unless simulate?
+  end
+
+  def row_groups
+    @row_groups ||= {}
+  end
+
+  def output_filename
+    return nil unless input_name
+
+    File.join(input_dir, "#{input_name}_sorted#{input_ext}")
+  end
+
+  def input_name
+    @input_name ||= File.basename(input_basename, input_ext)
+  end
+
+  def input_ext
+    @input_ext ||= input_basename.split('.')[1..].join('.').then do |name|
+      ".#{name}"
+    end
+  end
+
+  def input_basename
+    @input_basename ||= File.basename(input_full_filename)
+  end
+
+  def input_dir
+    @input_dir = File.dirname(input_full_filename)
+  end
+
+  def input_full_filename
+    @input_full_filename ||= File.expand_path(input_file)
+  end
+
+  def input_file
+    options.dig(:input, :file)
+  end
+
+  def require_sort_field!(row, file:)
+    return true if row.key?(sort_field)
+
+    msg = "Sort field '#{sort_field}' missing in header of file '#{file}'"
+    log(:error) { msg }
+    raise msg
+  end
+
+  def sort_field
+    @sort_field ||= opts_sort_by.tap do |pivot|
+      next if pivot
+
+      msg = "The pivot field should be specified with -by option"
+
+      log(:error) { msg }
+      raise msg
+    end
+  end
+
+  def opts_sort_by
+    options.dig(:input, :sort_by)
+  end
+end