eco-helpers 2.7.12 → 2.7.14

data/Gemfile

@@ -4,3 +4,5 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in eco-helpers.gemspec
 gemspec
+
+# gem 'rubocop-rake', require: false

data/eco-helpers.gemspec

@@ -10,13 +10,13 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Oscar Segura"]
   spec.email         = ["oscar@ecoportal.co.nz"]
   spec.date          = %q{2018-09-05}
-  spec.summary       = %q{eco-helpers to manage people api cases}
+  spec.summary       = %q(eco-helpers to manage people api cases)
   spec.homepage      = "https://www.ecoportal.com"
   spec.licenses      = %w[MIT]
 
   spec.required_ruby_version = '>= 2.7.2'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
 
@@ -25,29 +25,31 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler",   ">= 2.4.12", "< 3"
-  spec.add_development_dependency "rspec",     ">= 3.12.0", "< 4"
   spec.add_development_dependency "rake",      ">= 13.0.3", "< 14"
-  spec.add_development_dependency "yard",      ">= 0.9.34", "< 1"
   spec.add_development_dependency "redcarpet", ">= 3.6.0",  "< 4"
+  spec.add_development_dependency "rspec",     ">= 3.12.0", "< 4"
+  spec.add_development_dependency "rubocop",   "~> 1"
+  spec.add_development_dependency "rubocop-rake", "~> 0"
+  spec.add_development_dependency "yard",      ">= 0.9.34", "< 1"
 
-  spec.add_dependency 'ecoportal-api',         '>= 0.9.8', '< 0.10'
-  spec.add_dependency 'ecoportal-api-v2',      '>= 1.1.8', '< 1.2'
-  spec.add_dependency 'ecoportal-api-graphql', '>= 0.3.18', '< 0.4'
-  spec.add_dependency 'ed25519', '>= 1.2'
-  spec.add_dependency 'bcrypt_pbkdf', '>= 1.0'
+  spec.add_dependency 'amatch',        '>= 0.4.1',  '< 0.5'
   spec.add_dependency 'aws-sdk-s3',    '>= 1.142.0', '< 2'
   spec.add_dependency 'aws-sdk-ses',   '>= 1.58.0', '< 2'
+  spec.add_dependency 'bcrypt_pbkdf',  '>= 1.0'
+  spec.add_dependency 'docx',          '>= 0.8.0',  '< 0.9'
   spec.add_dependency 'dotenv',        '>= 2.8.1',  '< 3'
-  spec.add_dependency 'net-ssh',       '>= 7.2.1',  '< 8'
-  spec.add_dependency 'net-sftp',      '>= 4.0.0',  '< 5'
-  spec.add_dependency 'hashdiff',      '>= 1.1.0',  '< 1.2'
+  spec.add_dependency 'ecoportal-api',         '>= 0.9.8', '< 0.10'
+  spec.add_dependency 'ecoportal-api-graphql', '>= 0.3.18', '< 0.4'
+  spec.add_dependency 'ecoportal-api-v2',      '>= 1.1.8', '< 1.2'
+  spec.add_dependency 'ed25519', '>= 1.2'
+  spec.add_dependency 'fast_excel',    '>= 0.5.0',  '< 0.6'
   spec.add_dependency 'fuzzy_match',   '>= 2.1.0',  '< 2.2'
-  spec.add_dependency 'amatch',        '>= 0.4.1',  '< 0.5'
+  spec.add_dependency 'hashdiff',      '>= 1.1.0',  '< 1.2'
   spec.add_dependency 'jaro_winkler',  '>= 1.5.6',  '< 1.6'
+  spec.add_dependency 'net-sftp',      '>= 4.0.0',  '< 5'
+  spec.add_dependency 'net-ssh',       '>= 7.2.1',  '< 8'
   spec.add_dependency 'nokogiri',      '>= 1.13',   '< 1.17'
   spec.add_dependency 'roo',           '>= 2.10.1', '< 2.11'
   spec.add_dependency 'roo-xls',       '>= 1.2.0',  '< 1.3'
-  spec.add_dependency 'fast_excel',    '>= 0.5.0',  '< 0.6'
-  spec.add_dependency 'docx',          '>= 0.8.0',  '< 0.9'
   spec.add_dependency 'rubyzip',       '>= 2.3.2',  '< 2.4'
 end

data/lib/eco/api/common/session/logger.rb

@@ -3,17 +3,20 @@ module Eco
     module Common
       module Session
         class Logger < Eco::Language::BasicLogger
-          TIMESTAMP_PATTERN = '%Y-%m-%dT%H:%M:%S'
+          TIMESTAMP_PATTERN = '%Y-%m-%dT%H:%M:%S'.freeze
 
           attr_reader :cache
 
           def initialize(file_level: ::Logger::DEBUG, log_file: nil, enviro: nil, **kargs)
             super(**kargs)
-            raise "Required Environment object (enviro:). Given: #{enviro}" if enviro && !enviro.is_a?(Eco::API::Common::Session::Environment)
+
+            msg = "Required Environment object (enviro:). Given: #{enviro.class}"
+            raise ArgumentError, msg if enviro && !enviro.is_a?(Eco::API::Common::Session::Environment)
+
             @enviro = enviro
             @cache  = Logger::Cache.new
 
-            if log_file = fetch_log_file(log_file)
+            if (log_file = fetch_log_file(log_file))
               loggers[:file] = ::Logger.new(log_file).tap do |logger|
                 logger.formatter = format_proc(console: false) do |severity, datetime, msg, formatted_msg|
                   cache.add(severity, datetime, msg, formatted_msg)
@@ -34,15 +37,16 @@ module Eco
           private
 
           def config(attr)
-            return nil unless cnf = @enviro&.config&.logger
+            return unless (cnf = @enviro&.config&.logger)
+
             cnf.send(attr) if cnf.respond_to?(attr)
           end
 
           def if_config(attr)
-            unless (value = config(attr)).nil?
-              yield(value) if block_given?
-              value
-            end
+            return if (value = config(attr)).nil?
+
+            yield(value) if block_given?
+            value
           end
 
           def fetch_log_file(log_file)

data/lib/eco/api/session/batch/jobs.rb

@@ -24,7 +24,7 @@ module Eco
           end
 
           def empty?
-            count == 0
+            count.zero?
           end
 
           def each(&block)
@@ -90,15 +90,16 @@ module Eco
           # @return [Hash<Eco::API::Session::Batch::Job, Eco::API::Session::Batch::Status>]
           def launch(simulate: false)
             each_with_index do |job, idx|
-              if job.pending?
-                status[job] = job_status = job.launch(simulate: simulate)
-                callback    = @callbacks[job]
-                callback.call(job, job_status) if callback
-                Eco::API::Session::Batch::JobsGroups.counter(delay_between_jobs) if !simulate && idx < self.length - 1
-              end
+              next unless job.pending?
+
+              status[job] = job_status = job.launch(simulate: simulate)
+              callback    = @callbacks[job]
+              callback&.call(job, job_status)
+              Eco::API::Session::Batch::JobsGroups.counter(delay_between_jobs) if !simulate && idx < length - 1
             end
             launch(simulate: simulate) if pending?
-            return status
+
+            status
           end
 
           def find_jobs(type:)
@@ -113,7 +114,7 @@ module Eco
                 yield(job, job_status)
               end
               self
-            else
+            else # rubocop:disable Naming/MemoizedInstanceVariableName
               @jobs_status ||= {}
             end
           end

data/lib/eco/api/usecases/default/people/migrate/cli/remap_tags_cli.rb

@@ -0,0 +1,23 @@
+class Eco::API::UseCases::Default::People::Migrate::RemapTags
+  class Cli < Eco::API::UseCases::Cli
+    desc 'Maps filter_tags and default_tag based on input file'
+
+    callback do |_people, _session, options, _usecase|
+      if (file = SCR.get_file('-remap-tags', required: true, should_exist: true))
+        options.deep_merge!(input: {file: {name: file}})
+      end
+    end
+
+    add_option('-remove-source-tag', 'Whether source mapped tags should be removed') do |options|
+      options.deep_merge!(usecase: {remove_source_tag: true})
+    end
+
+    add_option('-clear-unknown-tag', 'Whether unknown tags should be cleared') do |options|
+      options.deep_merge!(usecase: {clear_unknown_tags: true})
+    end
+
+    add_option('-clear-archived-nodes', 'Whether archived nodes should be cleared') do |options|
+      options.deep_merge!(usecase: {clear_archived_nodes: true})
+    end
+  end
+end

data/lib/eco/api/usecases/default/people/migrate/remap_tags_case.rb

@@ -0,0 +1,269 @@
+class Eco::API::UseCases::Default::People::Migrate::RemapTags < Eco::API::Common::Loaders::UseCase
+  name 'remap-tags'
+  type :transform
+
+  require_relative 'cli/remap_tags_cli'
+
+  REGISTER_TAGS = %w[
+    EVENT INJURY RISK CONTRACTOR PERMIT
+    AUDIT JSEA
+    TRAINING INDUCTION
+    MEETING PPE CHEMICAL
+    PLANT ASSET
+    POLICY IDEA REPORTS
+  ].freeze
+
+  def main(*_args)
+    tune_options!
+
+    people.each do |person|
+      update_job.add(person)
+
+      update_filter_tags!(person)
+      update_default_tag!(person)
+      clear_unkown_tags!(person)    if clear_unknown_tags?
+      clear_archived_nodes!(person) if clear_archived_nodes?
+    end
+  end
+
+  def update_job
+    @update_job ||= session.new_job('main', 'update', :update, usecase) do |_job, _job_status|
+      report_removed_src_tags
+      report_archived_cleared
+      report_unknown_cleared
+    end
+  end
+
+  private
+
+  # If the map on `value` exists, it returns the new value,
+  # otherwise mirrors the `value` (to prevent data loss)
+  def apply_map(value)
+    case value
+    when Array
+      value.map {|val| apply_map(val)}
+    when String
+      if (new_tag = id_maps[value])
+        new_tag
+      else
+        value
+      end
+    end
+  end
+
+  def update_filter_tags!(person)
+    mapped_tags = apply_map(person.filter_tags)
+
+    if remove_source_tag?
+      removed = person.filter_tags - mapped_tags
+      removed_src_tag!(*removed)
+      person.filter_tags = mapped_tags
+    else
+      person.filter_tags |= mapped_tags
+    end
+  end
+
+  # We always map the default_tag
+  def update_default_tag!(person)
+    return unless (account = person.account)
+
+    default_tag = apply_map(account.default_tag)
+    return unless default_tag.nil? || active_node?(default_tag)
+
+    account.default_tag = default_tag
+  end
+
+  # @note it still keeps archived nodes
+  def clear_unkown_tags!(person)
+    person.filter_tags = person.filter_tags.select do |tag|
+      known_tag?(tag).tap do |known|
+        cleared_unknown_tag!(tag) unless known
+      end
+    end
+  end
+
+  # @note it only targets archived nodes
+  def clear_archived_nodes!(person)
+    person.filter_tags = person.filter_tags.reject do |tag|
+      archived_node?(tag).tap do |known|
+        cleared_archived_node!(tag) unless known
+      end
+    end
+  end
+
+  def report_removed_src_tags
+    return if removed_src_tags.empty?
+
+    log(:info) {
+      msg  = "Here list of the #{removed_src_tags.count} removed source-mapped tags..."
+      msg << "\n  • "
+      msg << removed_src_tags.sort.join("\n  • ")
+      msg
+    }
+  end
+
+  # Clean OLD Tags
+  def report_unknown_cleared
+    report_cleared(cleared_unknown_tags, msg: 'Cleared up the following non existing tags:')
+  end
+
+  def report_archived_cleared
+    report_cleared(cleared_archived_nodes, msg: 'Cleared up the following archived nodes:')
+  end
+
+  # Whether it should remove the original tag
+  # @note we might want to preserve it at the moment, and remove them
+  #   at a later stage (i.e. after migrating the pages)
+  def remove_source_tag?
+    options.dig(:usecase, :remove_source_tag) || false
+  end
+
+  # Whether it should clear out any unkonwn tag
+  def clear_unknown_tags?
+    options.dig(:usecase, :clear_unknown_tags) || false
+  end
+
+  # Whether it should clear out any archived node
+  def clear_archived_nodes?
+    options.dig(:usecase, :clear_archived_nodes) || false
+  end
+
+  def report_cleared(data, msg:)
+    return if data.empty?
+
+    cleared_sorted = data.sort_by {|tag, count| [count * -1, tag]}
+
+    msg << "\n  • "
+    msg += cleared_sorted.map do |tag, count|
+      "'#{tag}' (#{count})"
+    end.join("\n  • ")
+
+    log(:info) { msg }
+    data.clear
+  end
+
+  def removed_src_tag!(*tags)
+    @removed_src_tags = removed_src_tags | tags
+  end
+
+  def removed_src_tags
+    @removed_src_tags ||= []
+  end
+
+  def cleared_unknown_tag!(tag)
+    cleared_unknown_tags[tag] ||= 0
+    cleared_unknown_tags[tag] += 1
+  end
+
+  def cleared_unknown_tags
+    @cleared_unknown_tags ||= {}
+  end
+
+  def cleared_archived_node!(tag)
+    cleared_archived_nodes[tag] ||= 0
+    cleared_archived_nodes[tag] += 1
+  end
+
+  def cleared_archived_nodes
+    @cleared_archived_nodes ||= {}
+  end
+
+  # @note validations on src -> dst location ids isn't done because
+  #   we might be in a case where we are moving from old locations to
+  #   location ids, but it might be a case where we are mapping an actual
+  #   location id to a custom tag as well... so trying to keep this standard.
+  def id_maps
+    @id_maps ||= input_csv.each.with_object({}) do |row, out|
+      from, to = row.fields.to_a.first(2)
+      froms    = from.split('|').map(&:strip).map(&:upcase).reject(&:empty?)
+      tos      = to.split('|').map(&:strip).map(&:upcase).reject(&:empty?)
+      from     = froms.first
+      to       = tos.first
+      next if from == to
+      next unless from && to
+
+      out[from] = to
+    end
+  end
+
+  def input_csv
+    @input_csv ||= Eco::CSV.read(input_file, encoding: input_encoding)
+  end
+
+  def input_encoding
+    options.dig(:input, :file, :encoding) || 'utf-8'
+  end
+
+  def input_file
+    @input_file ||= options.dig(:input, :file, :name).tap do |file|
+      if file.nil?
+        log(:warn) { "No input file specified" }
+        exit(1)
+      elsif File.exist?(file)
+        log(:info) { "Using input file '#{file}'" }
+      else
+        log(:error) { "File not found '#{file}'" }
+        exit(1)
+      end
+    end
+  end
+
+  def default_tag(person)
+    return unless (account = person.account)
+    account.default_tag
+  end
+
+  def known_tag?(value)
+    register_tag?(value) || tag?(value)
+  end
+
+  def unknown_tag?(value)
+    !known_tag?(value)
+  end
+
+  def register_tag?(value)
+    register_tags.include?(value)
+  end
+
+  def archived_tag?(value)
+    return false if register_tag?(value)
+    return false unless (node = tagtree.node(value))
+    node.archived
+  end
+
+  def tag?(value)
+    return false if value.nil?
+    return true  if tagtree.tag?(value)
+    register_tags.any? { |reg| value.upcase == reg }
+  end
+
+  def active_node?(value)
+    return false if value.nil?
+    return false unless tagtree.tag?(value)
+    !archived_node?(value)
+  end
+
+  def archived_node?(value)
+    return false unless (node = tagtree.node(value))
+    node.archived
+  end
+
+  # Get all the location structures merged into a single one
+  def tagtree
+    @tagtree ||= session.tagtree(
+      live:             true,
+      include_archived: true,
+      merge:            true
+    )
+  end
+
+  def register_tags
+    @register_tags ||= self.class::REGISTER_TAGS.compact.map(&:upcase)
+  end
+
+  def tune_options!
+    # options.deep_merge!(include: {excluded:     true})
+    options.deep_merge!(skip:    {api_policies: true})
+    options.deep_merge!(skip:    {batch_policy: true})
+  end
+end

data/lib/eco/api/usecases/default/people/migrate.rb

@@ -0,0 +1,6 @@
+module Eco::API::UseCases::Default::People
+  module Migrate
+  end
+end
+
+require_relative 'migrate/remap_tags_case'

data/lib/eco/api/usecases/default/people.rb

@@ -12,4 +12,4 @@ end
 require_relative 'people/treat'
 require_relative 'people/utils'
 require_relative 'people/amend'
-
+require_relative 'people/migrate'

data/lib/eco/api/usecases/default/utils/cli/split_csv_cli.rb

@@ -0,0 +1,15 @@
+class Eco::API::UseCases::Default::People::Utils::SplitCsv
+  class Cli < Eco::API::UseCases::Cli
+    desc "Splits an input file into multiple ones"
+
+    callback do |_sess, options, _case|
+      file = SCR.get_file(cli_name, required: true, should_exist: true)
+      options.deep_merge!(source: {file: file})
+    end
+
+    add_option("-max-rows", "The max count of rows of the output files") do |options|
+      count = SCR.get_arg("-max-rows", with_param: true)
+      options.deep_merge!(output: {file: {max_rows: count}})
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils/split_csv_case.rb

@@ -0,0 +1,34 @@
+class Eco::API::UseCases::Default::People::Utils::SplitCsv < Eco::API::Common::Loaders::UseCase
+  require_relative 'cli/split_csv_cli'
+
+  MAX_ROWS = 15_000
+
+  name "split-csv"
+  type :other
+
+  def main(*_args)
+    Eco::CSV.split(input_file, max_rows: max_rows).each do |file|
+      log(:info) {
+        "Generated file '#{file}'"
+      }
+    end
+  end
+
+  private
+
+  def input_file
+    options.dig(:source, :file)
+  end
+
+  def max_rows
+    max_rows_options || self.class::MAX_ROWS
+  end
+
+  def max_rows_options
+    return nil unless (num = options.dig(:output, :file, :max_rows))
+
+    num = num.to_i
+    num = nil if num.zero?
+    num
+  end
+end

data/lib/eco/api/usecases/default/utils.rb

@@ -0,0 +1,12 @@
+module Eco
+  module API
+    class UseCases
+      class Default
+        module Utils
+        end
+      end
+    end
+  end
+end
+
+require_relative 'utils/split_csv_case'

data/lib/eco/api/usecases/default.rb

@@ -14,3 +14,4 @@ end
 
 require_relative 'default/people'
 require_relative 'default/locations'
+require_relative 'default/utils'

data/lib/eco/api/usecases/graphql/samples/location/command/service/tree_update.rb

@@ -69,8 +69,7 @@ class Eco::API::UseCases::GraphQL::Samples::Location
         end
 
         # Generates the file and pushes to the SFTP folder
-        # @note this method can only work if we can run cummulative dry-runs to the back-end.
-        #   This is only possible using a draft, which is not that desired.
+        # @note it also displays the mappings on screen
         # @note the SFTP push only happens if `remote_subfolder` is defined, via:
         #   1. `options.dig(:sftp, :remote_subfolder)`
         #   2. `REMOTE_FOLDER` const

data/lib/eco/api/usecases/graphql/samples/location/command/track_changed_ids.rb

@@ -25,10 +25,10 @@ class Eco::API::UseCases::GraphQL::Samples::Location
     end
 
     # Generates the file
-    # @note this method can only work if we can run cummulative dry-runs to the back-end.
-    #   This is only possible using a draft, which is not that desired.
+    # @note this method used to only work if we could run cummulative dry-runs to the back-end.
+    #   However, after RS P3, as mappings are one-to-one (not many-to-many per row),
+    #   we can just display the mappings in dry-run as well.
     def close_handling_tags_remap_csv
-      return false if simulate?
       if tags_remap_table.any?
         puts "REMAP LOC IDs CSV (content):"
         puts tags_remap_table
@@ -50,7 +50,9 @@ class Eco::API::UseCases::GraphQL::Samples::Location
       msg = "Expecting CommandResults object. Given: #{results.class}"
       raise msg unless results.is_a?(request_results_class)
 
-      results.applied.each do |result|
+      target = simulate?? results.results : results.applied
+
+      target.each do |result|
         prev_id, new_id = result.command_input_data.values_at(:nodeId, :newId)
         next if new_id.nil? # not an id change
         next if prev_id == new_id

data/lib/eco/csv/split.rb

@@ -0,0 +1,114 @@
+module Eco
+  class CSV
+    class Split
+      include Eco::Language::AuxiliarLogger
+
+      attr_reader :filename
+
+      def initialize(filename, max_rows:, **kargs)
+        raise ArgumentError, "File '#{filename}' does not exist" unless ::File.exist?(filename)
+        @filename = filename
+        @max_rows = max_rows
+        @params   = kargs
+        init
+      end
+
+      # @yield [idx, file] a block to spot the filename
+      # @yieldparam idx [Integer] the number of the file
+      # @yieldparam file [String] the default name of the file
+      # @yieldreturn [String] the filename of the file `idx`.
+      #   - If `nil` it will create its own filename convention
+      # @return [Array<String>] names of the generated files
+      def call(&block)
+        stream.for_each do |row, ridx|
+          copy_row(row, ridx, &block)
+        end
+        out_files
+      ensure
+        puts "Close at row #{row_idx}"
+        @csv&.close
+      end
+
+      private
+
+      attr_reader :params
+      attr_reader :idx, :max_rows
+      attr_reader :headers, :row_idx
+
+      attr_accessor :exception
+
+      def copy_row(row, ridx, &block)
+        @headers ||= row.headers
+        @row_idx   = ridx
+        current_csv(ridx, &block) << row.fields
+      end
+
+      def current_csv(ridx)
+        if split?(ridx) || @csv.nil?
+          puts "Split at row #{row_idx}"
+          @csv&.close
+          out_filename = generate_name(nidx = next_idx)
+          out_filename = yield(nidx, out_filename) if block_given?
+          @csv       = ::CSV.open(out_filename, "w")
+          @csv      << headers
+          out_files << out_filename
+        end
+        @csv
+      end
+
+      def split?(ridx)
+        ((ridx + 1) % max_rows).zero?
+      end
+
+      def next_idx
+        idx.tap { @idx += 1 }
+      end
+
+      def init
+        @idx ||= 0 # rubocop:disable Naming/MemoizedInstanceVariableName
+      end
+
+      def stream
+        @stream ||= Eco::CSV::Stream.new(filename, **params)
+      end
+
+      def generate_name(fidx)
+        File.join(input_dir, "#{input_name}_#{file_number(fidx)}#{input_ext}")
+      end
+
+      def file_number(num)
+        "#{zeroed}#{num}"[-5..]
+      end
+
+      def zeroed
+        "0" * 5
+      end
+
+      def out_files
+        @out_files ||= []
+      end
+
+      def input_name
+        @input_name ||= File.basename(input_basename, input_ext)
+      end
+
+      def input_ext
+        @input_ext ||= input_basename.split('.')[1..].join('.').then do |name|
+          ".#{name}"
+        end
+      end
+
+      def input_basename
+        @input_basename ||= File.basename(input_full_filename)
+      end
+
+      def input_dir
+        @input_dir = File.dirname(input_full_filename)
+      end
+
+      def input_full_filename
+        @input_full_filename ||= File.expand_path(filename)
+      end
+    end
+  end
+end