eco-helpers 3.2.12 → 3.2.14

data/lib/eco/api/usecases/default/utils/merge_csv_case.rb

@@ -0,0 +1,313 @@
+# This script assumes that for the `MERGE_BY_FIELD` rows are consecutive.
+# @note you might run first the `sort-csv` case.
+# @note at the moment, it does NOT add new fields from the merge file.
+#   It only uses the headers of the original file.
+# @note you must inherit from this case and define the constants.
+#
+#      MERGE_BY_FIELD = 'target_csv_field'.freeze
+#      # those not merged are overridden
+#      JOINED_FIELDS  = [
+#        'joined_field_1',
+#        'joined_field_2',
+#        'joined_field_3',
+#      ].freeze
+#
+class Eco::API::UseCases::Default::Utils::MergeCsv < Eco::API::Custom::UseCase
+  name 'merge-csv'
+  type :other
+
+  require_relative 'cli/merge_csv_cli'
+
+  def main(*_args)
+    if simulate?
+      count = Eco::CSV.count(input_file)
+      log(:info) { "CSV '#{input_file}' has #{count} rows." }
+    else
+      generate_file
+    end
+  end
+
+  private
+
+  def generate_file # rubocop:disable Metrics/AbcSize
+    in_index = nil
+
+    CSV.open(output_filename, 'wb') do |out_csv|
+      pending = false
+      first   = true
+      m_first = true
+      row     = nil
+      idx     = nil
+
+      puts "\n"
+
+      streamed_merging.for_each do |m_row, m_idx|
+        if m_first
+          m_first = false
+          require_merge_by_field!(m_row, file: merge_file)
+        end
+
+        next unless pivotable?(m_row, m_idx, file: merge_file)
+
+        merging_row(m_row)
+        merge_done = false
+
+        loop do
+          unless pending
+            row = nil
+            streamed_input.shift do |o_row, i|
+              idx = i
+              row = o_row
+
+              if first
+                first = false
+                headers!(row)
+                out_csv << headers
+                require_merge_by_field!(row, file: input_file)
+              end
+            end
+          end
+
+          break unless row
+
+          in_index = idx
+          next unless pivotable?(row, idx, file: input_file)
+
+          row_count!
+          added = original_row(row) do |merged_row, merged:|
+            out_csv << merged_row.values_at(*headers)
+            merge_done = true if merged
+          end
+
+          pending = !added
+
+          break if merge_done
+          break unless added
+          break if streamed_input.eof?
+        end
+
+        row = nil unless pending
+
+        if pending || streamed_input.eof?
+          msg  = "Could not merge row #{m_idx} (#{merging_row[merge_by_field]}) "
+          msg << "because the pivot value does not exist in the original file"
+          msg << ". Skipping (discarded) ..."
+          log(:warn) { msg }
+        end
+      end
+
+      # finalize
+      loop do
+        row = nil
+        streamed_input.shift do |o_row, i|
+          idx = i
+          row = o_row
+        end
+
+        break unless row
+
+        in_index = idx
+        next unless pivotable?(row, idx, file: input_file)
+
+        row_count!
+        out_csv << row.values_at(*headers)
+
+        break if streamed_input.eof?
+      end
+    ensure
+      msg  = "Generated file '#{output_filename}' "
+      msg << "with #{row_count} rows (out of #{in_index + 1})."
+
+      log(:info) { msg } unless simulate?
+    end
+  end
+
+  # It tracks the current merging row
+  # @return [Nil, Hash] the last merge row when `row` doesn't belong
+  #   or `nil` otherwise
+  def merging_row(row = nil)
+    return @merging_row unless row
+
+    @merging_row = row.to_h
+  end
+
+  # It tracks the current grouped row
+  # @return [Nil, Hash] the last grouped row when `row` doesn't belong
+  #   or `nil` otherwise
+  def original_row(row)
+    pivot_value = row[merge_by_field]
+    merge_pivot = merging_row[merge_by_field]
+
+    if pivot_value > merge_pivot
+      # as both files are sorted, we can't add the original row now
+      # and we need to just return false
+      return false
+    elsif pivot_value < merge_pivot
+      yield(row.to_h, merged: false) if block_given?
+      return true
+    end
+
+    merged_row = {}
+    merged_row = {merge_by_field => pivot_value}
+
+    joined_fields.each do |field|
+      original_values = row[field].to_s.split('|').compact.uniq
+      merge_values    = merging_row[field].to_s.split('|').compact.uniq
+
+      merged_row[field] = (original_values | merge_values).join('|')
+      merged_row[field] = nil if merged_row[field].to_s.strip.empty?
+    end
+
+    headers_rest.each do |field|
+      merged_row[field] = row[field]
+      merged_row[field] = merging_row[field] if merging_row.key?(field)
+      merged_row[field] = nil if merged_row[field].to_s.strip.empty?
+    end
+
+    missed_headers = (merging_row.keys - headers)
+    if missed_headers.any? && !warned_missed_headers?
+      msg = "Missing headers in merged file: #{missed_headers.join(', ')}"
+      log(:warn) { msg }
+      @warned_missed_headers = true
+    end
+
+    merged_row = merged_row.slice(*headers)
+    yield(merged_row, merged: true) if block_given?
+
+    true
+  end
+
+  attr_reader :merge, :row_count
+  attr_reader :headers, :headers_rest
+
+
+  # Whether if we already warned about merging headers that
+  # are not in the original
+  def warned_missed_headers?
+    @warned_missed_headers ||= false
+  end
+
+  def headers!(row)
+    return if headers?
+
+    @headers       = row.to_h.keys
+    @joined_fields = @headers & joined_fields
+    @headers_rest  = @headers - @joined_fields - [merge_by_field]
+    @headers       = [merge_by_field, *@joined_fields, *@headers_rest]
+  end
+
+  def headers?
+    instance_variable_defined?(:@headers)
+  end
+
+  def row_count!
+    @row_count ||= 0
+    (@row_count  += 1).tap do |cnt|
+      if (cnt % 500).zero?
+        print "... Done #{cnt} rows            \r"
+        $stdout.flush
+      end
+    end
+  end
+
+  def pivotable?(row, idx, file:)
+    return false if row.nil?
+    return true  unless row[merge_by_field].to_s.strip.empty?
+
+    msg  = "Row #{idx} doesn't have value for pivot field '#{merge_by_field}'"
+    msg << " (file: '#{file}'). Skipping (discarded) ..."
+    log(:warn) { msg }
+    false
+  end
+
+  def streamed_input
+    @streamed_input ||= Eco::CSV::Stream.new(input_file)
+  end
+
+  def streamed_merging
+    @streamed_merging ||= Eco::CSV::Stream.new(merge_file)
+  end
+
+  def input_file
+    options.dig(:input, :file, :name)
+  end
+
+  def merge_file
+    options.dig(:input, :merge_file, :name)
+  end
+
+  def output_filename
+    return unless input_name
+
+    File.join(input_dir, "#{input_name}_merged#{input_ext}")
+  end
+
+  def input_name
+    @input_name ||= File.basename(input_basename, input_ext)
+  end
+
+  def input_ext
+    @input_ext ||= input_basename.split('.')[1..].join('.').then do |name|
+      ".#{name}"
+    end
+  end
+
+  def input_basename
+    @input_basename ||= File.basename(input_full_filename)
+  end
+
+  def input_dir
+    @input_dir = File.dirname(input_full_filename)
+  end
+
+  def input_full_filename
+    @input_full_filename ||= File.expand_path(input_file)
+  end
+
+  def require_merge_by_field!(row, file:)
+    return true if row.key?(merge_by_field)
+
+    msg = "Pivot field '#{merge_by_field}' missing in header of file '#{file}'"
+    log(:error) { msg }
+    raise msg
+  end
+
+  def merge_by_field
+    return @merge_by_field if instance_variable_defined?(:@merge_by_field)
+
+    return (@merge_by_field = opts_merge_by) if opts_merge_by
+
+    unless self.class.const_defined?(:MERGE_BY_FIELD)
+      msg = "(#{self.class}) You must define MERGE_BY_FIELD constant"
+      log(:error) { msg }
+      raise msg
+    end
+
+    @merge_by_field = self.class::MERGE_BY_FIELD
+  end
+
+  def joined_fields
+    return @joined_fields if instance_variable_defined?(:@joined_fields)
+
+    unless self.class.const_defined?(:JOINED_FIELDS)
+      msg = "(#{self.class}) You must define JOINED_FIELDS constant"
+      log(:error) { msg }
+      raise msg
+    end
+
+    @joined_fields ||= [self.class::JOINED_FIELDS].flatten.compact.tap do |flds|
+      next unless flds.empty?
+
+      log(:warn) {
+        msg  = 'There were no fields to be joined (JOINED_FIELDS). '
+        msg << 'This means all fields present in the merging file '
+        msg << ' will be overridden in the original file.'
+        msg
+      }
+    end
+  end
+
+  def opts_merge_by
+    options.dig(:input, :merge_by_field)
+  end
+end

data/lib/eco/api/usecases/default/utils/split_csv_case.rb

@@ -1,7 +1,7 @@
 class Eco::API::UseCases::Default::Utils::SplitCsv < Eco::API::Common::Loaders::UseCase
   require_relative 'cli/split_csv_cli'
 
-  MAX_ROWS = 15_000
+  MAX_ROWS = :unused
 
   name 'split-csv'
   type :other
@@ -15,6 +15,7 @@ class Eco::API::UseCases::Default::Utils::SplitCsv < Eco::API::Common::Loaders::
         input_file,
         max_rows: max_rows,
         start_at: start_at,
+        **params,
         &filter
       ).tap do |split|
         msg  = []
@@ -31,6 +32,10 @@ class Eco::API::UseCases::Default::Utils::SplitCsv < Eco::API::Common::Loaders::
 
   private
 
+  def params
+    {}
+  end
+
   def filter
     nil
   end

data/lib/eco/api/usecases/default/utils/track_files_case.rb

@@ -0,0 +1,179 @@
+# Tracks the files of a source folder into a file
+class Eco::API::UseCases::Default::Utils::TrackFiles < Eco::API::Custom::UseCase
+  name 'track-files'
+  type :other
+
+  require_relative 'cli/track_files_cli'
+
+  OUT_HEADERS = %w[
+    ref_id
+    filename
+    filesize
+    s3_path
+  ].freeze
+
+  REF_ID_PATH_POSITION = :last
+  BASE_S3_PATH         = 'uploads'.freeze
+  # S3_SUBPATH   = 'org-name'.freeze
+
+  def main(*_args)
+    if simulate?
+      count_files
+    else
+      generate_file
+    end
+  end
+
+  private
+
+  attr_reader :folder_count, :file_count
+
+  def folder_count!(cnt = 1)
+    @folder_count ||= 0
+
+    print '.'
+    @folder_count += cnt
+  end
+
+  def file_count!(cnt = 1)
+    @file_count ||= 0
+    @file_count  += cnt
+  end
+
+  def count_files
+    with_each_file
+
+    log(:info) {
+      "Found #{file_count} files, in #{folder_count} folders (with files)."
+    }
+  end
+
+  def ref_id_path_position
+    self.class::REF_ID_PATH_POSITION
+  end
+
+  def generate_file
+    CSV.open(output_filename, 'wb') do |csv|
+      csv << self.class::OUT_HEADERS
+
+      with_each_file do |file, src_path|
+        ref_id =
+          case ref_id_path_position
+          when :first then src_path.first
+          when :last  then src_path.last
+          else
+            raise ArgumentError, "Unknown REF_ID_PATH_POSITION: #{ref_id_path_position}  "
+          end
+
+        file_row  = [ref_id]
+        file_row << file_name = File.basename(file)
+        file_row << File.size(file)
+        file_row << s3_path(file_name, src_path)
+
+        csv      << file_row
+      end
+    end
+  ensure
+    msg  = "Generated file '#{output_filename}' "
+    msg << "with #{file_count} files/rows "
+    msg << "organized in #{folder_count} folders."
+
+    log(:info) { msg } unless simulate?
+  end
+
+  def with_each_file(folders = top_subfolders, src_path: [], &block)
+    folders.each do |folder|
+      folder_name = File.basename(folder)
+      path        = src_path[0..-1]
+      path       << folder_name
+
+      files       = folder_files(folder)
+      subfolders  = top_subfolders(folder)
+
+      next if files.empty? && subfolders.empty? # skip
+
+      if files.any? && subfolders.any?
+        msg  = "Folder '#{folder}' contains both files and subfolders."
+        msg << "\nFor correctly tracking and handling file attachments, "
+        msg << "this is not supported."
+
+        raise ArgumentError, msg
+      end
+
+      unless files.empty?
+        folder_count!
+        file_count!(files.count)
+
+        files.each do |file|
+          yield(file, path) if block_given?
+        end
+      end
+
+      next if subfolders.empty?
+
+      with_each_file(
+        subfolders,
+        src_path: path,
+        &block
+      )
+    end
+  end
+
+  def s3_path(filename, path)
+    [
+      self.class::BASE_S3_PATH,
+      s3_subpath,
+      *path,
+      filename
+    ].compact.join('/')
+  end
+
+  def s3_subpath
+    options.dig(:output, :s3_path) ||
+      s3_subpath_const             ||
+      config.active_enviro
+  end
+
+  def s3_subpath_const
+    self.class::S3_SUBPATH if self.class.const_defined?(:S3_SUBPATH)
+  end
+
+  def top_subfolders(base_folder = input_base_folder)
+    Dir[
+      File.join(base_folder, "*")
+    ].select do |f|
+      File.directory?(f)
+    end
+  end
+
+  def folder_files(dir)
+    Dir[
+      File.join(dir, "*")
+    ].select do |f|
+      File.file?(f)
+    end
+  end
+
+  def output_filename
+    return unless input_folder_name
+
+    File.join(
+      config.active_enviro,
+      'sftp',
+      "#{input_folder_name}_files.csv"
+    )
+  end
+
+  def input_folder_name
+    @input_folder_name ||= File.basename(input_base_folder)
+  end
+
+  def input_base_folder
+    options.dig(:input, :folder).tap do |folder|
+      next if File.directory?(folder)
+
+      msg = "Expecting '#{folder}' to be a directory, but it isn't."
+      raise ArgumentError, msg
+    end
+  end
+end

data/lib/eco/api/usecases/default/utils.rb

@@ -14,4 +14,7 @@ require_relative 'utils/split_json_case'
 require_relative 'utils/json_to_csv_case'
 require_relative 'utils/sort_csv_case'
 require_relative 'utils/group_csv_case'
+require_relative 'utils/merge_csv_case'
 require_relative 'utils/entries_to_csv_case'
+require_relative 'utils/track_files_case'
+require_relative 'utils/add_page_id_case'

data/lib/eco/api/usecases/graphql/helpers/location/command/result.rb

@@ -40,7 +40,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
       return nil unless error?
 
       msg  = []
-      msg << "(#{command} '#{node_id}') #{error.message}"
+      msg << "(#{command_type} '#{node_id}') #{error.message}"
 
       feed = []
       feed.concat(error.validationErrors.map(&:message)) unless error.validationErrors.empty?
@@ -55,7 +55,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     end
 
     def command_input_data
-      input[command]
+      input[command_type]
     end
 
     def command_id

data/lib/eco/api/usecases/graphql/helpers/location/command/results.rb

@@ -53,7 +53,8 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
         next applied unless with_id_change
 
         applied.select do |result|
-          next false unless (command = result.command_result_data)
+          # next false unless (command = result.command_result_data)
+          next false unless (command = result.command_input_data)
 
           command.keys.include?(:newId)
         end

data/lib/eco/api/usecases/graphql/helpers/location/tags_remap/tags_map.rb

@@ -22,9 +22,10 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
         # both are being moved (specific/long mappings first)
         return  1 if from.subset_of?(other.from)
         return -1 if from.superset_of?(other.from)
-        return -1 if (from & other.from).empty?
+        return -1 unless from.intersect?(other.from)
         return -1 if from.length >= other.from.length
         return  1 if from.length <  other.from.length
+
         -1
       end
 
@@ -49,16 +50,19 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       def maps?
         return false if any?(&:empty?)
         return false if from == to
+
         true
       end
 
       def rename?
         return false unless maps?
+
         both? {|set| set.length == 1}
       end
 
       def move?
         return false unless maps?
+
         !rename?
       end
     end

data/lib/eco/api/usecases/graphql/helpers/location/tags_remap/tags_set.rb

@@ -4,7 +4,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       class << self
         def attr_compare(*attrs)
           attrs.each do |attr|
-            meth = "#{attr}".to_sym # rubocop:disable Style/RedundantInterpolation
+            meth = :"#{attr}"
             define_method meth do |value|
               set.send(meth, to_set(value))
             end
@@ -13,7 +13,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
 
         def attr_operate(*attrs)
           attrs.each do |attr|
-            meth = "#{attr}".to_sym # rubocop:disable Style/RedundantInterpolation
+            meth = :"#{attr}"
             define_method meth do |value|
               self.class.new(set.send(meth, to_set(value)))
             end
@@ -57,6 +57,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       def include?(value)
         value = value.to_s.strip
         return false if value.empty?
+
         set.include?(value)
       end
 
@@ -82,7 +83,9 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
         return value.ini_tags.dup if value.is_a?(self.class)
         return value.dup          if value.is_a?(Array)
         return value.to_a         if value.is_a?(Set)
-        raise ArgumentError, "Expecting #{self.class}, Set or Array. Given: #{value.class}"
+
+        msg = "Expecting #{self.class}, Set or Array. Given: #{value.class}"
+        raise ArgumentError, msg
       end
 
       def to_set(value)

data/lib/eco/api/usecases/graphql/helpers/location/tags_remap.rb

@@ -22,7 +22,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
     end
 
     def to_csv(filename)
-      CSV.open(filename, "w") do |fd|
+      CSV.open(filename, 'w') do |fd|
         fd << %w[src_tags dst_tags]
 
         each do |tags_map|
@@ -67,7 +67,8 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
     end
 
     def <<(pair)
-      raise ArgumentError, "Expecting pair of Array in Array. Given: #{pair}" unless self.class.correct_pair?(pair)
+      msg = "Expecting pair of Array in Array. Given: #{pair}"
+      raise ArgumentError, msg unless self.class.correct_pair?(pair)
 
       add(*pair)
     end

data/lib/eco/api/usecases/graphql/samples/location/command/dsl.rb

@@ -76,6 +76,8 @@ class Eco::API::UseCases::GraphQL::Samples::Location
         ) do |input, stage|
           next unless input
 
+          self.id_name_input = input if simulate? && stage == :id_name
+
           some_update = true
 
           sliced_batches(
@@ -98,8 +100,8 @@ class Eco::API::UseCases::GraphQL::Samples::Location
         rearchive
       end
 
-      rescued { delete_or_publish_draft }
-      rescued { manage_remaps_table     }
+      rescued { delete_or_publish_draft            }
+      rescued { manage_remaps_table if some_update }
     end
   end
 
@@ -131,6 +133,8 @@ class Eco::API::UseCases::GraphQL::Samples::Location
 
   private
 
+  attr_accessor :id_name_input
+
   # Work with adapted diff builders.
   def nodes_diff_class
     Eco::API::UseCases::GraphQL::Helpers::Location::Command::Diffs
@@ -231,11 +235,17 @@ class Eco::API::UseCases::GraphQL::Samples::Location
   end
 
   def manage_remaps_table
-    return unless results.final_response?
-
     rescued do
-      results.applied_commands(with_id_change: true) do |result|
-        update_tags_remap_table(result.command)
+      if simulate? && id_name_input
+        id_name_input[:commands].each do |command|
+          update_tags_remap_table(command[:update])
+        end
+      elsif results.final_response?
+        results.applied_commands(with_id_change: true).each do |result|
+          update_tags_remap_table(result.command_input_data)
+        end
+      else
+        return
       end
     end
 

data/lib/eco/api/usecases/graphql/samples/location/command/service/tree_update.rb

@@ -36,8 +36,9 @@ class Eco::API::UseCases::GraphQL::Samples::Location
         # @note the SFTP push only happens if `remote_subfolder` is defined, via:
         #   1. `options.dig(:sftp, :remote_subfolder)`
         #   2. `REMOTE_FOLDER` const
-        def close_handling_tags_remap_csv
+        def close_handling_tags_remap_csv # rubocop:disable Naming/PredicateMethod
           return false unless super
+          return true  if simulate?
 
           upload(tags_remap_csv_file) unless remote_subfolder.nil?
           true