eco-helpers 2.6.4 → 2.7.1

data/lib/eco/api/usecases/graphql/helpers/location/command/diffs/stages/sortable.rb

@@ -0,0 +1,59 @@
+require_relative 'sortable/relation_safe_sort'
+
+class Eco::API::UseCases::GraphQL::Helpers::Location::Command::Diffs
+  module Stages
+    module Sortable
+      # from what diffs size we should use the cluster mode
+      CLUSTER_MODE_FROM = 400
+
+      class << self
+        def included(base)
+          super
+          base.extend(Stages)
+        end
+      end
+
+      private
+
+      def diffs_sort(list, cluster_mode: nil, &block)
+        RelationSafeSort.new(list, cluster_mode: cluster_mode, &block).sort
+      end
+
+      # @note As can't update archived nodes, it happens before :id update (:prev),
+      #   and therefore it should use `:prev` rather than `:curr`
+      # @note you wouldn't be able to unarchive a node that does not exist.
+      #   Therefore, it does have `:prev`
+      # @see spaceship operator `self.class#<=>`
+      def cluster_mode(stage)
+        case stage
+        when :archive, :unarchive
+          :prev
+        when :insert
+          :curr
+        else
+          return :curr if self.class.after_id_update?(stage)
+          :prev
+        end
+      end
+
+      def sort_top_to_bottom(raw_diffs, stage)
+        clustered = nil
+        clustered = cluster_mode(stage) if raw_diffs.count > CLUSTER_MODE_FROM
+
+        if self.class.during_or_after_move?(stage)
+          diffs_sort(raw_diffs, cluster_mode: clustered) do |a, b|
+            a.sort_after_move(b)
+          end
+        else
+          diffs_sort(raw_diffs, cluster_mode: clustered) do |a, b|
+            a.sort_before_move(b)
+          end
+        end
+      end
+
+      def sort_bottom_to_top(raw_diffs, stage)
+        sort_top_to_bottom(raw_diffs, stage).reverse
+      end
+    end
+  end
+end

data/lib/eco/api/usecases/graphql/helpers/location/command/diffs/stages.rb

@@ -0,0 +1,82 @@
+# @note The order in which updates should happen:
+#   (1) UNARCHIVE (top-bottom) should go FIRST, to ensure consistency througout all the stages sequence
+#     * Currently: CANNOT UPDATE (or move) ARCHIVED NODES
+#     * It can unarchive children with same archivedToken
+#       * Changing order to happen after MOVE does NOT prevent this to happen.
+#       * And archived children nodes cannot be moved somewhere else anyway either.
+#     * Within UNARCHIVE, the preferable order would be ALL nodes from leafs to root
+#       * to the purpose of preventing `already unarchived node` errors
+#       * because, from top to bottom, the `unarchivedToken` could have already unarchived children.
+#       * This still does not prevent the unintended consequence of automatic unarchive of children.
+#     * HOWEVER, the error _you can't unarchive a child of an archived node_ suggest that
+#       within UNARCHIVE, the order MUST BE from top-to-bottom, and MOREOVER
+#         * Among nodes with the same `archiveToken` only the topest one should be unarchived (the others follow),
+#           which prevents the errors `already unarchived node` and `can't unarchive child of archived node`
+#         * Children with same `archiveToken` that are not to be unarchived, should be rearchived
+#         * IMPORTANT: this could require to track all the way up to the first archived ancestor,
+#         path from leave-to archive-root, and then for unarchive from top to bottom only those
+#         with different archiveToken.
+#     (1.b) IMPORTANT: automatically unarchived nodes (archivedToken) that should remain archived
+#       should be RE-ARCHIVEd as part of the update (in the ARCHIVE stage)
+#   (2) UPDATE ID (--) needs to happen before
+#     * This allows other commands to refer to nodes and their parents with current ids.
+#     * If INSERT happened before UPDATE ID, they would need to refer to the previous value of `parentId`,
+#       but inserts don't have previous source in NodeDiff (they are new) and only a working-around (i.e. general lookup)
+#       would allow to obtain this value (not really neat)
+#     * Technically, the internal ORDER does NOT MATTER, but if something has to fail, let's make it fail
+#       big: so let's do top-to-bottom.
+#     (2.b) UPDATE NAME can happen anywhere, so let's keep tree consistent and do it as soon as possible.
+#   (3) INSERT (top-bottom) has priority over MOVE, as there may be existing nodes being moved to or inserted to.
+#     * And should happen AFTER the UNARCHIVE, because we could be inserting to an unarchived node
+#       and that would make our inserted node an archived one as well.
+#     * The internal ORDER MATTERS and it's from top to bottom (root to leaves) to prevent unknown parentId error.
+#   (4) MOVE (--) should happen before ARCHIVE and after UNARCHIVE
+#     * This prevents active nodes to be archived from their previous parent node, now being archived.
+#     * It also prevents the error of moving to an archived node, or not being able to move an archived node.
+#     * If it happened before UNARCHIVE, we would be either move ARCHIVED nodes to be unarchived,
+#       or moving other nodes to them before they are unarchived
+#     * Technically, for MOVE, the internal ORDER does NOT MATTER. It would matter for new/inserted nodes,
+#       but we do not move inserted nodes (we add them directly to the parent).
+#       * However, to build the remap tags table, it does matter: from bottom to top. Children should be
+#       retagged first because retagging first parents changes the path of their children retags.
+#   (5) ARCHIVE (bottom-top) should happen always LAST, after nodes have been MOVED.
+#     • The order of archiving should probably be:
+#       (a) although firstly thought go from leafs to root to prevent the `archiveToken`
+#         it should go from top-to-bottom, so precisely the `archiveToken` is generated and history well sorted.
+#       (b) to preventing `already archived node` errors, we should only include the parent
+#   (6) RE-ARCHIVE (bottom-top) see (1.b) all those that were automatically unarchived but should remain archived.
+#     • The neat solution is to recalculate changes with the live_tree and the final desired result,
+#       but perhaps only target those to be re-archived?
+class Eco::API::UseCases::GraphQL::Helpers::Location::Command::Diffs
+  module Stages
+    STAGES = %i[unarchive id_name insert move archive].freeze
+
+    def stages
+      STAGES
+    end
+
+    def stage_idx(stage)
+      stages.index(stage)
+    end
+
+    def id_update_stage_idx
+      @id_update_stage_idx ||= stage_idx(:id) || stage_idx(:id_name)
+    end
+
+    def move_stage_idx
+      @move_stage_idx ||= stage_idx(:move)
+    end
+
+    def after_id_update?(stage)
+      (idx = stage_idx(stage)) && idx > id_update_stage_idx
+    end
+
+    def during_or_after_move?(stage)
+      (idx = stage_idx(stage)) && idx >= move_stage_idx
+    end
+  end
+end
+
+require_relative 'stages/sortable'
+require_relative 'stages/diff_sortable'
+require_relative 'stages/commandable'

data/lib/eco/api/usecases/graphql/helpers/location/command/diffs.rb

@@ -0,0 +1,20 @@
+# The order in which updates should happen:
+# @note several challenges were found during the design of this class:
+#   This translated into dividing the command sequence into different `STAGES`.
+module Eco::API::UseCases::GraphQL::Helpers::Location
+  class Command::Diffs < Eco::Data::Locations::NodeDiff::NodesDiff
+    class_resolver :diff_result_class, Command::Diff
+
+    require_relative 'diffs/stages'
+
+    include Stages::Commandable
+
+    def commands
+      self.class.stages.each_with_object([]) do |stage, out|
+        comms = stage_commands(stage)
+        out.concat(comms)
+        yield(comms, stage) if block_given?
+      end
+    end
+  end
+end

data/lib/eco/api/usecases/graphql/helpers/location/command/optimizations.rb

@@ -0,0 +1,84 @@
+module Eco::API::UseCases::GraphQL::Helpers::Location
+  module Command::Optimizations
+    DEFAULT_COMMANDS_PER_PAGE = 45
+    DEFAULT_FORCE_CONTINUE    = false
+
+    # Available options are:
+    #   - :per_request -> on each request
+    #   - :per_batch   -> at the end of each batch / stage (on last page)
+    #   - :once        -> only when the script starts to run
+    def default_tree_tracking_mode
+      :per_request
+    end
+
+    # Helper to identify the commands payload block
+    # @note
+    #   1. Gives flexibility on at what time the structure should be retrieved
+    #   2. This increases performacne, as we don't retrieve the full tree on
+    #     each request unless necessary/specified
+    #   3. `nil` falls the block back to the `ecoportal-api-graphql` gem, which
+    #     has a default block that retrieves the structure
+    # @return [Proc, NilClass] the payload block
+    def scope_commands_block(idx, total, track_tree_mode: default_tree_tracking_mode)
+      case track_tree_mode
+      when :per_request
+        nil
+      when :once
+        commands_payload_without_structure_block
+      when :per_batch
+        return nil if idx == total
+        commands_payload_without_structure_block
+      end
+    end
+
+    # Prevents each request from timing out
+    def commands_per_page
+      if self.class.const_defined?(:COMMANDS_PER_PAGE)
+        self.class::COMMANDS_PER_PAGE
+      else
+        DEFAULT_COMMANDS_PER_PAGE
+      end
+    end
+
+    # Whether to stop or continue on command fail
+    def force_continue?
+      if self.class.const_defined?(:FORCE_CONTINUE)
+        self.class::FORCE_CONTINUE
+      else
+        DEFAULT_FORCE_CONTINUE
+      end
+    end
+
+    # Commands payload without querying Structure
+    # @note this servces the purpose of optimizing/speeding up the requests.
+    def commands_payload_without_structure_block
+      proc {
+        clientMutationId
+        error { # rubocop:disable Style/BlockDelimiters
+          message
+          conflictingIds
+          validationErrors { # rubocop:disable Style/BlockDelimiters
+            error
+            message
+          }
+        }
+        results { # rubocop:disable Style/BlockDelimiters
+          command { # rubocop:disable Style/BlockDelimiters
+            id
+            state
+            __typename
+          }
+          ok
+          error { # rubocop:disable Style/BlockDelimiters
+            conflictingIds
+            message
+            validationErrors { # rubocop:disable Style/BlockDelimiters
+              error
+              message
+            }
+          }
+        }
+      }
+    end
+  end
+end

data/lib/eco/api/usecases/graphql/helpers/location/command/result.rb

@@ -42,11 +42,11 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     def error_msg
       return nil unless error?
       msg  = ''
-      msg << "(#{command} '#{node_id}') #{error.message}\n" || ''
+      msg << "(#{command} '#{node_id}') #{error.message}\n"
       return msg if error.validationErrors.empty?
-      msg << "  • " + error.validationErrors.map do |err|
-        err.message
-      end.join("\n  • ")
+
+      msg << "  • "
+      msg << error.validationErrors.map(&:message).join("\n  • ")
       msg
     end
 

data/lib/eco/api/usecases/graphql/helpers/location/command/results.rb

@@ -8,12 +8,10 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     end
 
     def stats
-      msg  = ''
-      first_err = force? ? '' : "(stopped on node: '#{first_errored&.node_id}' - idx: #{first_errored_idx})"
-      msg << "  • Errored: #{errored.count} #{first_err}\n" if errored?
-      last_okay = force? ? '' : "(last node done: '#{last_applied&.node_id}' - idx: #{last_applied_idx})"
-      msg << "  • Applied: #{errored.count} #{last_okay}\n" if some_applied?
-      msg << "  • Pending: #{pending.count}\n"              if some_pending?
+      msg       = ''
+      msg << "  • Errored: #{errored.count} #{first_err_str}\n" if errored?
+      msg << "  • Applied: #{errored.count} #{last_okay_str}\n" if some_applied?
+      msg << "  • Pending: #{pending.count}\n"                  if some_pending?
       msg
     end
 
@@ -38,7 +36,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     end
 
     def success?
-      !error? && results.all? {|r| r.success?}
+      !error? && results.all?(&:success?)
     end
 
     def results
@@ -64,7 +62,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     end
 
     def errored
-      @errored ||= results.select {|r| r.error?}
+      @errored ||= results.select(&:error?)
     end
 
     def errored?
@@ -80,15 +78,15 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     end
 
     def applied
-      @applied ||= results.select {|r| r.applied?}
+      @applied ||= results.select(&:applied?)
     end
 
     def applied?
-      results.all? {|r| r.applied?}
+      results.all?(&:applied?)
     end
 
     def some_applied?
-      applied.count > 0
+      applied.count.positive?
     end
 
     def last_applied
@@ -100,7 +98,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     end
 
     def pending
-      @pending ||= results.select {|r| r.pending?}
+      @pending ||= results.select(&:pending?)
     end
 
     def some_pending?
@@ -122,5 +120,19 @@ module Eco::API::UseCases::GraphQL::Helpers::Location::Command
     def response_results
       response&.results || []
     end
+
+    # First error is relevant only if request was NOT forced to continue.
+    def first_err_str
+      return '' if force?
+      pre_str = 'stopped on node'
+      pre_str = 'first error' unless force?
+      "(#{pre_str}: '#{first_errored&.node_id}' - idx: #{first_errored_idx})"
+    end
+
+    # Last successfully applied is relevant only if request was NOT forced to continue.
+    def last_okay_str
+      return '' if force?
+      "(last node done: '#{last_applied&.node_id}' - idx: #{last_applied_idx})"
+    end
   end
 end

data/lib/eco/api/usecases/graphql/helpers/location/command.rb

@@ -3,25 +3,12 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
     include Eco::Language::AuxiliarLogger
     include Eco::API::UseCases::GraphQL::Helpers::Location::Base
 
-    DEFAULT_COMMANDS_PER_PAGE = 45
-    DEFAULT_FORCE_CONTINUE    = false
+    require_relative 'command/optimizations'
+    include Eco::API::UseCases::GraphQL::Helpers::Location::Command::Optimizations
 
-    # Prevents each request from timing out
-    def commands_per_page
-      if self.class.const_defined?(:COMMANDS_PER_PAGE)
-        self.class::COMMANDS_PER_PAGE
-      else
-        DEFAULT_COMMANDS_PER_PAGE
-      end
-    end
-
-    # Whether to stop or continue on command fail
-    def force_continue?
-      if self.class.const_defined?(:FORCE_CONTINUE)
-        self.class::FORCE_CONTINUE
-      else
-        DEFAULT_FORCE_CONTINUE
-      end
+    # Actual GraphQL request
+    def apply_commands(input, &block)
+      graphql.locationStructure.applyCommands(input: input, &block)
     end
 
     # With given the commands, it generates the input of the endpoint mutation.
@@ -29,14 +16,20 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
     def input(commands, force_continue: force_continue?)
       {
         clientMutationId: "",
-        id:       target_structure_id,
-        force:    force_continue,
-        commands: commands
+        id:               target_structure_id,
+        force:            force_continue,
+        commands:         commands
       }
     end
 
+    # @param track_tree_mode [Symbol] when should updates happen
     # @return see #with_sliced_input
-    def sliced_batches(batch_input, size: commands_per_page, desc: :input, logging: true)
+    def sliced_batches(
+      batch_input,
+      size:            commands_per_page,
+      desc:            :input,
+      track_tree_mode: default_tree_tracking_mode
+    )
       dry_run_msg = simulate? ? '(dry-run) ' : ''
 
       if batch_input[:commands].empty?
@@ -60,7 +53,8 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
         if simulate?
           log(:info) { sliced_input.pretty_inspect } if page < 3
         else
-          response = graphql.locationStructure.applyCommands(input: sliced_input)
+          curr_block = scope_commands_block(page, pages, track_tree_mode: track_tree_mode)
+          response   = apply_commands(sliced_input, &curr_block)
           backup(response, type: "tree_update_#{desc}_response_#{page}_of_#{pages}")
         end
 
@@ -75,7 +69,8 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       comms = input_data[:commands]
       total = comms.count
       pages = (total.to_f / size).ceil.to_i
-      page  = 1; out = []
+      page  = 1
+      out   = []
       comms.each_slice(size) do |comms_slice|
         sliced_input = input_data.slice(:clientMutationId, :id).merge(commands: comms_slice)
         yield(sliced_input, page, pages, comms_slice.count, total).tap do |response|
@@ -90,3 +85,5 @@ end
 
 require_relative 'command/result'
 require_relative 'command/results'
+require_relative 'command/diff'
+require_relative 'command/diffs'

data/lib/eco/api/usecases/graphql/helpers/location/tags_remap/tags_map.rb

@@ -13,7 +13,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       end
 
       # @note to create a stable sort we assume `self` is a sorted element
-      def <=>(other)
+      def <=>(other) # rubocop:disable Metrics/AbcSize
         return -1 if maps?   && !other.maps?
         return  1 if !maps?  && other.maps?
         return -1 if rename? && other.move?

data/lib/eco/api/usecases/graphql/helpers/location/tags_remap/tags_set.rb

@@ -4,15 +4,16 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       class << self
         def attr_compare(*attrs)
           attrs.each do |attr|
-            meth = "#{attr}".to_sym
+            meth = "#{attr}".to_sym # rubocop:disable Style/RedundantInterpolation
             define_method meth do |value|
               set.send(meth, to_set(value))
             end
           end
         end
+
         def attr_operate(*attrs)
           attrs.each do |attr|
-            meth = "#{attr}".to_sym
+            meth = "#{attr}".to_sym # rubocop:disable Style/RedundantInterpolation
             define_method meth do |value|
               self.class.new(set.send(meth, to_set(value)))
             end
@@ -21,12 +22,12 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       end
 
       COMPARE_METHODS = %i[< <= == > >=].freeze
-      OPERATE_METHIDS = %i[& + - ^].freeze
+      OPERATE_METHODS = %i[& + - ^].freeze
 
-      attr_compare *COMPARE_METHODS
-      attr_operate *OPERATE_METHIDS
+      attr_compare(*COMPARE_METHODS)
+      attr_operate(*OPERATE_METHODS)
 
-      attr_reader :tags, :set
+      attr_reader :set
 
       def initialize(tags)
         @ini_tags = value_to_a(tags)
@@ -48,7 +49,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       def to_s
         tags.join('|')
       end
-      
+
       def empty?
         set.empty?
       end
@@ -60,7 +61,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       end
 
       def include_any?(value)
-        value.to_a.any? {|tag| self.include?(tag)}
+        value.to_a.any? {|tag| include?(tag)}
       end
 
       def subset_of?(value)
@@ -73,9 +74,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
 
       protected
 
-      def ini_tags
-        @ini_tags
-      end
+      attr_reader :ini_tags
 
       private
 

data/lib/eco/api/usecases/graphql/helpers/location/tags_remap.rb

@@ -5,9 +5,14 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       correct_pair && pair.length == 2
     end
 
-    attr_reader :src_maps
+    include Enumerable
+
     attr_reader :from, :to
 
+    def src_maps
+      @src_maps ||= []
+    end
+
     def each(&block)
       return to_enum(:each) unless block
       src_maps.each(&block)
@@ -15,7 +20,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
 
     def to_csv(filename)
       CSV.open(filename, "w") do |fd|
-        fd << ["src_tags", "dst_tags"]
+        fd << %w[src_tags dst_tags]
         each do |tags_map|
           fd << tags_map.to_csv_row
         end
@@ -26,8 +31,7 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
       "".tap do |str|
         each.map do |tags_map|
           from, to = tags_map.to_csv_row
-          str << " • from: #{from}\n"
-          str << "   • to: #{to}\n"
+          str << " • '#{from}' => '#{to}'\n"
         end
       end
     end
@@ -71,16 +75,11 @@ module Eco::API::UseCases::GraphQL::Helpers::Location
 
     private
 
-    attr_reader :src_maps
     attr_reader :from_key, :to_key
 
     def new_src(tags_map)
       src_maps << tags_map
     end
-
-    def src_maps
-      @src_maps ||= []
-    end
   end
 end
 

data/lib/eco/api/usecases/graphql/samples/location/command/dsl.rb

@@ -1,6 +1,7 @@
 class Eco::API::UseCases::GraphQL::Samples::Location
   module Command::DSL
     include Eco::API::UseCases::GraphQL::Helpers::Location::Command
+    include Eco::API::UseCases::GraphQL::Samples::Location::Command::TrackChangedIds
     include Eco::API::UseCases::GraphQL::Samples::Location::Command::Results
 
     # @example of implementation:
@@ -15,40 +16,68 @@ class Eco::API::UseCases::GraphQL::Samples::Location
     #         end
     #       end
     #     end
-    def inputs(*args, force_continue: force_continue?, **kargs, &block)
+    def inputs(*_args, force_continue: force_continue?, **_kargs, &_block) # rubocop:disable Lint/UnusedMethodArgument
       msg  = "You should implement this method in your child class.\n"
       msg << "Which should yield the input Hash and the stage or descriptor."
       raise Eco::API::UseCases::GraphQL::Base::NotImplementedMethod, msg
     end
 
     # Main processor
-    def process
-      begin
-        super if defined?(super)
-      rescue Eco::API::UseCases::GraphQL::Base::NotImplementedMethod
-      end
+    def process # rubocop:disable Metrics/AbcSize
+      self.error     = false
+      self.exception = nil
+
+      super if defined?(super)
 
-      self.error = false
-      # this triggers a backup of the tagtree
+      # this may trigger a backup of the tagtree
       self.current_tree ||= live_tree
 
       inputs(force_continue: force_continue?) do |input, stage|
         results[stage] ||= []
-        sliced_batches(input, desc: stage) do |sliced_input, response, page, pages, done, total|
+
+        track_mode = batch_tree_track_mode(stage)
+
+        # yields the result of each batch
+        sliced_batches(input, desc: stage, track_tree_mode: track_mode) do |sliced_input, response, page, pages, done, total| # rubocop:disable Metrics/ParameterLists, Layout/LineLength
           track_current_tree(response&.structure)
-          page_results = nil
+
           results[stage] << (page_results = request_results_class.new(sliced_input, response))
-          update_tags_remap_table(page_results, stage)
-          break if self.error = page_errors?(page_results, page, pages, done, total, stage: stage)
+          update_tags_remap_table(page_results, stage, current_tree)
+
+          self.error = page_errors?(page_results, page, pages, done, total, stage: stage)
+          break if error
         end
 
         break if error
       end
+    rescue SystemStackError
+      puts $! # rubocop:disable Style/SpecialGlobalVars
+      puts caller[0..100]
+      raise
     rescue StandardError => e
       log(:error) { self.exception ||= e.patch_full_message }
       raise
     ensure
       rescued { self.tags_remap_csv_file = generate_tags_remap_csv }
+      rescued { close_handling_tags_remap_csv }
+    end
+  end
+
+  # Default tree tacking behaviour
+  # @note
+  #   1. This aims to optimize the time run
+  #   2. Based on update stage, there are differentiated tracking needs
+  # @return [Symbol] the tracking mode
+  def batch_tree_track_mode(stage)
+    case stage
+    when :unarchive, :archive
+      :once
+    when :id, :id_name
+      :per_request
+    when :insert, :move
+      :per_batch
+    else
+      default_tree_tracking_mode
     end
   end
 end