eco-helpers 2.6.4 → 2.7.0

data/lib/eco/data/locations/node_base/parsing.rb

@@ -12,7 +12,9 @@ module Eco::Data::Locations::NodeBase
       raise ArgumentError, "The input csv does not have the required format to read a locations structure."
     end
 
-    # @yield [Node] optional custom serializer
+    # @yield [node, json] optional custom serializer
+    # @yieldparam node [Node] the node that is being serialized
+    # @yieldparam json [Hash] the default serialization
     # @yieldreturn [Hash] the serialized Node
     # @return [Array<Hash>] a hierarchical tree of nested Hashes via `nodes` key.
     def hash_tree_from_csv(csv, &block)
@@ -24,7 +26,7 @@ module Eco::Data::Locations::NodeBase
     # @param filename [String] the csv file.
     # @return [Array<NodePlain>, Array<NodeLevel>] with integrity issues resolved.
     def csv_nodes_from(filename, encoding: 'utf-8')
-      nodes_from_csv(csv_from(filename, encoding: 'utf-8'))
+      nodes_from_csv(csv_from(filename, encoding: encoding))
     end
   end
 end

data/lib/eco/data/locations/node_base/treeify.rb

@@ -7,6 +7,20 @@ module Eco::Data::Locations::NodeBase
   module Treeify
     include Eco::Language::AuxiliarLogger
 
+    # @yield [node, json] optional custom serializer
+    # @yieldparam node [Node] the node that is being serialized
+    # @yieldparam json [Hash] the default serialization
+    # @yieldreturn [Hash] the serialized Node
+    def serialize_node(node, parent_id: :unused)
+      msg = "Expecting Eco::Data::Locations::NodeBase. Given: #{node.class}"
+      raise ArgumentError, msg unless node.is_a?(Eco::Data::Locations::NodeBase)
+
+      node.node_hash.tap do |json|
+        json.merge!({"parent_id" => parent_id}) unless parent_id == :unused
+        json.merge!(yield(node, json))          if block_given?
+      end
+    end
+
     # @note if block is no given, it auto-detects the `serializer` **block**.
     # @yield [NodeBase] for each included node
     # @yieldreturn [Hash] custom hash model when treeifying (allows to set more keys/properties).
@@ -14,20 +28,20 @@ module Eco::Data::Locations::NodeBase
     # @return [Array<Hash>] a hierarchical tree of nested Hashes via `nodes` key.
     def treeify(nodes, skipped: [], unlinked_trees: [], &block)
       return [] if nodes.empty?
-      block ||= nodes.first.class.serializer
+      block  ||= nodes.first.class.serializer
       done_ids = {}
       warns    = []
       parents  = parents_hash(nodes)
-      get_children(nil, parents, done_ids: done_ids, skipped: skipped, warns: warns, &block).tap do |tree|
+      get_children(
+        nil,                parents,
+        done_ids: done_ids, skipped: skipped,
+        warns: warns,       &block
+      ).tap do |tree|
         check_results(
-          tree,
-          nodes,
-          parents,
-          done_ids: done_ids,
-          skipped: skipped,
-          unlinked_trees: unlinked_trees,
-          warns: warns,
-          &block
+          tree,             nodes,
+          parents,          done_ids:       done_ids,
+          skipped: skipped, unlinked_trees: unlinked_trees,
+          warns:   warns,   &block
         )
         log(:warn) { warns.join("\n") } unless warns.empty?
       end
@@ -49,56 +63,44 @@ module Eco::Data::Locations::NodeBase
     #   3. The above can translate into some
     # @yield [node]
     # @yieldreturn [Hash] custom hash model when treeifying
-    def get_children(node_id, parents, parent: nil, level: 0, done_ids: {}, skipped: [], warns: [], &block)
+    def get_children(
+      node_id,       parents,
+      parent:   nil, level:   0,
+      done_ids: {},  skipped: [],
+      warns:    [],  &block
+    )
       level_ids = []
       (parents[node_id] ||= []).each_with_object([]) do |child, results|
         # Skipping done id. Add proper warnings...
         # => rely on `done_ids` to identify if an `id` has already been done
-        next report_skipped_node(
-          child,
-          parent,
-          done_ids,
-          level,
-          level_ids,
-          parents,
-          skipped: skipped,
-          warns:   warns
-        ) if done_ids[child.id]
+        if done_ids[child.id]
+          next report_skipped_node(
+            child,            parent,
+            done_ids,         level,
+            level_ids,        parents,
+            skipped: skipped, warns: warns
+          )
+        end
 
         # Fill in tracking data
         child.parent        = parent
         child.tracked_level = level + 1
-        level_ids << child.id
-
-        node_hash = {
-          "id"        => child.id,
-          "name"      => child.name,
-          "parent_id" => node_id
-        }
-        node_hash.merge(yield(child)) if block_given?
-        # we must register the `id` before recursing down
-        done_ids[child.id] = child
-
-        children = get_children(
-          child.id,
-          parents,
-          parent:   child,
-          done_ids: done_ids,
-          level:    level + 1,
-          skipped:  skipped,
-          warns:    warns,
-          &block
+        level_ids          << child.id
+        done_ids[child.id]  = child
+
+        node_hash = serialize_node(child, parent_id: node_id, &block)
+        results  << node_hash
+        node_hash["nodes"] = get_children(
+          child.id,          parents,
+          parent: child,     done_ids: done_ids,
+          level:  level + 1, skipped:  skipped,
+          warns:  warns,     &block
         ).tap do |desc|
-          if (nil_count = desc.count(nil)) > 0
-            log(:debug) {
-              "get_children gave #{nil_count} nil values for nodes of #{child.id}"
-            }
-          end
+          next unless (nil_count = desc.count(nil)).positive?
+          log(:debug) {
+            "get_children gave #{nil_count} nil values for nodes of #{child.id}"
+          }
         end
-
-        results << node_hash.merge({
-          "nodes" => children.compact
-        })
       end
     end
 
@@ -111,7 +113,7 @@ module Eco::Data::Locations::NodeBase
     end
 
     def indent(level)
-      "#{" " * level}"
+      " " * level
     end
 
     # Method to ensure the results are consistent
@@ -120,83 +122,103 @@ module Eco::Data::Locations::NodeBase
     #     because otherwise would be part of `done_ids` anyway.
     # @param unlinked_trees [Array<Hash>] by excluding those done and skipped,
     #   it will treeify the unlinked nodes (the exclusion applies to `parants_hash`)
-    def check_results(tree, nodes, parents, done_ids: {}, skipped: [], unlinked_trees: [], warns: [], &block)
+    def check_results( # rubocop:disable Metrics/AbcSize
+      _tree,        nodes,       parents,
+      done_ids: {}, skipped: [], unlinked_trees: [],
+      warns:    [], &block
+    )
       update_skipped(skipped, parents, done_ids: done_ids) unless skipped.empty?
+      return if done_ids.count == nodes.count
 
-      if done_ids.count != nodes.count
-        tracked_nodes       = done_ids.values
-        untracked_nodes     = nodes - tracked_nodes - skipped
-        # skipped keys is inherent, as they were excluded because of id clash with done_ids
-        unlinked_parent_ids = (parents.keys - done_ids.keys).compact
+      tracked_nodes       = done_ids.values
+      untracked_nodes     = nodes - tracked_nodes - skipped
+      # skipped keys is inherent, as they were excluded because of id clash with done_ids
+      unlinked_parent_ids = (parents.keys - done_ids.keys).compact
 
+
+      # The reason of missing nodes in the output tree is unknown!
+      if skipped.empty? && unlinked_parent_ids.empty?
         msg = []
+        msg << "BUG in this library (open issue with maintainers)."
+        msg << "There were no skipped nodes nor missin referred parents, and yet:"
+        msg << "  • the tree nodes count: #{done_ids.count} ..."
+        msg << "  • doesn't match the original nodes count: #{nodes.count}"
+        raise msg.join("\n")
+      end
 
-        # The reason of missing nodes in the output tree is unknown!
-        if skipped.empty? && unlinked_parent_ids.empty?
-          msg << "BUG in this library (open issue with maintainers)."
-          msg << "There were no skipped nodes nor missin referred parents, and yet:"
-          msg << "  • the tree nodes count: #{done_ids.count} ..."
-          msg << "  • doesn't match the original nodes count: #{nodes.count}"
-          raise msg.join("\n")
-        end
+      unless unlinked_parent_ids.empty?
+        str_skipped = (skipped.count < 15 ? " => #{skipped.map(&:id).join(', ')}" : '')
 
-        unless unlinked_parent_ids.empty?
-          msg << "There are #{unlinked_parent_ids.count} referred parent_id's NOT linked to the root:"
-          msg << "  • total_nodes: #{nodes.count}"
-          msg << "  • tracked_nodes: #{tracked_nodes.count}"
-          msg << "  • untracked_nodes: #{untracked_nodes.count}"
-          msg << "  • unlinked_parents: #{unlinked_parent_ids.count}"
-          msg << "  • skipped (repeated) nodes: #{skipped.count}" unless skipped.empty?
-
-          unlinked_nodes   = nodes - skipped
-          unlinked_parents = parents.slice(*unlinked_parent_ids) # doesn'thave skipped ones
-
-          residual_skipped = []
-          unlinked_trees.concat \
-            get_unlinked_trees(
-              unlinked_nodes,
-              unlinked_parents,
-              done_ids: done_ids,
-              skipped:  residual_skipped,
-              warns:    warns,
-              &block
-            )
-
-          update_skipped(skipped, parents, with: residual_skipped, done_ids: done_ids) unless residual_skipped.empty?
-
-          tracked_nodes       = done_ids.values
-          untracked_nodes     = nodes - tracked_nodes - skipped
-          unlinked_parent_ids = (parents.keys - done_ids.keys).compact
-
-          msg << "After treeifying via the unlinked_parents:"
-          msg << "  • total_nodes: #{nodes.count}"
-          msg << "  • tracked_nodes: #{tracked_nodes.count}"
-          msg << "  • untracked_nodes: #{untracked_nodes.count}"
-          msg << "  • unlinked_parents: #{unlinked_parent_ids.count}"
-          msg << "  • skipped in this step: #{residual_skipped.count}"
-        end
+        msg = []
+        msg << "There are #{unlinked_parent_ids.count} referred parent_id's NOT linked to the root:"
+        msg << "  • total_nodes: #{nodes.count}"
+        msg << "  • tracked_nodes: #{tracked_nodes.count}"
+        msg << "  • untracked_nodes: #{untracked_nodes.count}"
+        msg << "  • unlinked_parents: #{unlinked_parent_ids.count}"
+        msg << "  • skipped (repeated) nodes: #{skipped.count}#{str_skipped}" unless skipped.empty?
+        warns << msg.join("\n")
+
+        unlinked_nodes   = nodes - skipped
+        unlinked_parents = parents.slice(*unlinked_parent_ids) # doesn'thave skipped ones
+
+        residual_skipped = []
+        ulnk_trees       = get_unlinked_trees(
+          unlinked_nodes,     unlinked_parents,
+          done_ids: done_ids, skipped: residual_skipped,
+          warns: warns,       &block
+        )
+        unlinked_trees.concat(ulnk_trees)
+
+        update_skipped(skipped, parents, with: residual_skipped, done_ids: done_ids) unless residual_skipped.empty?
+
+        tracked_nodes        = done_ids.values
+        untracked_nodes      = nodes - tracked_nodes - skipped
+        unlinked_parent_ids  = (parents.keys - done_ids.keys).compact
+
+        str_unlinked_parents =
+          unlinked_parent_ids.count < 35 ? unlinked_parent_ids.join(', ') : unlinked_parent_ids.count
 
-        msg << "  • total skipped (repeated) nodes: #{skipped.count} !!" unless skipped.empty?
+        str_skipped = (residual_skipped.count < 15 ? " => #{residual_skipped.map(&:id).join(', ')}" : '')
+
+        msg = []
+        msg << "After treeifying via the unlinked_parents:"
+        msg << "  • total_nodes: #{nodes.count}"
+        msg << "  • tracked_nodes: #{tracked_nodes.count}"
+        msg << "  • untracked_nodes: #{untracked_nodes.count}"
+        msg << "  • missing_parents: #{str_unlinked_parents}"
+        msg << "  • skipped in this step: #{residual_skipped.count}#{str_skipped}" unless residual_skipped.empty?
         warns << msg.join("\n")
-        nil
       end
+
+      str_skipped = (skipped.count < 15 ? " => #{skipped.map(&:id).join(', ')}" : '')
+      warns << "  • total skipped (repeated) nodes: #{skipped.count}!!#{str_skipped}" unless skipped.empty?
+      nil
     end
 
     # Treeifies the unlinked nodes by scoping existing parent ids.
-    def get_unlinked_trees(nodes, parents, done_ids: {}, skipped: [], warns: [], &block)
+    def get_unlinked_trees(
+      nodes,        parents,
+      done_ids: {}, skipped: [],
+      warns: [],    &block
+    )
       node_ids           = nodes.map(&:id)
       parent_ids         = parents.keys & node_ids
       missing_parent_ids = parents.keys - parent_ids
       missing_parents    = parents.slice(*missing_parent_ids)
-      warns << "  • missing_parents: #{missing_parents.count}"
-      nil_parent_nodes   = missing_parents.each_with_object([]) do |(id, nodes), mem|
-        nodes.each {|node| node.parent_id = nil}
-        mem.concat(nodes)
+      warns             << "  • missing_parents: #{missing_parents.count}"
+
+      nil_parent_nodes = missing_parents.each_with_object([]) do |(_id, nds), mem|
+        nds.each {|node| node.parent_id = nil}
+        mem.concat(nds)
       end
       rest_parents = parents.slice(*parent_ids).merge({
         nil => nil_parent_nodes
       })
-      get_children(nil, rest_parents, done_ids: done_ids, skipped: skipped, warns: warns, &block)
+      get_children(
+        nil,                rest_parents,
+        done_ids: done_ids, skipped: skipped,
+        warns:    warns,    &block
+      )
     end
 
     # Same as `get_children` but not performing checks and with
@@ -205,28 +227,18 @@ module Eco::Data::Locations::NodeBase
     #   skipped, because their parent was skipped.
     def get_tree_nodes_raw(node_id, parents, src_plain: true, &block)
       (parents[node_id] ||= []).each_with_object([]) do |child, results|
-        unless src_plain
-          node_hash = {
-            "id"        => child.id,
-            "name"      => child.name,
-            "parent_id" => node_id
-          }
-          node_hash.merge(yield(child)) if block_given?
-        end
-
+        node_hash   = serialize_node(child, parent_id: node_id, &block) unless src_plain
         descendants = get_tree_nodes_raw(child.id, parents, src_plain: src_plain, &block).tap do |desc|
-          if (nil_count = desc.count(nil)) > 0
-            puts "get_tree_nodes_raw gave #{nil_count} nil values for nodes of #{child.id}"
-          end
+          next unless (nil_count = desc.count(nil)).positive?
+          log(:debug) {
+            "get_tree_nodes_raw gave #{nil_count} nil values for nodes of #{child.id}"
+          }
         end
+        next results.concat(descendants) if src_plain
 
-        if src_plain
-          results.concat(descendants)
-        else
-          results << node_hash.merge({
-            "nodes" => descendants.compact
-          })
-        end
+        results << node_hash.merge({
+          "nodes" => descendants.compact
+        })
       end
     end
 
@@ -244,22 +256,27 @@ module Eco::Data::Locations::NodeBase
 
     # With given a skipped `node` (repeated `id`), it gives different warnings,
     # provided that the context in which the double-up `id` happened is identified.
-    def report_skipped_node(node, parent, done_ids, level, level_ids, parents, skipped: [], warns: [])
-      skipped << node
+    def report_skipped_node( # rubocop:disable Metrics/AbcSize
+      node,        parent,
+      done_ids,    level,
+      level_ids,   parents,
+      skipped: [], warns: []
+    )
+      skipped     << node
       lev          = level + 1
       done_node    = done_ids[node.id]
       prev_parent  = node.parent
       prev_level   = node.tracked_level
       node_dup     = done_node && (done_node != node)
       lev_dup      = level_ids.include?(node.id)
-      multi_parent = (!prev_parent == !!parent) || (prev_parent && (prev_parent.id != parent.id))
+      multi_parent = (!prev_parent == !!parent) || (prev_parent && (prev_parent.id != parent.id)) # rubocop:disable Style/DoubleNegation
 
       row_num      = node.respond_to?(:row_num) ? node.row_num : nil
       row_str      = row_num ? "(Row: #{row_num}) " : ''
       node_str     = "#{row_str}Node '#{node.id}' #{level_msg(lev)} (#{parent_msg(parent)})"
 
-      msg  = []
-      msg << "#{indent(level)}Skipping #{node_str}."
+      msg          = []
+      msg         << "#{indent(level)}Skipping #{node_str}."
 
       # Implementation integrity guard
       # => as we don't register in `done_ids` those that are skipped,

data/lib/eco/data/locations/node_base.rb

@@ -10,16 +10,16 @@ module Eco::Data::Locations
     require_relative 'node_base/builder'
     extend Eco::Data::Locations::NodeBase::Builder
 
-    ALL_ATTRS = []
+    ALL_ATTRS = [].freeze
 
     attr_accessor :tracked_level, :parent
 
     def copy
-      self.class.new.set_attrs(**self.to_h)
+      self.class.new.set_attrs(**to_h)
     end
 
     def attr(sym)
-      self.send(sym.to_sym)
+      send(sym.to_sym)
     end
 
     def attr?(sym)
@@ -32,13 +32,24 @@ module Eco::Data::Locations
     end
 
     def set_attr(attr, value)
-      self.send("#{attr}=", value)
+      send("#{attr}=", value)
     end
 
     def values_at(*attrs)
       attrs.map {|a| attr(a)}
     end
 
+    # @yield [node, json] optional custom serializer
+    # @yieldparam node [Node] self
+    # @yieldparam json [Hash] the default serialization
+    # @yieldreturn [Hash] the serialized Node
+    def node_hash(stringify_keys: true)
+      json = to_h(:id, :name, :parent_id)
+      json.transform_keys!(&:to_s)   if stringify_keys
+      json.merge!(yield(self, json)) if block_given?
+      json
+    end
+
     def to_h(*attrs)
       attrs = self.class::ALL_ATTRS if attrs.empty?
       attrs.zip(values_at(*attrs)).to_h

data/lib/eco/data/locations/node_diff/accessors.rb

@@ -10,30 +10,38 @@ class Eco::Data::Locations::NodeDiff
     end
 
     module ClassMethods
+      include Eco::Data::Strings::SnakeCase
+
       # Creates the defined accessor attributes against `NodeDiff`
       # It also creates a method with question mark that evaluates true if **any** the attr changed.
       # @note the defined attributes are expected to be the keys within
       #   the source Hashes that are being compared.
-      # @note accessing `src1` (prev) attributes, will have it's method as `prev_[attrName]`
+      # @note accessing `src_1` (prev) attributes, will have it's method as `prev_[attrName]`
       def attr_expose(*attrs)
         attrs.each do |attr|
           meth  = attr.to_sym
-          methp = "prev_#{meth}".to_sym
-          methq = "diff_#{meth}?".to_sym
+          methp = "#{meth}_prev".to_sym
+          methq = "#{meth}?".to_sym
 
+          # current value
           define_method meth do
             attr(meth)
           end
+          alias_method snake_case(meth), meth
 
+          # prev value
           define_method methp do
             attr_prev(meth)
           end
+          alias_method snake_case(methp), methp
 
-          exposed_attrs |= [meth]
-
+          # has it changed?
           define_method methq do
             diff_attr?(meth)
           end
+          alias_method snake_case(methq), methq
+
+          @exposed_attrs = (@exposed_attrs || []) | [meth]
         end
       end
 

data/lib/eco/data/locations/node_diff/nodes_diff/clustered_treeify.rb

@@ -0,0 +1,101 @@
+class Eco::Data::Locations::NodeDiff::NodesDiff
+  # treeify helper... with diffs you cannot expect all nodes
+  # to be there (as not all generated a difference)
+  # This class helps to cluster nodes that are related
+  # (then each cluster has as a top only one diff)
+  class ClusteredTreeify
+    class DataIntegrityError < StandardError
+    end
+
+    class << self
+      def mode(value)
+        case value
+        when :destination, :curr
+          :curr
+        when :source, :prev
+          :prev
+        else
+          raise ArgumentError, "Unknown mode ':#{value}'"
+        end
+      end
+    end
+
+    attr_reader :diffs, :mode
+
+    # @param on [Symbol] if it treeifies based on :prev (`:source`)
+    #   or `:curr` (`:destination`)
+    #   - default to `:prev` because it's mostly use to archive
+    def initialize(diffs, mode: :prev)
+      @diffs = diffs
+      @mode  = self.class.mode(mode)
+    end
+
+    # @return [Hash] where _key_ is the parent `diff` and _value_ are all
+    #   descendants thereof
+    def clusters
+      @clusters = only_present_parents_hash.reject do |_pid, tree|
+        all_children_diffs_hash.key?(tree.id)
+      end.each_with_object({}) do |(_pid, tree), clus|
+        clus[tree.diff] = tree.all_descendants.map(&:diff).uniq
+      end
+    end
+
+    private
+
+    def all_children_diffs_hash
+      @all_children_diffs_hash ||= all_children_diffs.each_with_object({}) do |node, out|
+        out[node.id] = node.diff
+      end
+    end
+
+    def all_children_diffs
+      @all_children_diffs ||= only_present_parents_hash.values.map(&:all_descendants).flatten.uniq
+    end
+
+    # wipes out tree nodes that got there just as a token (diff not actually present)
+    def only_present_parents_hash
+      @only_present_parents_hash ||= parents_hash.select do |_pid, tree|
+        tree.diff?
+      end
+    end
+
+    # builds up the hierarchy
+    def parents_hash
+      @parents_hash ||= diffs.each_with_object({}) do |diff, out|
+        id  = node_id(diff)
+        pid = parent_id(diff)
+
+        if id == pid
+          %i[unarchive? id? id_name? insert? move? archive?].each do |stage|
+            puts "Stage #{stage}: #{diff.send(stage)}"
+          end
+          msg = "Node '#{id}' seems to be parent of itself"
+          raise DataIntegrityError, msg
+        end
+
+        out[pid] ||= DiffsTree.new(id: pid)
+        out[id]  ||= DiffsTree.new(id: id)
+        out[pid].add_child(out[id])
+        out[id].diff = diff
+      end
+    end
+
+    def parent_id(diff)
+      case mode
+      when :prev
+        diff.parent_id_prev
+      else
+        diff.parent_id
+      end&.upcase
+    end
+
+    def node_id(diff)
+      case mode
+      when :prev
+        diff.node_id_prev
+      else
+        diff.node_id
+      end&.upcase
+    end
+  end
+end