eco-helpers 2.5.2 → 2.5.4

data/lib/eco/data/locations/node_base/csv_convert.rb

@@ -0,0 +1,57 @@
+module Eco::Data::Locations::NodeBase
+  module CsvConvert
+    include Eco::Data::Locations::NodeBase::Parsing
+
+    def tree_class
+      Eco::API::Organization::TagTree
+    end
+
+    # @yield [Node] optional custom serializer
+    # @yieldreturn [Hash] the serialized Node
+    # @param value [CSV::Table, Eco::API::Organization::TagTree]
+    # @return [Array<Hash>] a plain list of hash nodes
+    def hash_list(value, &block)
+      return hash_list(org_tree(value)) if value.is_a?(::CSV::Table)
+      return value.as_nodes_json        if value.is_a?(tree_class)
+      raise ArgumentError, "Expecting Eco::API::Organization::TagTree or CSV::Table. Given: #{value.class}"
+    end
+
+    # @yield [Node] optional custom serializer
+    # @yieldreturn [Hash] the serialized Node
+    # @param value [CSV::Table, Eco::API::Organization::TagTree]
+    # @return [Array<Hash>] a hierarchical tree of hash nodes,
+    #   ready to be parsed as an organization tagtree
+    def hash_tree(value, &block)
+      return hash_tree_from_csv(value, &block) if value.is_a?(::CSV::Table)
+      return value.as_json                     if value.is_a?(tree_class)
+      raise ArgumentError, "Expecting Eco::API::Organization::TagTree or CSV::Table. Given: #{value.class}"
+    end
+
+    # @yield [Node] optional custom serializer
+    # @yieldreturn [Hash] the serialized Node
+    # @param value [CSV::Table, Eco::API::Organization::TagTree]
+    # @return [Eco::API::Organization::TagTree]
+    def org_tree(value, &block)
+      return tree_class.new(hash_tree(value), &block) if value.is_a?(::CSV::Table)
+      return tree_class.new(value.as_json)            if value.is_a?(tree_class)
+      raise ArgumentError, "Expecting Eco::API::Organization::TagTree or CSV::Table. Given: #{value.class}"
+    end
+
+    # @yield [Node] optional custom serializer
+    # @yieldreturn [Hash] the serialized Node
+    # @return [CSV::Table] a table with L1 to Ln columns ready for dump to csv
+    def csv_tree(value, encoding: 'utf-8', &block)
+      Eco::CSV::Table.new(hash_tree_to_tree_csv(hash_tree(value, &block)))
+    end
+
+    # @note it just converts to an organizational tagtree and uses a helper method.
+    # @yield [Node] optional custom serializer
+    # @yieldreturn [Hash] the serialized Node
+    # @param value [CSV::Table, Eco::API::Organization::TagTree]
+    # @return [CSV::Table] a table with a list of nodes and their parents
+    def csv_list(value, &block)
+      value = org_tree(value, &block) unless value.is_a?(tree_class)
+      Eco::CSV.Table.new(hash_list(value))
+    end
+  end
+end

data/lib/eco/data/locations/node_base/parsing.rb

@@ -0,0 +1,30 @@
+module Eco::Data::Locations::NodeBase
+  module Parsing
+    include Eco::Data::Locations::Convert
+    include Eco::Data::Locations::NodeBase::Treeify
+
+    # @param csv [CSV::Table]
+    # @return [Array<NodePlain>, Array<NodeLevel>] with integrity issues resolved.
+    def nodes_from_csv(csv)
+      raise ArgumentError, "Expecting CSV::Table. Given: #{csv.class}" unless csv.is_a?(::CSV::Table)
+      return Eco::Data::Locations::NodePlain.nodes_from_csv(csv) if Eco::Data::Locations::NodePlain.csv_matches_format?(csv)
+      return Eco::Data::Locations::NodeLevel.nodes_from_csv(csv) if Eco::Data::Locations::NodeLevel.csv_matches_format?(csv)
+      raise ArgumentError, "The input csv does not have the required format to read a locations structure."
+    end
+
+    # @yield [Node] optional custom serializer
+    # @yieldreturn [Hash] the serialized Node
+    # @return [Array<Hash>] a hierarchical tree of nested Hashes via `nodes` key.
+    def hash_tree_from_csv(csv, &block)
+      raise ArgumentError, "Expecting CSV::Table. Given: #{csv.class}" unless csv.is_a?(::CSV::Table)
+      treeify(nodes_from_csv(csv), &block)
+    end
+
+    # Shortcut to obtain a list of parsed nodes out of a file
+    # @param filename [String] the csv file.
+    # @return [Array<NodePlain>, Array<NodeLevel>] with integrity issues resolved.
+    def csv_nodes_from(filename, encoding: 'utf-8')
+      nodes_from_csv(csv_from(filename, encoding: 'utf-8'))
+    end
+  end
+end

data/lib/eco/data/locations/node_base/serial.rb

@@ -0,0 +1,26 @@
+module Eco::Data::Locations::NodeBase
+  module Serial
+    include Eco::Data::Locations::NodeBase::Treeify
+    include Eco::Data::Locations::Convert
+
+    # @param item [Eco::Data::Locations::NodeBase] an instance object of a child class.
+    # @return [Proc] the serializer to be used.
+    def serializer(item)
+      raise "Execting a chidren of NodeBase. Given: #{item.class}" unless item.class < Eco::Data::Locations::NodeBase
+      item.serializer
+    end
+
+    # @paran nodes [Array<NodeBase>]
+    # @return [CSV::Table] ready to dump into a hierarhical **csv** (columns are tree levels)
+    def nodes_to_csv_tree(nodes)
+      hash_tree_to_tree_csv(treeify(nodes))
+    end
+
+    # @paran nodes [Array<NodeBase>]
+    # @return [CSV::Table] ready to dump into a nodes list **csv** (rows are nodes; a column holds `parent_id`)
+    def nodes_to_csv_list(nodes)
+      tree = Eco::API::Organization::TagTree.new(treeify(nodes))
+      Eco::CSV::Table.new(tree.as_nodes_json)
+    end
+  end
+end

data/lib/eco/data/locations/node_base/tag_validations.rb

@@ -0,0 +1,52 @@
+module Eco::Data::Locations::NodeBase
+  module TagValidations
+    ALLOWED_CHARACTERS = "A-Za-z0-9 &_'\/.-"
+    VALID_TAG_REGEX    = /^[#{ALLOWED_CHARACTERS}]+$/
+    INVALID_TAG_REGEX  = /[^#{ALLOWED_CHARACTERS}]+/
+    VALID_TAG_CHARS    = /[#{ALLOWED_CHARACTERS}]+/
+    DOUBLE_BLANKS      = /\s\s+/
+
+    def clean_id(str)
+      blanks_x2 = has_double_blanks?(str)
+      partial   = replace_not_allowed(str)
+      remove_double_blanks(partial).tap do |result|
+        next if invalid_warned?
+        if partial != str
+          invalid_chars = identify_invalid_characters(str)
+          puts "• (Row: #{self.row_num}) Invalid characters _#{invalid_chars}_ (removed): '#{str}' (converted to '#{result}')"
+        elsif blanks_x2
+          puts "• (Row: #{self.row_num}) Double blanks (removed): '#{str}' (converted to '#{result}')"
+        end
+        invalid_warned!
+      end
+    end
+
+    def invalid_warned?
+      @invalid_warned ||= false
+    end
+
+    def invalid_warned!
+      @invalid_warned = true
+    end
+
+    def has_double_blanks?(str)
+      return false if str.nil?
+      str.match(DOUBLE_BLANKS)
+    end
+
+    def remove_double_blanks(str)
+      return nil if str.nil?
+      str.gsub(DOUBLE_BLANKS, ' ').strip
+    end
+
+    def replace_not_allowed(str)
+      return nil if str.nil?
+      return str if str.match(VALID_TAG_REGEX)
+      str.gsub(INVALID_TAG_REGEX, ' ')
+    end
+
+    def identify_invalid_characters(str)
+      str.gsub(VALID_TAG_CHARS, '')
+    end
+  end
+end

data/lib/eco/data/locations/node_base/treeify.rb

@@ -0,0 +1,150 @@
+module Eco::Data::Locations::NodeBase
+  # Generic treeifier
+  # @note expects nodes to have these properties:
+  #   1. `id`, `name` and `parentId`
+  #   2. `parent`
+  #   3. `tracked_level`
+  module Treeify
+    include Eco::Language::AuxiliarLogger
+
+    # @note if block is no given, it auto-detects the `serializer` **block**.
+    # @yield [NodeBase] for each included node
+    # @yieldreturn [Hash] custom hash model when treeifying (allows to set more keys/properties).
+    # @nodes [Array<NodeBase>] list of nodes
+    # @return [Array<Hash>] a hierarchical tree of nested Hashes via `nodes` key.
+    def treeify(nodes, &block)
+      return [] if nodes.empty?
+      block ||= nodes.first.class.serializer
+      get_children(nil, parents_hash(nodes), &block)
+    end
+
+    private
+
+    def parents_hash(nodes)
+      nodes.each_with_object({}) do |node, parents|
+        (parents[node.parentId] ||= []).push(node)
+      end
+    end
+
+    # @note
+    #   1. It tracks the `level` where nodes are discovered
+    #   2. If the node had already a tracked level, it warns and keeps the previous level
+    #   3. The above can translate into some
+    # @yield [node]
+    # @yieldreturn [Hash] custom hash model when treeifying
+    def get_children(node_id, parents, parent: nil, done_ids: {}, level: 0, &block)
+      level_ids = []
+      (parents[node_id] ||= []).each_with_object([]) do |child, results|
+        # Skipping done id. Add proper warnings...
+        # => rely on `done_ids` to identify if an `id` has already been done
+        next report_skipped_node(child, parent, done_ids, level, level_ids, parents) if done_ids[child.id]
+
+        # Fill in tracking data
+        child.parent        = parent
+        child.tracked_level = level + 1
+        level_ids << child.id
+
+        node_hash = {
+          "id"        => child.id,
+          "name"      => child.name,
+          "parent_id" => node_id
+        }
+        node_hash.merge(yield(child)) if block_given?
+        # we must register the `id` before recursing down
+        done_ids[child.id] = child
+        results << node_hash.merge({
+          "nodes" => get_children(child.id, parents, parent: child, done_ids: done_ids, level: level + 1, &block).compact
+        })
+      end
+    end
+
+    def parent_msg(parent)
+      parent ? "child of '#{parent.id}'" : "top level"
+    end
+
+    def level_msg(level)
+      "at lev: #{level}"
+    end
+
+    def indent(level)
+      "#{" " * level}"
+    end
+
+    # Gives different warnings, depending the case
+    def report_skipped_node(node, parent, done_ids, level, level_ids, parents)
+      lev          = level + 1
+      done_node    = done_ids[node.id]
+      prev_parent  = node.parent
+      prev_level   = node.tracked_level
+      node_dup     = done_node && (done_node != node)
+      lev_dup      = level_ids.include?(node.id)
+      multi_parent = (!prev_parent == !!parent) || (prev_parent && (prev_parent.id != parent.id))
+
+      row_num      = node.respond_to?(:row_num) ? node.row_num : nil
+      row_str      = row_num ? "(Row: #{row_num}) " : ''
+      node_str     = "#{row_str}Node '#{node.id}' #{level_msg(lev)} (#{parent_msg(parent)})"
+
+      # Implementation integrity guard
+      # => as we don't register in `done_ids` those that are skipped,
+      #   when a `node` has already a tracked `parent` or `level`,
+      #   it should not happen that the `node.id` retrieves a different node in `node_ids`.
+      if (prev_parent || prev_level) && node_dup # && !done_node
+        str  = "Integrity issue in Treeify. "
+        str << "A Node with tracked level or parent should be present in done_ids, but it isn't."
+        str << "\n • #{node_str}."
+        raise str
+      end
+      # From here on, do NOT expect `node_dup` where `node` has tracked `parent` or `level`.
+
+      # Implementation integrity guard
+      # => as`level_ids` only relates to the current `parent`,
+      #   and as `done_ids` don't get those skipped,
+      #   when we get an ID double-up in `level_ids`,
+      #   there must be a `done_node` AND
+      #   `done_node` can only have `tracked_level` matching the current one
+      #   Moreover, they should have exactly the same parentId.
+      if lev_dup && (multi_parent || !done_node || done_node.tracked_level != lev)
+        str  = "Integrity issue in Treeify. "
+        str << "A Node with ID already in level_ids should have same tracked_level as current level."
+        str << "\n • #{node_str}."
+        raise str
+      end
+      # From here on, do NOT expect `lev_up` where there isn't `done_node` or it has different level or parent.
+
+      cyclic    = multi_parent && done_node == node
+      double_up = node_dup || lev_dup
+
+      msg  = []
+      msg << "#{indent(level)}WARNING: Skipping #{node_str}."
+
+      if cyclic
+        str  = "#{indent(level)+1}Cyclic definition. By skipping the node, "
+        str << "it will remain as #{parent_msg(done_node.parent)} (#{level_msg(prev_level)})."
+        msg << str
+      end
+
+      if double_up
+        str  = "#{indent(level)+1}The node ID has been tracked as #{level_msg(done_node.tracked_level)}, "
+        str << "as #{parent_msg(node_dup.parent)} "
+        str << "(same parent)."      if lev_dup
+        str << "(different parent)." if multi_parent
+        msg << str
+      end
+
+      unless cyclic || double_up
+        str  = "Integrity issue in Treeify. "
+        str  = "Skipping is only applicable to double_ups or cyclic nodes."
+        str << "\n • #{node_str}."
+        raise str
+      end
+
+      if children = parents[node.id]
+        str  = "#{indent(level)+1}Immediate children of skipped node (will probably be missing): "
+        str << children.map {|gc| "'#{gc.id}'"}.join(", ")
+        msg << str
+      end
+
+      log(:warn) { msg.join('\n') }
+    end
+  end
+end

data/lib/eco/data/locations/node_base.rb

@@ -0,0 +1,48 @@
+module Eco::Data::Locations
+  module NodeBase
+    require_relative 'node_base/tag_validations'
+    include Eco::Data::Locations::NodeBase::TagValidations
+
+    require_relative 'node_base/treeify'
+    require_relative 'node_base/parsing'
+    require_relative 'node_base/serial'
+    require_relative 'node_base/csv_convert'
+    require_relative 'node_base/builder'
+    extend Eco::Data::Locations::NodeBase::Builder
+
+    ALL_ATTRS = []
+
+    attr_accessor :tracked_level, :parent
+
+    def copy
+      self.class.new.set_attrs(**self.to_h)
+    end
+
+    def attr(sym)
+      self.send(sym.to_sym)
+    end
+
+    def set_attrs(**kargs)
+      kargs.each {|attr, value| set_attr(attr, value)}
+      self
+    end
+
+    def set_attr(attr, value)
+      self.send("#{attr}=", value)
+    end
+
+    def values_at(*attrs)
+      attrs.map {|a| attr(a)}
+    end
+
+    def to_h(*attrs)
+      attrs = self.class::ALL_ATTRS if attrs.empty?
+      attrs.zip(values_at(*attrs)).to_h
+    end
+
+    def slice(*attrs)
+      return {} if attrs.empty?
+      to_h(*attrs)
+    end
+  end
+end

data/lib/eco/data/locations/node_diff/accessors.rb

@@ -0,0 +1,46 @@
+class Eco::Data::Locations::NodeDiff
+  module Accessors
+    class << self
+      def included(base)
+        super(base)
+        base.extend Eco::Language::Models::ClassHelpers
+        base.extend ClassMethods
+        base.inheritable_class_vars :exposed_attrs
+      end
+    end
+
+    module ClassMethods
+      # Creates the defined accessor attributes against `NodeDiff`
+      # It also creates a method with question mark that evaluates true if **any** the attr changed.
+      # @note the defined attributes are expected to be the keys within
+      #   the source Hashes that are being compared.
+      # @note accessing `src1` (prev) attributes, will have it's method as `prev_[attrName]`
+      def attr_expose(*attrs)
+        attrs.each do |attr|
+          meth  = attr.to_sym
+          methp = "prev_#{meth}".to_sym
+          methq = "diff_#{meth}?".to_sym
+
+          define_method meth do
+            attr(meth)
+          end
+
+          define_method methp do
+            attr_prev(meth)
+          end
+
+          exposed_attrs |= [meth]
+
+          define_method methq do
+            diff_attr?(meth)
+          end
+        end
+      end
+
+      # Keeps track on what attributes have been exposed.
+      def exposed_attrs
+        @exposed_attrs ||= []
+      end
+    end
+  end
+end

data/lib/eco/data/locations/node_diff/nodes_diff.rb

@@ -0,0 +1,90 @@
+class Eco::Data::Locations::NodeDiff
+  # Adjusts ArrayDiff for Nodes diffs in a location structure
+  class NodesDiff < Eco::Data::Hashes::ArrayDiff
+    extend Eco::Data::Locations::NodeDiff::Selectors
+
+    SELECTORS = %i[id name id_name insert unarchive update move archive]
+    selector *SELECTORS
+
+    class_resolver :diff_result_class, Eco::Data::Locations::NodeDiff
+    attr_reader :original_tree
+
+    def initialize(*args, original_tree:, **kargs, &block)
+      super(*args, **kargs, &block)
+      @original_tree = original_tree
+    end
+
+    def diffs
+      @diffs ||= super.select do |res|
+        res.unarchive? || res.id_name? || res.insert? || res.move? || res.archive?
+      end
+    end
+
+    def diffs_details
+      section = '#' * 10
+      msg  = ''
+      if insert?
+        msg << " #{section}  I N S E R T S  #{section}\n"
+        msg << insert.map {|d| d.diff_hash.pretty_inspect}.join('')
+      end
+
+      if update?
+        msg << "\n #{section}  U P D A T E S  #{section}\n"
+        update.each do |d|
+          flags  = ''
+          #flags << 'i' if d.id?
+          flags << 'n' if d.diff_name?
+          flags << 'm' if d.move?
+          flags << 'u' if d.unarchive?
+          msg << "<< #{flags} >> "
+          msg << d.diff_hash.pretty_inspect
+        end
+      end
+
+      if archive?
+        msg << "\n #{section}  A R C H I V E S  #{section}\n"
+        msg << archive.map {|d| d.diff_hash.pretty_inspect}.join('')
+      end
+
+      msg
+    end
+
+    def diffs_summary
+      return "There were no differences identified" if diffs.empty?
+      msg  = "Identified #{diffs.count} differences:\n"
+      msg << when_present(insert, '') do |count|
+        "  • #{count} nodes to insert\n"
+      end
+      msg << when_present(update, '') do |count|
+        "  • #{count} nodes to update\n"
+      end
+      # msg << when_present(id, '') do |count|
+      #   "    • #{count} nodes to change id\n"
+      # end
+      msg << when_present(name, '') do |count|
+        "    • #{count} nodes to change name\n"
+      end
+      msg << when_present(move, '') do |count|
+        "    • #{count} nodes to move\n"
+      end
+      msg << when_present(unarchive, '') do |count|
+        "    • #{count} nodes to unarchive\n"
+      end
+      msg << when_present(archive, '') do |count|
+        "  • #{count} nodes to archive\n"
+      end
+      msg
+    end
+
+    private
+
+    def when_present(list, default = nil)
+      count = list.count
+      if count > 0
+        yield(count)
+      else
+        default
+      end
+    end
+  end
+end

data/lib/eco/data/locations/node_diff/selectors.rb

@@ -0,0 +1,20 @@
+class Eco::Data::Locations::NodeDiff
+  module Selectors
+    # Creates selector methods on `diffs` (nodes that changed)
+    # It also creates a method with question mark that evaluates true if **any** diff matches.
+    # @note the selector method name with a question mark should exist in the `diff_result_class`
+    def selector(*attrs)
+      attrs.each do |attr|
+        meth  = attr.to_sym
+        methq = "#{meth}?".to_sym
+        define_method meth do
+          diffs.select(&methq)
+        end
+
+        define_method methq do
+          diffs.any(&methq)
+        end
+      end
+    end
+  end
+end

data/lib/eco/data/locations/node_diff.rb

@@ -0,0 +1,55 @@
+module Eco::Data::Locations
+  # Differences between node before and after
+  # @note this works with `Hash` object where basic keys are:
+  #   - `nodeId`
+  #   - `name`
+  #   - `parentId`
+  #   - `archived`
+  # @note other properties can be part of the `Hash` although
+  #   they may not influence the results.
+  class NodeDiff < Eco::Data::Hashes::DiffResult
+    require_relative 'node_diff/accessors'
+    require_relative 'node_diff/selectors'
+    require_relative 'node_diff/nodes_diff'
+
+    include Eco::Data::Locations::NodeDiff::Accessors
+
+    key     :nodeId
+    compare :parentId, :name, :archived
+    case_sensitive false
+
+    attr_expose :nodeId, :name, :parentId, :archived
+
+    alias_method :insert?, :new?
+
+    alias_method :diff_name_src?, :diff_name?
+    # Has the property `name` changed?
+    def diff_name?
+      diff_name_src? && update?
+    end
+    alias_method :name?, :diff_name?
+    alias_method :id? , :diff_name? # currently a change of name is a change of id (tag)
+    #alias_method :id? , :key?
+    #alias_method :nodeId? , :id?
+
+    # Has any of `id` or `name` properties changed?
+    def id_name?
+      id? || diff_name?
+    end
+
+    # Has the parent id changed?
+    def move?
+      update? && diff_parentId?
+    end
+
+    # Has the `archived` property changed and it was `true`?
+    def unarchive?
+      !archived && update? && diff_archived?
+    end
+
+    # Has the `archived` property changed and it was `false`?
+    def archive?
+      !prev_archived && (del? || archived)
+    end
+  end
+end

data/lib/eco/data/locations/node_level/builder.rb

@@ -0,0 +1,6 @@
+class Eco::Data::Locations::NodeLevel
+  module Builder
+    include Eco::Data::Locations::NodeLevel::Parsing
+    include Eco::Data::Locations::NodeLevel::Serial
+  end
+end

data/lib/eco/data/locations/node_level/cleaner.rb

@@ -0,0 +1,74 @@
+class Eco::Data::Locations::NodeLevel
+  module Cleaner
+    include Eco::Language::AuxiliarLogger
+    include Eco::Data::Locations::Convert
+
+    # Prevents repeated node ids/tags, decouples merged levels,
+    # covers gaps (jumping multiple levels)
+    # @note
+    #   1. It first discards node ids/tags that have been already pulled (discard repeated)
+    #   2. For non repeated, it identifies if there's a gap (jump of multiple levels)
+    #   3. It covers the gap if present by decoupling merged parent(s) from the same node (see node.decouple)
+    #   4. Then, it delegates the filling in of parents to `fill_in_parents` function.
+    # @return [Array<NodeLevel>] child to parent relationships solved and no double-ups.
+    def tidy_nodes(nodes, prev_level: 0, main: true)
+      reset_trackers! if main
+      nodes.each_with_object([]) do |node, out|
+        node_id = node.id
+        if done_ids.include?(node_id)
+          repeated_ids << "#{node_id} (level: #{node.level})"
+        else
+          level = node.actual_level
+          if level > prev_level + 1
+            gap = level - (prev_level + 1)
+            msg = "(Row: #{node.row_num}) ID/Tag '#{node_id}' (lev #{level}) jumps #{gap} level(s) (expected #{prev_level + 1})."
+            #puts "  " + node.tags_array.pretty_inspect
+            missing_nodes = node.decouple(gap)
+
+            msg << "\n  Adding missing upper level(s): " + missing_nodes.map(&:raw_tag).pretty_inspect
+            log(:info) { msg }
+
+            out.push(*tidy_nodes(missing_nodes, prev_level: prev_level, main: false))
+            # puts node.actual_level
+            # pp node.tags_array
+            level = prev_level + 1
+          end
+          out       << node
+          done_ids  << node_id
+          prev_level = level
+        end
+      end.yield_self do |out|
+        report_repeated_node_ids(repeated_ids) if main
+        fill_in_parents(out)
+      end
+    end
+
+    # Sets the `parentId` property.
+    def fill_in_parents(nodes)
+      nodes.tap do |nodes|
+        prev_nodes = empty_level_tracker_hash(11)
+        nodes.each do |node|
+          if parent_node = prev_nodes[node.actual_level - 1]
+            node.parentId = parent_node.id
+          end
+          prev_nodes[node.raw_level] = node
+        end
+      end
+    end
+
+    # Tracker helper (those repeated)
+    def repeated_ids
+      @repeated_ids ||= []
+    end
+
+    # Tracker helper (those done)
+    def done_ids
+      @done_ids ||= []
+    end
+
+    def reset_trackers!
+      @done_ids     = []
+      @repeated_ids = []
+    end
+  end
+end