eco-helpers 2.0.19 → 2.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14260868c76936513a93d4d104eacebbd11e47ed05806d4102ee76196a300d2b
-  data.tar.gz: 35784d03a18f89d2ce8bf5c4105e0eaa647dd10b4e1fee03897319d9ad838760
+  metadata.gz: 5525ed41d4f4b42d96eb0d71f02911e7cfb9112890aff984fb872a60d7515976
+  data.tar.gz: d6dacbda91325cea0867c253b99458ae18b16e6450431ead852d686dd6e56859
 SHA512:
-  metadata.gz: 514d71e93bfa4fb854d9062be03306e154a4dfd184256ab03da30e2bf4bb2a45fb305efd5e2206821e522dc7cc0bfbc69e75285e3c6292dca78f359bd166f52a
-  data.tar.gz: c99a424905916cef61333c18bb31726e90fb9759a4a1b747b72ab123f4bc09ecc6962205a966b3ea46aa1f0a4cc3dd7f72a8c53829cc764a7993d64ad45495bf
+  metadata.gz: 1f4a164e153ac8e3d75bfc7024b2ee7c6c101a1703fb79b58a0d8c8fc31fc9978b6353c2a178a419c94ca22e6b49bf440364706642e7b087a4601f4cd7da3b51
+  data.tar.gz: 9fdd7340a79b853b0dd51676e9ea2ba773928c68928b62d7781a525b2c27c1032ec7da04cae1f57afb7b290649350280d0aa5d0f2e7b91ba8a46cfe685bbeadb

data/CHANGELOG.md

@@ -1,7 +1,33 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
-## [2.0.19] - 2021-05-xx
+## [2.0.21] - 2021-06-0x
+
+### Added
+  - `Eco::CSV::Table`, support to create the table out of an `Array<Hash>`
+    - This opens new methods to transform input Excel file to this data structure and unify input data structures.
+  - **new** use case `Eco::API::UseCases::DefaultCases::CleanUnknownTags` invokable via `clean-unknown-tags`
+
+### Changed
+  - `Eco::API::Common::People::EntryFactory` slight **refactor** to boost better support for multiple input formats
+
+### Fixed
+
+
+## [2.0.20] - 2021-05-31
+
+### Added
+  - **dependencies** to `creek`, `roo` and `roo-xls`
+  - **dependencies** to `hashdiff`
+  - `Eco::API::Session#parse_attribute` => added missing parameter `deps:`
+  - new option `-stdout [file]` to redirect the output to a file
+  - `Eco::CSV::Table`, **added** more helper methods `#group_by`, `#transform_values`, `#slice`, `#slice_columns`, `#delete_column`
+  - `Eco::API::Organization::TagTree` **added** more helper methods: `top?`, `tag=`, `as_json`, `dup`, `diff`
+
+### Fixed
+  - `Exception` patch: when `SystemStackError` there is not `backtrace` :/
+
+## [2.0.19] - 2021-05-31
 
 ### Added
   - Better error message for people searches & **offer** to select among the candidates:

data/eco-helpers.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://www.ecoportal.com"
   spec.licenses      = %w[MIT]
 
-  spec.required_ruby_version = '>= 2.4.4'
+  spec.required_ruby_version = '>= 2.5.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -36,7 +36,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-ses',   '>= 1.36.0', '< 2'
   spec.add_dependency 'dotenv',        '>= 2.7.6', '< 2.8'
   spec.add_dependency 'net-sftp',      '>= 3.0.0', '< 3.1'
+  spec.add_dependency 'hashdiff',      '>= 1.0.1', '< 1.1'
   spec.add_dependency 'fuzzy_match',   '>= 2.1.0', '< 2.2'
   spec.add_dependency 'amatch',        '>= 0.4.0', '< 0.5'
   spec.add_dependency 'jaro_winkler',  '>= 1.5.4', '< 1.6'
+  spec.add_dependency 'roo',           '>= 2.8.3', '< 2.9'
+  spec.add_dependency 'roo-xls',       '>= 1.2.0', '< 1.3'
+  spec.add_dependency 'creek',         '>= 2.5.2', '< 2.6'
 end

data/lib/eco/api/common/loaders/parser.rb

@@ -19,6 +19,7 @@ module Eco
               @attribute = value
             end
 
+            # TODO: it migh rather merge?
             # Some parsers require dependencies to do their job.
             def dependencies(value = nil)
               @dependencies ||= {}

data/lib/eco/api/common/people/entries.rb

@@ -99,6 +99,7 @@ module Eco
             newFrom to_a - discarded
           end
 
+          # TODO: it should rather use the the people-to-csv case somehow
           # Helper to dump the entries into a CSV
           # @param filename [String] the destination file
           def export(filename)

data/lib/eco/api/common/people/entry_factory.rb

@@ -2,7 +2,7 @@ module Eco
   module API
     module Common
       module People
-        # TODO: EntryFactory should suppport multiple schemas itself
+        # TODO: EntryFactory should suppport multiple schemas itself (rather that being done on `Session`)
         # => currently, it's through session.entry_factory(schema: id), but this is wrong
         # => This way, Entries and PersonEntry will be able to refer to attr_map and person_parser linked to schema_id
         # => "schema_id" should be an optional column in the input file, or parsable via a custom parser to scope the schema
@@ -88,26 +88,49 @@ module Eco
             fatal("Format should be a Symbol. Given '#{format}'") if format && !format.is_a?(Symbol)
             fatal("There is no parser/serializer for format ':#{format.to_s}'") unless no_format || @person_parser.defined?(format)
 
-            if file
-              arr_hash = []
-              if Eco::API::Common::Session::FileManager.file_exists?(file)
-                encoding   ||= Eco::API::Common::Session::FileManager.encoding(file)
-                encoding     = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
-                file_content = File.read(file, encoding: encoding)
-                arr_hash     = person_parser.parse(format, file_content).map.each_with_index do |entry_hash, i|
-                  j = (format == :csv)? i + 2 : i + 1
-                  entry_hash.tap {|hash| hash["idx"] = j}
-                end
+            kargs = {}
+            kargs.merge!(content:  data)     unless no_data
+            kargs.merge!(file:     file)     unless no_file
+            kargs.merge!(format:   format)   unless no_format
+            kargs.merge!(encoding: encoding) if encoding
+
+            Entries.new(to_array_of_hashes(**kargs), klass: PersonEntry, factory: self)
+          end
+
+          def to_array_of_hashes(**kargs)
+            data = []
+            content, file, encoding, format = kargs.values_at(:content, :file, :encoding, :format)
+
+            content = get_file_content(file, format, encoding) if file
+
+            case content
+            when !content
+              logger.error("Could not obtain any data out of these: #{kargs}")
+              exit(1)
+            when Hash
+              logger.error("Input data as 'Hash' not supported. Expecting 'Enumerable' or 'String'")
+              exit(1)
+            when String
+              data = person_parser.parse(format, content).map.each_with_index do |entry_hash, i|
+                j = (format == :csv)? i + 2 : i + 1
+                entry_hash.tap {|hash| hash["idx"] = j}
+              end
+              to_array_of_hashes(content: data)
+            when Enumerable
+              sample = content.to_a.first
+              case sample
+              when Hash, Array, ::CSV::Row
+                Eco::CSV::Table.new(content).to_array_of_hashes
               else
-                logger.warn("File does not exist: #{file}")
+                logger.error("Input 'Array' of '#{sample.class}' is not supported.")
               end
-
-              entries(data: arr_hash)
             else
-              Entries.new(data, klass: PersonEntry, factory: self)
+              logger.error("Could not obtain any data out of content: '#{content.class}'")
+              exit(1)
             end
           end
 
+
           # Helper that generates a file out of `data:`.
           # @raise Exception
           #   - if you try to provide `data:` in the wrong format.
@@ -150,6 +173,17 @@ module Eco
 
           private
 
+          def get_file_content(file, format, encoding)
+            unless Eco::API::Common::Session::FileManager.file_exists?(file)
+              logger.error("File does not exist: #{file}")
+              exit(1)
+            end
+            ext        = File.extname(file)
+            encoding ||= Eco::API::Common::Session::FileManager.encoding(file)
+            encoding   = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
+            content    = File.read(file, encoding: encoding)
+          end
+
           def fatal(msg)
             logger.fatal(msg)
             raise msg

data/lib/eco/api/common/version_patches/exception.rb

@@ -2,8 +2,11 @@ class ::Exception
   def patch_full_message
     begin
       msg = []
-      msg << "\n#{backtrace.first} \n#{message} (#{self.class.to_s})"
-      backtrace[1..-1].each_with_index {|bt, i| msg << "#{" "*8}#{i+1}: from #{bt}"}
+      tracing   = backtrace ? backtrace : []
+      tracing   = (self.class == SystemStackError) ? tracing[1..30] : tracing[1..-1]
+      tracing ||= []
+      msg << "\n#{tracing.first} \n#{message} (#{self.class.to_s})"
+      tracing.each_with_index {|bt, i| msg << "#{" "*8}#{i+1}: from #{bt}"}
       msg.join("\n")
     rescue Exception => e
       puts "Something is wrong with 'patch_full_message': #{e}"

data/lib/eco/api/organization/people.rb

@@ -216,8 +216,8 @@ module Eco
         # @!endgroup
 
         # @!group Helper methods
-        def analytics
-          Eco::API::Organization::PeopleAnalytics.new(self.to_a)
+        def similarity
+          Eco::API::Organization::PeopleSimilarity.new(self.to_a)
         end
         # @!endgroup
 

data/lib/eco/api/organization/people_similarity.rb

@@ -13,7 +13,19 @@ module Eco
         # @!group Config
         # @return [String, Proc, nil] the target attribute to be read.
         def attribute=(attr)
-          @attribute ||= "name"
+          @attribute = attr
+        end
+
+        def attribute
+          @attribute ||= :name
+        end
+
+        # Returns the target value to analyse
+        # @param person [Ecoportal::API::V1::Person]
+        def item_value(person)
+          return attr.call(item) if attribute.is_a?(Proc)
+          attr = attribute.to_sym
+          return item.send(attr) if item.respond_to?(attr)
         end
 
         # Define the order or relevant of per user matches
@@ -37,6 +49,16 @@ module Eco
           @threshold ||= 0.15
         end
 
+        # Generates a new object with same config but different base `data`.
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newFrom(data)
+          super(data).tap do |simil|
+            simil.threshold = threshold
+            simil.order     = order
+            simil.attribute = attribute
+          end
+        end
+
         # @!endgroup
 
         # @!group Searchers
@@ -50,36 +72,158 @@ module Eco
           end
         end
 
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def unnamed
+          select do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def named
+          reject do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def blank_attribute
+          select do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` **n0t** empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def attribute_present
+          reject do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
         # @!endgroup
 
-        # @!group Analysers
+        # @!group Analisys starters
 
         # Analyses People bases on `options`
+        # @param needle_read [Proc, Symbol] when the value to read from `needle` object is different to the `:read` (`attribute`).
+        #   This allows to for example, facet `needle.name` (needle_read) against `haystack_item.details[alt_id]` (read).
+        # @param keep_empty [Boolean] to indicate if it should get rid of people with no results (based on threshold)
         # @return [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
-        def analyse(**options)
+        def analyse(needle_read: nil, keep_empty: false, **options)
           options = { read: self.attribute }.merge(options)
+          total = count; i = 1
           each_with_object({}) do |person, results|
-            results[person.id] = find_all_with_score(person, **options)
+            needle_str = needle_read ? item_string(person, needle_read) : nil
+            results[person.id] = find_all_with_score(person, needle_str: needle_str, **options)
+            print_progress("Analysed", total, i)
+            i += 1
+          end.yield_self do |analysed|
+            analysed = clean_empty(analysed) unless keep_empty
+            #puts "... #{analysed.count} results after cleaning empty"
+            analysed
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Results Treatment
+
+        # Gets a new instance object of this class, with only people in results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newSimilarity(analysed)
+          newFrom(people_in_results(analysed))
+        end
+
+        def people_in_results(analysed)
+          analysed.each_with_object([]) do |(id, results), people|
+            related = results.each_with_object([self[id]]) do |result, related|
+              related << result.match
+            end
+            related.each {|person| people << person unless people.include?(person)}
           end
         end
 
+        # Removes from results those that do not have similar entries
+        def clean_empty(analysed)
+          analysed.select do |id, results|
+            !results.empty?
+          end
+        end
+
+        # Helper to do some treatment fo the results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        def with_analysed(analysed, keep_empty: false)
+          analysed.each_with_object({}) do |(id, results), reanalysed|
+            reanalysed[id] = yield(self[id], results)
+          end.yield_self do |reanalysed|
+            reanalysed = clean_empty(reanalysed) unless keep_empty
+            reanalysed
+          end.tap {|out| "with_analysed... returns #{out.count} records"}
+        end
+
         # Launches a reanalyis on `analysed` based on `options`
         # @param analysed [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
-        def re_analyse(analysed, **options)
-          analysed.each_with_object({}) do |(id, results), out|
-            out[id] = results.relevant_results(**options)
+        def rearrange(analysed, **options)
+          with_analysed(analysed) do |person, results|
+            results.relevant_results(**options)
           end
         end
 
-        # @!group Helpers
+        # Reanalyses by using a block to treat the needle and item values
+        def reanalyse(analysed, msg: "Reanalysing", **options, &block)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress(msg, total, i)
+            i += 1
+            recalculate_results(results, &block)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words(analysed, **options)
+          prompt = "Reanalysing by ignoring matching words"
+          reanalyse(analysed, msg: prompt, **options) do |needle_str, item_str, needle, item|
+            self.class.remove_matching_words(needle_str, item_str)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words_old(analysed, **options)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress("Reanalysing by ignoring matching words", total, i)
+            i += 1
+            ignore_same_words_score(results, **options)
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Reporting Helpers
 
         # @return [String] well structured text
-        def analysis(analysed, format: :txt)
+        def report(analysed, format: :txt)
           case
           when format == :txt
             analysed.each_with_object("") do |(id, results), out|
               msg = results.results.map {|r| r.print}.join("\n  ")
-              "'#{self[id].identify}':\n  " + msg
+              out << "#{self[id].identify}:\n  " + msg + "\n"
             end
           end
         end
@@ -91,7 +235,7 @@ module Eco
         def print_analysis(**options)
           analysed = options[:analysed] || results_with_false_positives.analyse(**options)
           analysed.each_with_object({}) do |(id, results), out|
-            puts analysis(analysed)
+            puts report(analysed)
           end
         end
         # @!endgroup
@@ -105,6 +249,22 @@ module Eco
 
         private
 
+        def print_progress(msg, total, num)
+          return unless total > 10
+          puts "" unless num > 1
+          @print_msg_len ||= 0
+          percent = (100 * num.to_f / total).round(1)
+          msg     = "  #{msg}: #{percent}% (#{num} of #{total})\r"
+          @print_msg_len = msg.length unless @print_msg_len > msg.length
+          print msg
+          $stdout.flush
+          if percent > 99.9
+            sleep(0.2)
+            print "#{" " * @print_msg_len}\r"
+            $stdout.flush
+          end
+        end
+
 
       end
     end

data/lib/eco/api/organization/tag_tree.rb

@@ -42,6 +42,39 @@ module Eco
           init_hashes
         end
 
+        # Updates the tag of the current tree
+        def tag=(value)
+          @tag = value
+        end
+
+        # @return [Eco::API::Organization::TagTree]
+        def dup
+          self.class.new(as_json)
+        end
+
+        # @return [Array] with the differences
+        def diff(tagtree, differences: {}, level: 0, **options)
+          require 'hashdiff'
+          Hashdiff.diff(self.as_json, tagtree.as_json, **options.slice(:array_path, :similarity, :use_lcs))
+        end
+
+        def top?
+          depth == -1
+        end
+
+        # @return [Array[Hash]] where `Hash` is a `node` `{"tag" => TAG, "nodes": Array[Hash]}`
+        def as_json
+          nodes_json = nodes.map {|node| node.as_json}
+          if top?
+            nodes_json
+          else
+            {
+              "tag"   => tag,
+              "nodes" => nodes_json
+            }
+          end
+        end
+
         # @return [Boolean] `true` if there are tags in the node, `false` otherwise.
         def empty?
           @has_tags.empty?