eco-helpers 2.0.16 → 2.0.22

data/lib/eco/data/fuzzy_match/result.rb

@@ -0,0 +1,87 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Result < Struct.new(:match, :value, :needle_value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
+        ALL_METHODS = [:dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position]
+
+        attr_accessor :pivot
+
+        def dice; super&.round(3); end
+        def levenshtein; super&.round(3); end
+        def jaro_winkler; super&.round(3); end
+        def ngrams; super&.round(3); end
+        def words_ngrams; super&.round(3); end
+        def chars_position; super&.round(3); end
+
+        #Shortcuts
+        def lev; levenshtein; end
+        def jaro; jaro_winkler; end
+        def wngrams; words_ngrams; end
+        def pos; chars_position; end
+
+        def average
+          values = [dice, levenshtein, jaro_winkler, ngrams, words_ngrams, chars_position]
+          (values.inject(0.0, :+) / values.length).round(3)
+        end
+
+        # TODO: print in the order of `order`
+        def print
+          msg  = "(Dice: #{dice}) (Lev Dst: #{levenshtein}) "
+          msg << "(Jaro: #{jaro_winkler}) "
+          msg << "(Ngram: #{ngrams}) (WNgrams: #{words_ngrams}) "
+          msg << "(C Pos: #{chars_position}) "
+          msg << "(Avg: #{average}) "
+          msg << "'#{value}'"
+        end
+
+        def all_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.all? {|method| threshold?(method, threshold)}
+        end
+
+        def any_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.any? {|method| threshold?(method, threshold)}
+        end
+
+        def threshold?(method = :dice, threshold = 0.15)
+          raise "Uknown method '#{method}'" unless self.respond_to?(method)
+          self.send(method) >= threshold
+        end
+
+        def order=(values)
+          @order = [values].flatten.compact.tap do |o|
+            o << [:words_ngrams, :dice] if o.empty?
+          end
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        def <=>(result)
+          compare(result)
+        end
+
+        def values_at(*keys)
+          keys.map do |key|
+            self.send(key) if self.respond_to?(key)
+          end
+        end
+
+        private
+
+        def compare(other, order: self.order)
+          return 0  unless method = order.first
+          raise "Uknown method '#{method}'"  unless self.respond_to?(method) && other.respond_to?(method)
+          return -1 if self.send(method) > other.send(method)
+          return  1 if self.send(method) < other.send(method)
+          compare(other, order: order[1..-1])
+        end
+
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/results.rb

@@ -0,0 +1,77 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Results < Struct.new(:needle, :value, :raw_results)
+        include Enumerable
+
+        attr_accessor :threshold
+
+        def empty?
+          count < 1
+        end
+
+        def each(&block)
+          return to_enum(:each) unless block
+          raw_results.each(&block)
+        end
+
+        # Merges the results of both Results object
+        def merge(res)
+          unless self.needle == res.needle
+            raise "To merge 2 Results, needle should be the same ('#{value}'). Given '#{res.value}'"
+          end
+          self.class.new(needle, value, raw_results.concat(res.raw_results))
+        end
+
+        def results_with_false_positives
+          relevant_results(order: :jaro_winkler, threshold: 0.5)
+        end
+
+        def relevant_results(**options)
+          options = {order: order, threshold: threshold || 0.5}.merge(options)
+          raw_results.select do |result|
+            result.all_threshold?(options[:order], options[:threshold])
+          end.yield_self do |filtered|
+            self.class.new(needle, value, filtered).tap do |results|
+              results.order = options[:order]
+            end
+          end
+        end
+
+        # @param values[Array<Symbol>] the algorithms' results it should be ordered by
+        #   * Possible values: `:dice`, `:levenshtein`, `:jaro_winkler`, `:ngrams`, `:words_ngrams`, `:chars_position`, `:average`
+        def order=(values)
+          @order = [values].flatten.compact.tap do |o|
+            raw_results.each {|r| r.order = o}
+          end
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        def results
+          raw_results.sort
+        end
+
+        def print
+          msg = results.map do |result|
+            result.print
+          end.join("\n  ")
+
+          puts "'#{value}':\n  " + msg
+        end
+
+        private
+
+        def item_string(item, attr = self.method)
+          return item if !item || item.is_a?(String) || !attr
+          attr = attr.to_sym
+          return item.send(attr) if item.respond_to?(attr)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/score.rb

@@ -0,0 +1,49 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Score < Struct.new(:score, :total)
+
+        def ratio(decimals = 6)
+          tot = self.total; sc = self.score
+          tot = tot && tot > 0 ? tot : 1
+          sc  = sc  && sc  > 0 ? sc  : 0
+          (sc.to_f / tot).round(decimals)
+        end
+
+        def percent(decimals = 3)
+          (100 * ratio).round(decimals)
+        end
+
+        def increase(value = 1)
+          self.score += value
+          raise "Score #{self.score} (increase: #{value}) can't be greater than total #{self.total}" if self.score > self.total
+          self.score
+        end
+
+        def increase_total(value)
+          self.total += value
+        end
+
+        def values_at(*keys)
+          keys.map do |key|
+            self.send(key) if self.respond_to?(key)
+          end
+        end
+
+        # Merges 2 Score instance objects
+        def merge(scr)
+          Score.new(*values_at(:score, :total)).merge!(scr)
+        end
+
+        def merge!(scr)
+          raise "Expecting Score object. Given: #{scr.class}" unless scr.is_a?(Score)
+          increase_total(scr.total)
+          increase(scr.score)
+          self
+        end
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/stop_words.rb

@@ -0,0 +1,35 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module StopWords
+        PREPOSITIONS = [
+          "aboard", "about", "above", "across", "after", "against", "along", "amid", "among", "around", "as", "at",
+          "before", "behind", "below", "beneath", "beside", "between", "beyond", "but", "by",
+          "concerning", "considering", "despite", "down", "during", "except", "following", "for", "from",
+          "in", "inside", "into", "like", "minus", "near", "next",
+          "of", "off", "on", "onto", "opposite", "out", "outside", "over", "past", "per", "plus",
+          "regarding", "round", "save", "since", "than", "through", "till", "to", "toward",
+          "under", "underneath", "unlike", "until", "up", "upon", "versus", "via",
+          "with", "within", "without"
+        ]
+        PRONOUNS = [
+          "all", "another", "any", "anybody", "anyone", "anything", "as", "aught",
+          "both", "each", "each other", "either", "enough", "everybody", "everyone", "everything",
+          "few", "he", "her", "hers", "herself", "him", "himself", "his", "I", "idem", "it", "its", "itself",
+          "many", "me", "mine", "most", "my", "myself", "naught", "neither", "no one", "nobody", "none", "nothing", "nought",
+          "one", "one another", "other", "others", "ought", "our", "ours", "ourself", "ourselves",
+          "several", "she", "some", "somebody", "someone", "something", "somewhat", "such", "suchlike",
+          "that", "thee", "their", "theirs", "theirself", "theirselves", "them", "themself", "themselves", "there",
+          "these", "they", "thine", "this", "those", "thou", "thy", "thyself", "us",
+          "we", "what", "whatever", "whatnot", "whatsoever", "whence", "where", "whereby", "wherefrom",
+          "wherein", "whereinto", "whereof", "whereon", "wherever", "wheresoever", "whereto", "whereunto",
+          "wherewith", "wherewithal", "whether", "which", "whichever", "whichsoever", "who", "whoever", "whom",
+          "whomever", "whomso", "whomsoever", "whose", "whosever", "whosesoever", "whoso", "whosoever",
+          "ye", "yon", "yonder", "you", "your", "yours", "yourself", "yourselves"
+        ]
+        ARTICLES = ["a", "an", "the"]
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/string_helpers.rb

@@ -0,0 +1,82 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module StringHelpers
+        # Downcases and trims
+        def normalize_string(value)
+          case value
+          when Array
+            value.map {|val| normalize_string(val)}
+          when Symbol
+            normalize_string(value.to_sym)
+          when String
+            value.downcase.strip
+          end
+        end
+
+        def get_words(str, normalized: false)
+          return [] unless str
+          str = normalize_string(str) unless normalized
+          str.scan(/[a-zA-Z'-]+/).compact
+        end
+
+        # Keeps the start order of the `words` and consecutive `words` together/consecutive.
+        # @param str [String] the input string with the words.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] combinations of `range` length of `words`.
+        def string_ngrams(str, range=2..3, normalized: false)
+          ngrams(get_words(str, normalized: normalized), range)
+        end
+
+        # Keeps the start order of the `words` of the input `Array` `words`.
+        # It does **not** keep consecutive `words` together (it can jump/skip items).
+        # @param str [String] the input string with the words.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] combinations of `range` length of `words`
+        def string_combinations(str, range=2..3, normalized: false)
+          combinations(get_words(str, normalized: normalized), range)
+          .map {|comb| comb.join(' ')}
+        end
+
+        # It includes `combinations` that break the initial order of the `Array`.
+        # It does **not** keep consecutive `words` together (it can jump/skip items).
+        # @param str [String] the input string with the words.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] permutations of `range` length of `words`
+        def string_permutations(str, range=2..3, normalized: false)
+          permutations(get_words(str, normalized: normalized), range)
+          .map {|comb| comb.join(' ')}
+        end
+
+        # Keeps the start order of the `charts` and consecutive `charts` together/consecutive.
+        # @param str [String] the input `word` string.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] combinations of `range` length of `words`.
+        def word_ngrams(str, range=2..3, normalized: false)
+          str = normalize_string(str) unless normalized
+          ngrams(str.to_s.chars, range)
+          .map {|comb| no_blanks(comb)}
+        end
+
+        def no_blanks(str)
+          return nil unless str && str.is_a?(String)
+          str.tr(' ', '')
+        end
+
+        # Deletes the words of `str1` and `str2` that match
+        # @return [Array<String>] pair of words.
+        def remove_matching_words(str1, str2, normalized: false)
+          unless normalized
+            str1 = normalize_string(str1)
+            str2 = normalize_string(str2)
+          end
+          return [str1, str2] if !str1 || !str2 || str1.empty? || str2.empty?
+          ws1 = get_words(str1)
+          ws2 = get_words(str2)
+          [(ws1 - ws2).join(" "), (ws2 - ws1).join(" ")]
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "2.0.16"
+  VERSION = "2.0.22"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 2.0.16
+  version: 2.0.22
 platform: ruby
 authors:
 - Oscar Segura
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.11
+        version: 2.2.17
     - - "<"
       - !ruby/object:Gem::Version
         version: '2.3'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.11
+        version: 2.2.17
     - - "<"
       - !ruby/object:Gem::Version
         version: '2.3'
@@ -116,7 +116,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.2
+        version: 0.8.3
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -126,7 +126,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.2
+        version: 0.8.3
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -136,7 +136,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.6
+        version: 0.8.7
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -146,7 +146,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.6
+        version: 0.8.7
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -230,6 +230,126 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: hashdiff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: fuzzy_match
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: amatch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: jaro_winkler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: roo-xls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 description: 
 email:
 - oscar@ecoportal.co.nz
@@ -272,6 +392,7 @@ files:
 - lib/eco/api/common/people/default_parsers/policy_groups_parser.rb
 - lib/eco/api/common/people/default_parsers/select_parser.rb
 - lib/eco/api/common/people/default_parsers/send_invites_parser.rb
+- lib/eco/api/common/people/default_parsers/xls_parser.rb
 - lib/eco/api/common/people/entries.rb
 - lib/eco/api/common/people/entry_factory.rb
 - lib/eco/api/common/people/person_attribute_parser.rb
@@ -285,6 +406,8 @@ files:
 - lib/eco/api/common/session/base_session.rb
 - lib/eco/api/common/session/environment.rb
 - lib/eco/api/common/session/file_manager.rb
+- lib/eco/api/common/session/helpers.rb
+- lib/eco/api/common/session/helpers/prompt_user.rb
 - lib/eco/api/common/session/logger.rb
 - lib/eco/api/common/session/logger/cache.rb
 - lib/eco/api/common/session/logger/log.rb
@@ -318,7 +441,9 @@ files:
 - lib/eco/api/microcases/people_load.rb
 - lib/eco/api/microcases/people_refresh.rb
 - lib/eco/api/microcases/people_search.rb
-- lib/eco/api/microcases/refresh_abilities.rb
+- lib/eco/api/microcases/preserve_default_tag.rb
+- lib/eco/api/microcases/preserve_filter_tags.rb
+- lib/eco/api/microcases/preserve_policy_groups.rb
 - lib/eco/api/microcases/refresh_default_tag.rb
 - lib/eco/api/microcases/s3upload_targets.rb
 - lib/eco/api/microcases/set_account.rb
@@ -335,13 +460,13 @@ files:
 - lib/eco/api/organization.rb
 - lib/eco/api/organization/login_providers.rb
 - lib/eco/api/organization/people.rb
+- lib/eco/api/organization/people_similarity.rb
 - lib/eco/api/organization/person_schemas.rb
 - lib/eco/api/organization/policy_groups.rb
 - lib/eco/api/organization/preferences.rb
 - lib/eco/api/organization/preferences_reference.json
 - lib/eco/api/organization/presets_factory.rb
 - lib/eco/api/organization/presets_integrity.json
-- lib/eco/api/organization/presets_reference.json
 - lib/eco/api/organization/presets_values.json
 - lib/eco/api/organization/tag_tree.rb
 - lib/eco/api/policies.rb
@@ -376,8 +501,10 @@ files:
 - lib/eco/api/usecases/base_io.rb
 - lib/eco/api/usecases/default_cases.rb
 - lib/eco/api/usecases/default_cases/abstract_policygroup_abilities_case.rb
+- lib/eco/api/usecases/default_cases/analyse_people_case.rb
 - lib/eco/api/usecases/default_cases/append_usergroups_case.rb
 - lib/eco/api/usecases/default_cases/change_email_case.rb
+- lib/eco/api/usecases/default_cases/clean_unknown_tags_case.rb
 - lib/eco/api/usecases/default_cases/codes_to_tags_case.rb
 - lib/eco/api/usecases/default_cases/create_case.rb
 - lib/eco/api/usecases/default_cases/create_details_case.rb
@@ -390,7 +517,6 @@ files:
 - lib/eco/api/usecases/default_cases/new_id_case.rb
 - lib/eco/api/usecases/default_cases/new_id_case0.rb
 - lib/eco/api/usecases/default_cases/org_data_convert_case.rb
-- lib/eco/api/usecases/default_cases/refresh_abilities_case.rb
 - lib/eco/api/usecases/default_cases/refresh_case.rb
 - lib/eco/api/usecases/default_cases/reinvite_sync_case.rb
 - lib/eco/api/usecases/default_cases/reinvite_trans_case.rb
@@ -447,6 +573,16 @@ files:
 - lib/eco/data/files/directory.rb
 - lib/eco/data/files/file_pattern.rb
 - lib/eco/data/files/helpers.rb
+- lib/eco/data/fuzzy_match.rb
+- lib/eco/data/fuzzy_match/array_helpers.rb
+- lib/eco/data/fuzzy_match/chars_position_score.rb
+- lib/eco/data/fuzzy_match/ngrams_score.rb
+- lib/eco/data/fuzzy_match/pairing.rb
+- lib/eco/data/fuzzy_match/result.rb
+- lib/eco/data/fuzzy_match/results.rb
+- lib/eco/data/fuzzy_match/score.rb
+- lib/eco/data/fuzzy_match/stop_words.rb
+- lib/eco/data/fuzzy_match/string_helpers.rb
 - lib/eco/data/mapper.rb
 - lib/eco/language.rb
 - lib/eco/language/curry.rb
@@ -473,7 +609,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.4
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="