eco-helpers 2.0.18 → 2.0.19

data/lib/eco/cli/config/default/options.rb

@@ -35,7 +35,8 @@ ASSETS.cli.config do |cnf|
       session.schema = sch_id
     end
 
-    desc = "Deprecated: used to be used to specify the input file when using -get-partial"
+    desc  = "Used to be used to specify the input file when using -get-partial. "
+    desc += "It can also be useful to obtain `-get-partial` of people base on `:export` use cases (i.e. -people-to-csv)"
     options_set.add("-entries-from", desc) do |options, session|
       options.deep_merge!(input: {entries_from: true})
     end

data/lib/eco/cli/config/default/usecases.rb

@@ -48,6 +48,8 @@ ASSETS.cli.config do |cnf|
       options.deep_merge!(export: {options: {detailed: true}})
     end.add_option("-permissions-custom", "Used with -detailed. Adds the permissions_custom abilities") do |options|
       options.deep_merge!(export: {options: {permissions_custom: true}})
+    end.add_option("-split-schemas", "It will generate 1 file per each schema") do |options|
+      options.deep_merge!(export: {options: {split_schemas: true}})
     end
 
     desc = "Adds a column 'ecoPortalTag' to the input CSV with the tags that the location codes map to"

data/lib/eco/cli/config/default/workflow.rb

@@ -28,7 +28,10 @@ ASSETS.cli.config do |config|
           cases_with_input =  config.usecases.active(io: io).select do |usecase, data|
             io.class.input_required?(usecase.type)
           end
-          next io unless (!io.input || io.input.empty?) && !cases_with_input.empty?
+
+          input_is_required = !cases_with_input.empty? || io.options.dig(:input, :entries_from)
+          missing_input     = !io.input || io.input.empty?
+          next io unless missing_input && input_is_required
 
           if io.options.dig(:input, :entries_from)
             io = io.new(input: config.input.get(io: io))

data/lib/eco/csv.rb

@@ -18,8 +18,10 @@ module Eco
         kargs = {headers: true,  skip_blanks: true}.merge(kargs)
 
         args = [file].tap do |arg|
-          coding = Eco::API::Common::Session::FileManager.encoding(file)
-          arg.push("rb:bom|utf-8") if coding == "bom"
+          encoding = Eco::API::Common::Session::FileManager.encoding(file)
+          #encoding = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
+          #arg.push(encoding)
+          arg.push("rb:bom|utf-8") if encoding == "bom"
         end
 
         out = super(*args, **kargs).reject do |row|

data/lib/eco/data/fuzzy_match.rb

@@ -27,17 +27,28 @@ module Eco
         include CharsPositionScore
         include NGramsScore
 
-        def jaro_winkler(str1, str2)
+        def jaro_winkler(str1, str2, **options)
           options = {
             ignore_case: true,
             weight:      0.25
-          }
+          }.merge(options)
           JaroWinkler.distance(str1, str2, **options)
         end
 
       end
 
       module InstanceMethods
+        FUZZY_MATCH_OPTIONS = [
+          :identities, :groupings, :stop_words, :read,
+          :must_match_grouping, :must_match_at_least_one_word,
+          :gather_last_result, :threshold
+        ]
+
+        JARO_OPTIONS     = [:ignore_case, :weight]
+        NGRAMS_OPTIONS   = [:range]
+        POSITION_OPTIONS = [:max_distance]
+        RESULTS_OPTIONS  = [:order, :threshold]
+
         include StopWords
 
         attr_accessor :fuzzy_options
@@ -46,19 +57,14 @@ module Eco
           @fuzzy_options ||= {}
         end
 
-        def fuzzy_match(haystack = nil, **options)
-          return @fuzzy_match if instance_variable_defined?(:@fuzzy_match)
-          @fuzzy_options = options.merge({
-            stop_words: PREPOSITIONS + PRONOUNS + ARTICLES
-          })
+        def fuzzy_match(haystack_data = nil, **options)
+          if instance_variable_defined?(:@fuzzy_match) && !haystack_data
+            return @fuzzy_match if fuzzy_match_options == fuzzy_match_options(options)
+          end
+          @fuzzy_options = options
           # make it run with a native C extension (for better performance: ~130 % increase of performance)
           ::FuzzyMatch.engine = :amatch
-          haystack = obtain_haystack(haystack).tap do |items|
-            if !fuzzy_read_method && found = items.find {|item| !item.is_a?(String)}
-              raise "To use non String objects as 'haystack' you should provide `read:` or `options[:read]`. Given element: #{found.class}"
-            end
-          end
-          @fuzzy_match = ::FuzzyMatch.new(haystack, fuzzy_options)
+          @fuzzy_match = ::FuzzyMatch.new(haystack(haystack_data), fuzzy_match_options)
         end
 
         # @note
@@ -71,37 +77,73 @@ module Eco
             unless item == needle
               needle_str = item_string(needle)
               item_str   = item_string(item)
-              jaro_res   = self.class.jaro_winkler(needle_str, item_str)
-              ngram_res  = self.class.ngrams_score(needle_str, item_str, range: 3..5).ratio
-              wngram_res = self.class.words_ngrams_score(needle_str, item_str, range: 3..7).ratio
-              pos_res    = self.class.chars_position_score(needle_str, item_str).ratio
+              jaro_res   = jaro(needle_str, item_str)
+              ngram_res  = ngram(needle_str, item_str)
+              wngram_res = words_ngram(needle_str, item_str)
+              pos_res    = position(needle_str, item_str)
+
               results << Result.new(item, item_str, dice, lev, jaro_res, ngram_res, wngram_res, pos_res)
             end
           end
-          Results.new(needle, item_string(needle), results)
+          Results.new(needle, item_string(needle), results).tap do |res|
+            res.order     = fuzzy_options[:order]     if fuzzy_options[:order]
+            res.threshold = fuzzy_options[:threshold] if fuzzy_options[:threshold]
+          end
         end
 
         private
 
+        def jaro(str1, str2)
+          options = fuzzy_options.slice(*JARO_OPTIONS)
+          self.class.jaro_winkler(str1, str2, **options)
+        end
+
+        def ngram(str1, str2)
+          options = { range: 3..5 }.merge(fuzzy_options.slice(*NGRAMS_OPTIONS))
+          self.class.ngrams_score(str1, str2, **options).ratio
+        end
+
+        def words_ngram(str1, str2)
+          options = { range: 3..7 }.merge(fuzzy_options.slice(*NGRAMS_OPTIONS))
+          self.class.words_ngrams_score(str1, str2, **options).ratio
+        end
+
+        def position(str1, str2)
+          options = fuzzy_options.slice(*POSITION_OPTIONS)
+          self.class.chars_position_score(str1, str2, **options).ratio
+        end
+
         # @note
         #   - When used in an `Enumerable` it will use `to_a`, or `values` if it's a `Hash`
         # @param data [Enumerable, nil]
         # @return [Array<Object>] the non-repeated values of `data`
-        def obtain_haystack(data = nil)
+        def haystack(data = nil)
           data = self if self.is_a?(Enumerable) && !data
           raise "'data' should be an Enumerable. Given: #{data.class}" unless data.is_a?(Enumerable)
           data = self.is_a?(Hash) ? self.values.flatten : to_a.flatten
-          data.uniq.compact
+          data.uniq.compact.tap do |items|
+            if !fuzzy_read_method && found = items.find {|item| !item.is_a?(String)}
+              raise "To use non String objects as 'haystack' you should provide `read:` or `options[:read]`. Given element: #{found.class}"
+            end
+          end
         end
 
         def item_string(item, attr = fuzzy_read_method)
           return item if !item || item.is_a?(String) || !attr
+          return attr.call(item) if attr.is_a?(Proc)
           attr = attr.to_sym
           return item.send(attr) if item.respond_to?(attr)
         end
 
+        def fuzzy_match_options(options = nil)
+          options = fuzzy_options unless options
+          options.slice(*FUZZY_MATCH_OPTIONS).merge({
+            stop_words: PREPOSITIONS + PRONOUNS + ARTICLES
+          })
+        end
+
         def fuzzy_read_method
-          fuzzy_options[:read]
+          fuzzy_match_options[:read]
         end
 
       end

data/lib/eco/data/fuzzy_match/ngrams_score.rb

@@ -16,8 +16,13 @@ module Eco
 
           Score.new(0, 0).tap do |score|
             next if !str2 || !str1
-            next score.increase(score.total) if str1 == str2
-            next if str1.length < 2 || str1.length < 2
+            if str1 == str2
+              score.increase_total(len1)
+              score.increase(score.total)
+            end
+            if str1.length < 2 || str1.length < 2
+              score.increase_total(len1)
+            end
 
             paired_words(str1, str2, normalized: true) do |needle, item|
               ngrams_score(needle, item, range: range, normalized: true)

data/lib/eco/data/fuzzy_match/pairing.rb

@@ -27,7 +27,6 @@ module Eco
         def paired_words(str1, str2, format: [:pair, :score], normalized: false)
           str1, str2 = normalize_string([str1, str2]) unless normalized
           return {} if !str2 || !str1
-          return score.increase(score.total) if str1 == str2
           return {str1 => nil} if str1.length < 2 || str1.length < 2
 
           needles    = get_words(str1, normalized: true)

data/lib/eco/data/fuzzy_match/result.rb

@@ -11,12 +11,18 @@ module Eco
         def words_ngrams; super&.round(3); end
         def chars_position; super&.round(3); end
 
+        def average
+          values = [dice, levenshtein, jaro_winkler, ngrams, words_ngrams, chars_position]
+          (values.inject(0.0, :+) / values.length).round(3)
+        end
+
         # TODO: print in the order of `order`
         def print
           msg  = "(Dice: #{dice}) (Lev Dst: #{levenshtein}) "
           msg << "(Jaro: #{jaro_winkler}) "
           msg << "(Ngram: #{ngrams}) (WNgrams: #{words_ngrams}) "
           msg << "(C Pos: #{chars_position}) "
+          msg << "(Avg: #{average}) "
           msg << "'#{value}'"
         end
 
@@ -37,7 +43,7 @@ module Eco
 
         def order=(values)
           @order = [values].flatten.compact.tap do |o|
-            o = [:words_ngrams, :dice] if o.empty?
+            o << [:words_ngrams, :dice] if o.empty?
           end
         end
 

data/lib/eco/data/fuzzy_match/results.rb

@@ -3,23 +3,29 @@ module Eco
     module FuzzyMatch
       class Results < Struct.new(:needle, :value, :raw_results)
 
+        attr_accessor :threshold
+
         def results_with_false_positives
-          relevant_results(methods: :jaro_winkler, threshold: 0.5)
+          relevant_results(order: :jaro_winkler, threshold: 0.5)
         end
 
-        def relevant_results(methods: order, threshold: 0.5)
+        def relevant_results(**options)
+          options = {order: order, threshold: threshold || 0.5}.merge(options)
           raw_results.select do |result|
-            result.all_threshold?(methods, threshold)
+            result.all_threshold?(options[:order], options[:threshold])
           end.yield_self do |filtered|
             self.class.new(needle, value, filtered).tap do |results|
-              results.order = methods
+              results.order = options[:order]
             end
           end
         end
 
+        # @param values[Array<Symbol>] the algorithms' results it should be ordered by
+        #   * Possible values: `:dice`, `:levenshtein`, `:jaro_winkler`, `:ngrams`, `:words_ngrams`, `:chars_position`, `:average`
         def order=(values)
-          @order = [values].flatten.compact
-          raw_results.each {|r| r.order = @order}
+          @order = [values].flatten.compact.tap do |o|
+            raw_results.each {|r| r.order = o}
+          end
         end
 
         def order

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "2.0.18"
+  VERSION = "2.0.19"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 2.0.18
+  version: 2.0.19
 platform: ruby
 authors:
 - Oscar Segura
@@ -345,6 +345,8 @@ files:
 - lib/eco/api/common/session/base_session.rb
 - lib/eco/api/common/session/environment.rb
 - lib/eco/api/common/session/file_manager.rb
+- lib/eco/api/common/session/helpers.rb
+- lib/eco/api/common/session/helpers/prompt_user.rb
 - lib/eco/api/common/session/logger.rb
 - lib/eco/api/common/session/logger/cache.rb
 - lib/eco/api/common/session/logger/log.rb
@@ -397,7 +399,7 @@ files:
 - lib/eco/api/organization.rb
 - lib/eco/api/organization/login_providers.rb
 - lib/eco/api/organization/people.rb
-- lib/eco/api/organization/people_analytics.rb
+- lib/eco/api/organization/people_similarity.rb
 - lib/eco/api/organization/person_schemas.rb
 - lib/eco/api/organization/policy_groups.rb
 - lib/eco/api/organization/preferences.rb

data/lib/eco/api/organization/people_analytics.rb

@@ -1,60 +0,0 @@
-module Eco
-  module API
-    module Organization
-      class PeopleAnalytics <  Eco::API::Organization::People
-        include Eco::Data::FuzzyMatch
-
-        # @!group Helpers
-
-        # @!endgroup
-
-        # @!group Searchers
-
-        # It gathers those that have the same `email`
-        # @return [Hash] where `keys` are `email`s and `values` an `Array<Person>`
-        def repeated_emails
-          init_caches
-          @by_email.select do |email, people|
-            people.count > 1
-          end
-        end
-
-        # @!endgroup
-
-        # @!group Analysers
-
-        # TODO: Sort results by `results.first.methods`
-        def similarity(**options)
-          each_with_object({}) do |person, results|
-            results[person.id] = find_all_with_score(person, **options)
-          end
-        end
-
-
-        def print_analysis(threshold)
-          similarity.each do |id, results|
-            msg = results.results.select do |result|
-              result.threshold?(threshold)
-            end.map do |result|
-              result.print
-            end.join("\n  ")
-
-            puts "'#{self[id].identify}':\n  " + msg
-          end
-        end
-        # @!endgroup
-
-        protected
-
-        def on_change
-          remove_instance_variable(@fuzzy_match)
-          super
-        end
-
-        private
-
-
-      end
-    end
-  end
-end