evoc 3.5.0

data/lib/evoc/logger.rb

@@ -0,0 +1,34 @@
+# enable logging in classes through 'include Logging'
+module Logging
+  def logger
+    @logger ||= Logging.logger_for(self.class.name)
+  end
+
+  # Use a hash class-ivar to cache a unique Logger per class:
+  @loggers = {}
+  @logger_level = 'info'
+
+  class << self
+    def logger_for(classname)
+      @loggers[classname] ||= configure_logger_for(classname)
+    end
+
+    def configure_logger_for(classname)
+      logger = Logger.new('evoc.log','daily')
+      logger.progname = classname
+      logger.level = const_get('Logger::'+@logger_level.upcase)
+      logger
+    end
+
+    def set_level(level)
+      possible_levels = %w(debug info warn error info)
+      if possible_levels.include?(level)
+        STDERR.puts "Logging level has been set to '#{level}' for output to evoc.log"
+        @loggers.each {|l| l.level = const_get('Logger::'+level.upcase)}
+        @logger_level = level
+      else
+        STDERR.puts "Unable to set logger level to #{level}, possible values are #{possible_levels}. Defaulting to 'info'."
+      end
+    end
+  end
+end

data/lib/evoc/memory_profiler.rb

@@ -0,0 +1,43 @@
+require 'objspace'
+
+module Kernel
+  def tick_every sec, &pr
+    Thread.new do loop do
+      pr.call
+      t = Time.now.to_f
+      frac = t.modulo(sec.to_f)
+      sleep(sec - frac)
+    end end
+  end
+end
+
+module MemoryProfiler 
+
+  @@thread = nil
+
+  ##
+  # @param [string] tag tag to add to dumped files
+  # @oaram [int] n dump every n seconds
+  def self.start(tag,n)
+    dump_dir = 'mem_dumps'
+    index = "#{dump_dir}/#{tag}-index.txt"
+    ObjectSpace.trace_object_allocations_start
+    if !Dir.exists?(dump_dir)
+      Dir.mkdir dump_dir
+    end
+    File.open(index,"w")
+    @@thread = tick_every(n) do
+      GC.start
+      i = Time.now.strftime('%Y-%m-%dT%H:%M:%S')
+      dump = "#{tag}-#{i}.dump"
+      dump_path = "#{dump_dir}/#{dump}"
+      ObjectSpace.dump_all(output: open(dump_path, "w"))
+      File.open(index,'a') {|index| index.puts "#{dump_path},#{i}" }
+    end
+  end
+
+  def self.stop
+    @@thread.kill
+  end
+
+end

data/lib/evoc/recommendation_cache.rb

@@ -0,0 +1,152 @@
+module Evoc
+    class RecommendationCache
+        extend Logging
+
+        # create accessors for class level instance variables
+        # 
+        # tag: the string representation of the currently cached recommendation
+        # recommendation: the currently cached recommendation
+        # time: the time it took to generate the currently cached recommendation
+        # model_size: the number of transactions used when generating the currently cached recommendation
+        class << self
+            attr_accessor :tag, :base_recommendation, :last_recommendation, :time_rulegeneration, :time_measurecalculation, :time_aggregation, :time_evaluation, :filtered_model_size, :evaluation
+        end
+
+        def self.recommendation_cached?(algorithm:,
+                                    query:,
+                                    model_start:,
+                                    model_end:,
+                                    max_size: nil)
+            return self.tag == [algorithm,query,model_start,model_end,max_size].hash
+        end
+
+        def self.get_recommendation(algorithm:,
+                                    query:,
+                                    model_start:,
+                                    model_end:,
+                                    max_size: nil,
+                                    aggregator: nil,
+                                    measures: [])
+          # check if a new base recommendation needs to be generated
+            tag = [algorithm,query,model_start,model_end,max_size].hash
+            if self.tag != tag
+                # clear out any evaluation done
+                self.evaluation = Hash.new
+                # new recommendation
+                logger.debug "Caching new recommendation: algorithm: #{algorithm}, query: #{query}, model_start/end: #{model_start} - #{model_end}, maxsize: #{max_size}"
+                self.tag = tag
+                tx_store = Evoc::HistoryStore.get_history(model_start,
+                                                             model_end,
+                                                             max_size)
+                self.filtered_model_size = tx_store.size
+
+                t1 = Time.new
+                self.base_recommendation = Evoc::Algorithm.execute(tx_store: tx_store,
+                                                             query: query,
+                                                             algorithm: algorithm)
+                self.last_recommendation = self.base_recommendation
+                t2 = Time.new
+                self.time_rulegeneration = TimeDifference.between(t1,t2).in_seconds.round(8)
+            end
+
+            # calculate measures on rules
+            t1 = Time.new
+            self.base_recommendation.calculate_measures(measures)
+            t2 = Time.new
+            self.time_measurecalculation = TimeDifference.between(t1,t2).in_seconds.round(8)
+
+            # perform aggregation
+            if !aggregator.nil?
+              t1 = Time.new
+              self.last_recommendation = self.base_recommendation.aggregate_by(aggregator: aggregator.to_sym,measures: measures) {|r| r.rhs}
+              t2 = Time.new
+              self.time_aggregation = TimeDifference.between(t1,t2).in_seconds.round(8)
+            else
+              self.last_recommendation = self.base_recommendation
+            end
+            return self.last_recommendation
+        end
+
+        ##
+        # Evaluate the currently cached recommendation
+        #
+        # @param [Array<String>] evaluators the evaluators to apply
+        # @param [Array<String>] expected_outcome the expected outcome to use in evaluations
+        # @param [Array<String>] measure_combinations the list of measures to use when sorting a recommendation before evaluating
+        #
+        # @return [Hash[aggregator][evaluator][result]] the hash of results
+        def self.evaluate(evaluators: ,expected_outcome:,measure_combination: )
+          if !self.last_recommendation.nil?
+            t1 = Time.new
+            evaluators.each do |evaluator|
+              self.evaluation[evaluator] = self.last_recommendation.evaluate_with(evaluator: evaluator,expected_outcome: expected_outcome,measure_combination: measure_combination)
+            end
+            t2 = Time.new
+            self.time_evaluation = TimeDifference.between(t1,t2).in_seconds.round(8)
+          else
+            STDERR.puts "TAG = #{self.tag}No recommendation to evaluate"
+          end
+        end
+
+        ##
+        # format:
+        #   {
+        #       time: 'execution time',
+        #       filtered_model_size: 
+        #       number_of_rules :
+        #       evaluation: {
+        #           average_precision: ..,
+        #           ..next evaluator..
+        #       }
+        #       rules: [
+        #         {
+        #           lhs: [lhs]
+        #           rhs: [rhs],
+        #           measures: {
+        #             measure_1: value,
+        #             measure_n: value
+        #           }
+        #         },
+        #         ..next rule..
+        #   ]
+        #   }
+        #
+        # measures: the interestingness measures that you want to output in the hash
+        def self.to_h(measures: Evoc::Rule.measures)
+            recommendation_hash = Hash.new
+            recommendation_hash[:recommendation_tag] = self.tag
+            recommendation_hash[:time_rulegeneration] = self.time_rulegeneration
+            recommendation_hash[:time_measurecalculation] = self.time_measurecalculation
+            recommendation_hash[:time_aggregation] = self.time_aggregation
+            recommendation_hash[:time_evaluation] = self.time_evaluation
+            recommendation_hash[:filtered_model_size] = self.filtered_model_size
+            recommendation_hash[:number_of_baserules] = self.base_recommendation.size
+            recommendation_hash[:number_of_rules] = self.last_recommendation.size
+            recommendation_hash[:aggregator] = self.last_recommendation.aggregator
+            recommendation_hash[:number_of_hyper_rules] = self.last_recommendation.number_of_hyper_rules
+            recommendation_hash[:mean_hyper_coefficient] = self.last_recommendation.inject(0.0){ |sum, r| 
+              sum + r.get_measure('m_hyper_coefficient').value } / self.last_recommendation.size
+            recommendation_hash[:largest_antecedent] = self.last_recommendation.largest_antecedent
+            if !self.evaluation.nil?
+                self.evaluation.each do |evaluator,value|
+                    recommendation_hash[evaluator.to_sym] = value[:value]
+                    recommendation_hash[:unique_consequents] = value[:unique_consequents]
+                end
+            end
+            recommendation_hash[:rules] = []
+            self.last_recommendation.each do |rule|
+                rule_hash = Hash.new
+                rule_hash[:lhs] = rule.lhs.is_a?(String) ? rule.lhs : rule.lhs.join(',')
+                rule_hash[:rhs] = rule.rhs.is_a?(String) ? rule.rhs : rule.rhs.join(',')
+                rule_hash[:measures] = Hash.new
+                measures.each do |m| 
+                    if rule.measure_instantiated?(m)
+                        rule_hash[:measures][m] = rule.get_measure(m).value
+                    end
+                end
+                recommendation_hash[:rules] << rule_hash    
+            end
+            return recommendation_hash
+        end
+    end
+end

data/lib/evoc/rule.rb

@@ -0,0 +1,32 @@
+module Evoc
+  class Rule
+    include InterestingnessMeasures, Logging, Comparable
+    attr_accessor :lhs, :rhs, :name, :tx_store 
+
+    def initialize(lhs:,rhs:,tx_store: nil,**measures)
+      self.lhs = lhs.is_a?(Array) ? lhs.sort : [lhs]
+      self.rhs = rhs.is_a?(Array) ? rhs.sort : [rhs]
+      self.name = "#{self.lhs.join(",")} -> #{self.rhs.join(",")}" 
+      self.tx_store = tx_store
+      measures.each do |measure,value|
+        set_measure(measure,value)
+      end
+    end
+
+    def <=> other
+     other.name <=> self.name
+    end
+
+    def to_s
+      name
+    end
+
+    def lhs=input
+      input.is_a?(Array) ? @lhs = input : @lhs = [input]
+    end
+
+    def rhs=input
+      input.is_a?(Array) ? @rhs = input : @rhs = [input]
+    end
+  end
+end

data/lib/evoc/rule_store.rb

@@ -0,0 +1,340 @@
+module Evoc
+  class RuleStore
+    include Enumerable, Logging
+    attr_accessor :query, :rules, :aggregator
+
+    def initialize(rules = [],query: nil,aggregator: nil)
+      self.rules = rules
+      self.query = query
+      self.aggregator = aggregator
+    end
+    
+    ## 
+    # CLASS METHODS
+    ## 
+    
+    def self.parse_file(path_to_rules)
+        rule_store = Evoc::RuleStore.new
+        CSV.foreach(path_to_rules, :headers => true) do |row|
+          params = row.to_h.symbolize_keys.convert_values(except: [:lhs,:rhs], converter: Evoc::InterestingnessMeasures::VALUE_TYPE)
+          rule = Evoc::Rule.new(params)
+          rule_store << rule
+        end
+        rule_store
+    end
+
+    def self.parse_string(string)
+        rule_store = Evoc::RuleStore.new
+        CSV.parse(string, :headers => true) do |row|
+          params = row.to_h.symbolize_keys.convert_values(except: [:lhs,:rhs], converter: Evoc::InterestingnessMeasures::VALUE_TYPE)
+          rule = Evoc::Rule.new(params)
+          rule_store << rule
+        end
+        rule_store
+    end
+
+    ##
+    # \CLASS METHODS
+    ##
+
+    # required by Enumerable
+    def each &block
+      @rules.each do |rule|
+        if block_given?
+          block.call rule
+        else
+          yield rule
+        end
+      end
+    end
+
+    def [] index
+      @rules[index]
+    end
+
+    def []=(index,value)
+      @rules[index] = value
+    end
+
+    def group_by(&block)
+      res = Hash.new { |hash, key| hash[key] = [] }
+      each do |e|
+        res[block.call(e)] << e
+      end
+      res
+    end
+
+    ##
+    # Calculates the requested measures on the current rule set 
+    # @param measures [Array<Symbol>] the set of measures to calculate
+    def calculate_measures(measures)
+      if measures.nil?
+        raise ArgumentError.new, "Tried to calculate measures, but list of measures was 'nil'"
+      else
+        self.each do |rule|
+          measures.each do |m|
+            rule.get_measure(m)
+          end
+        end
+      end
+    end
+
+    ##
+    # Aggregates the current set of rules using the given aggregator over the rule clusters specified by the given block
+    #
+    # @param: [Symbol] aggregator the name of a defined aggregator function
+    # @param: [Array<String>] measures the measures to aggregate
+    # @param: [block] define the rules clusters which should be aggregated
+    def aggregate_by(aggregator: ,measures:,&block)
+        rule_clusters = group_by(&block)
+        # remove clusters with only one item
+        #aggregatable_rules = rule_clusters.select {|g,cluster| cluster.size > 1}
+        #non_aggregatable_rules = rule_clusters.select {|g,cluster| cluster.size == 1}
+        aggregation = Evoc::RuleStore.new(query: self.query, aggregator: aggregator)
+        # aggregate the rules in each cluster
+        rule_clusters.each do |_,cluster|
+            # we create one aggregated rule from each rule cluster
+          if cluster.size > 1
+            aggregation << Evoc::HyperRule.new(cluster,aggregator,measures)
+          else
+            aggregation << cluster.first
+          end
+        end
+        return aggregation
+    end
+
+    ##
+    # @return the number of hyper rules in this store
+    def number_of_hyper_rules
+      self.hyper_rules.size
+    end
+
+    ##
+    # @return the hyper rules in the store
+    def hyper_rules
+      self.select {|r| r.is_a?(Evoc::HyperRule)}
+    end
+
+
+    ##
+    # Evaluate this recommendation using the given evaluator
+    #
+    # Note that the hyper coefficient is added as a last tie breaker for
+    # aggregation functions called with '*aggregator*_hc'
+    # Not pretty, sorry..
+    #
+    # @param [String] evaluator the method to use for evaluating
+    # @param [Array] expected_outcome the list of items to evaluate against
+    # @param [Array] measure_combination the list of measures used to first sort the recommendation
+    def evaluate_with(evaluator: :average_precision,expected_outcome:,measure_combination: )
+      if measure_combination.empty? then raise ArgumentError, "Cannot evalute a recommendation without specifying which measures to rank on" end
+      result = Hash.new 
+      logger.debug "#{__method__} params: evaluator: #{evaluator}, measure_combination: #{measure_combination}"
+      # sort the rules on each combination and evaluate 
+      sorted_rules = []
+      if self.aggregator =~ /_hc\z/
+        sorted_rules = self.sort_on(measures: measure_combination+['m_hyper_coefficient'])
+      elsif !self.aggregator.nil?
+        sorted_rules = self.sort_on(measures: measure_combination)
+      else
+        # not an aggregation
+        # get the strongest unique rules
+        unique_rules = self.unique_by(measure_combination.first)
+        sorted_rules = self.sort_on(rules: unique_rules,measures: measure_combination)
+      end
+      # get the recommended items
+      recommendation = sorted_rules.map(&:rhs)
+      # evaluate the sorted list against the expected outcome 
+      result[:value] = Evoc::Evaluate.execute(recommendation,expected_outcome,evaluator)
+      return result
+    end
+    
+    ##
+    # Sort rules on one or more measures
+    # If a measure is undefined/nil for a rule, we treat it as -infinity
+    # for purposes of sorting
+    # @param: [Array<String>] measures the list of measures to sort by
+    def sort_on(rules: self, measures:)
+      rules.sort_by {|r| measures.map {|m| r.get_measure(m).value.nil? ? Float::INFINITY : -r.get_measure(m)}}
+    end
+
+    ##
+    # returns the set of unique consequents
+    # where each consequent is the strongest given by the input measure
+    #
+    # @param: [String] measure the measure used to find the strongest rules
+    def unique_by(measure)
+      selected_rules = Evoc::Env::GOOGLE_HASH ? GoogleHashSparseIntToRuby.new : Hash.new
+      self.each do |rule|
+        if !rule.get_measure(measure).value.nil?
+	key = rule.rhs.first
+          if selected_rules[key].nil?
+            selected_rules[key] = Evoc::Env::GOOGLE_HASH ? GoogleHashSparseRubyToRuby.new : Hash.new
+            selected_rules[key][:value] = rule.get_measure(measure).value
+            selected_rules[key][:rule] = rule
+          end
+          if rule.get_measure(measure).value > selected_rules[key][:value]
+            selected_rules[key][:value] = rule.get_measure(measure).value
+            selected_rules[key][:rule] = rule
+          end
+        end
+      end
+      return selected_rules.values.map {|k,v| k[:rule]}
+    end
+
+    ##
+    # @return [Integer] the size of the largest rule, measures by antecedent size
+    def largest_antecedent
+      self.map {|r| r.lhs.size}.max
+    end
+
+    ##
+    # @return [True/False/Nil] if the lhs of one of the rules is equal to the query
+    def exact_match
+      match = nil
+      if !self.query.nil?
+        match = false
+        self.each do |rule|
+          if (rule.lhs.sort == self.query.sort)
+            match = true
+            break
+          end
+        end
+      else
+        logger.debug "Tried to calculate exact match, but query was nil "
+      end
+      return match
+    end
+
+
+    def << rule
+      self.rules << rule
+    end
+
+    def pretty_print
+      CSV.generate do |csv|
+        # write header
+        defined_measures = []
+        if aggregator.nil?
+          defined_measures = self.map {|r| r.instantiated_measures}.array_union
+        else
+          defined_measures = self.hyper_rules.map {|r| r.instantiated_measures}.array_union
+        end
+        csv << ['rule'] + defined_measures
+        self.each do |rule|
+          row = CSV::Row.new([],[],false)
+          row << rule.name
+          defined_measures.each do |m| 
+            row << rule.get_measure(m).value
+          end
+          csv << row
+        end
+      end
+    end
+
+    def to_s
+      CSV.generate do |csv|
+        # write header
+        csv << ['lhs','rhs'] + Evoc::Rule.measures
+        self.each do |rule|
+          row = CSV::Row.new([],[],false)
+          row << (rule.lhs.respond_to?(:join) ? rule.lhs.join(',') : rule.lhs)
+          row << (rule.rhs.respond_to?(:join) ? rule.rhs.join(',') : rule.rhs)
+          Evoc::Rule.measures.each do |m| 
+            row << (rule.measure_instantiated?(m) ? rule.get_measure(m).value : nil)
+          end
+          csv << row
+        end
+      end
+    end
+
+    def print(measures = Evoc::Rule.measures)
+      CSV {|row| row << ['lhs','rhs'] + measures}
+      if self.size > 0
+        name_mapping = self.first.tx_store.int_2_name
+        self.sort_on(measures: measures).each do |rule|
+          row = CSV::Row.new([],[],false)
+          row << rule.lhs.map{|i| name_mapping[i]}.join(',')
+          row << rule.rhs.map{|i| name_mapping[i]}.join(',')
+          measures.each do |m| 
+            row << (rule.measure_instantiated?(m) ? rule.get_measure(m).value : nil)
+          end
+          CSV {|r| r << row}
+        end
+      end
+      nil
+    end
+
+
+    ##
+    # Print the current rule set to a csv file
+    # @param measures [Array<String>] the measures to include in output. Default is all measures.
+    # @param file [String] the file to write to.
+    def print_to_file(measures: Evoc::Rule.measures, file:)
+      CSV.open(file, "wb") do |csv| 
+        # write header
+        csv << ['lhs','rhs'] + measures
+        self.each do |rule|
+          row = CSV::Row.new([],[],false)
+          row << rule.lhs.join(',')
+          row << rule.rhs.join(',')
+          measures.each do |m| 
+            row << (rule.measure_instantiated?(m) ? rule.get_measure(m).value : nil)
+          end
+          csv << row
+        end
+      end
+    end
+
+    def ==other
+      ( self.map(&:lhs)  == other.map(&:lhs) ) &
+      ( self.map(&:rhs)  == other.map(&:rhs) ) &
+      Evoc::Rule.measures.each do |m|
+        self.map {|r| r.get_measure(m)} == other.map {|r| r.get_measure(m)}
+      end
+    end
+
+    def size
+      self.rules.size
+    end
+
+    def empty?
+      self.rules.empty?
+    end
+
+    def clear
+      self.rules.clear
+    end
+
+    def to_h
+        if self.rules.nil?
+            {}
+        else
+            self.rules.map {|r| 
+                h = Hash.new
+                h[:lhs] = r.lhs
+                h[:rhs] = r.rhs
+                r.instantiated_measures.each {|m| h[m] = r.get_measure(m).value.to_r}
+                h}
+        end
+    end
+
+    def instance_values_for_csv
+      dont_include = ['rules']
+      self.instance_values.delete_if {|k,v| dont_include.include?(k)}
+    end
+
+    ##
+    # generate an array suitable for a csv header
+    def csv_header
+      self.instance_values_for_csv.keys
+    end
+
+    ##
+    # generate an array of the current values of <self>
+    # converts any array values to a comma separated string representation
+    def to_csv_row
+      self.instance_values_for_csv.values.map {|val| val.is_a?(Array) ? val.join(',') : val}
+    end
+  end
+end