evoc 3.5.0

data/lib/evoc/tx_store.rb

@@ -0,0 +1,379 @@
+module Evoc
+  class TxStore
+    include Enumerable,Logging
+    attr_reader :txes, :items, :tx_index_mapping
+    attr_accessor :name_2_int, :int_2_name
+
+    # CLASS VARIABLES
+    @@case_id = nil
+
+    def initialize(path: nil,case_id: nil, granularity: 'mixed')
+      @txes = []
+      @items = Evoc::Env::GOOGLE_HASH ? GoogleHashSparseIntToRuby.new : Hash.new
+      # keeps track of each txs location
+      # in the @txes array using the tx.index variable
+      @tx_index_mapping = Evoc::Env::GOOGLE_HASH ? GoogleHashSparseIntToInt.new : Hash.new
+      # keeps a dictionary of item ids and their full filename
+      # populated when first importing the json file
+      @name_2_int = Evoc::Env::GOOGLE_HASH ? GoogleHashSparseRubyToInt.new : Hash.new  
+      @int_2_name = Evoc::Env::GOOGLE_HASH ? GoogleHashSparseIntToRuby.new : Hash.new
+      if !path.nil?
+        load_transactions(path: path, granularity: granularity)
+      end
+
+      # Initialize class variables
+      @@case_id ||= case_id
+    end
+
+
+    ##
+    # self << tx
+    #
+    # adds <tx> and updates @items with which items are changed in which tx
+    def << tx
+      if tx.respond_to?(:items)
+        if tx.index.nil?
+          tx.index = self.size
+        end
+        ##
+        # BUILD INTEGER REPRESENTATION
+        # internally, items (e.g., files/methods) are stored as unique integers
+        # but a dictionary is kept updated with item -> integer mappings
+        if !tx.items.all? {|i| i.is_a?(Integer)}
+          integer_representation = []
+          tx.items.each do |item|
+            if !self.name_2_int.key?(item)
+              int = self.name_2_int.size
+              self.name_2_int[item] = int
+              self.int_2_name[int] = item
+            end
+            integer_representation << self.name_2_int[item]
+          end
+          tx.items = integer_representation
+        end
+        ##
+        # BUILD ITEM <-> TX MAPPING
+        tx.items.each do |item|
+          if !@items.key?(item)
+            @items[item] = [tx]
+          else
+            @items[item] << tx
+          end
+        end
+        @tx_index_mapping[tx.index] = @txes.size
+        @txes << tx
+      else
+        raise Evoc::Exceptions::NotATransaction.new(tx)
+      end
+    end
+
+    # implementing #each gives us access to all Enumerable methods
+    # select, find_all etc
+    def each &block
+      @txes.each do |tx|
+        if block_given?
+          block.call tx
+        else
+          yield tx
+        end
+      end
+    end
+
+    def first
+      @txes.first
+    end
+
+    def last
+      @txes.last
+    end
+
+    def [] index
+      @txes[index]
+    end
+
+    def to_s
+      history = ""
+      self.txes.reverse.each do |tx|
+        history << tx.items.map {|i| self.int_2_name[i] }.join(',') + "\n"
+      end
+      history
+    end
+
+    ##
+    # clear out the currently loaded transactions
+    def clear
+      @txes.clear
+      @items.clear
+    end
+
+
+    ##
+    # Retrieve a transaction using the given identifier
+    #
+    def get_tx(id:,id_type: :index)
+      tx = nil
+      case id_type
+      when :index
+        raise ArgumentError, "Index must be a Fixnum, #{id} was #{id.class}" unless id.is_a?(Fixnum)
+        if index = @tx_index_mapping[id]
+          tx = @txes[index]
+        end
+      when :id
+        tx = @txes.find {|tx| tx.id == id }
+      end
+      if tx.nil?
+        raise ArgumentError, "No transaction with #{id_type} #{id}"
+      else
+        return tx
+      end
+    end
+
+    # Given an item, find those transactions
+    # where the item has been modified
+    # parameters:
+    # item: the item to check
+    # identifier: how to represent the found transactions, either using :index or :id
+    def transactions_of(item, identifier: :index)
+      # if there are no transactions 
+      # just return an empty list
+      if self.size.zero?
+        txes = []
+      else
+      # get the transactions
+      # return [] if the item cannot be found
+        txes = self.items.key?(item) ? self.items[item] : []
+      end
+      txes.map(&identifier)
+    end
+
+    ##
+    # Returns the relevant transactions of the query
+    # That is: all the transactions where at least one
+    # item from the query were changed
+    #
+    # parameters:
+    # query: a list of items
+    # (optional) strict: if set to true, all the items of the query has had
+    # to be changed in the transaction for it to be included
+    def transactions_of_list(items, strict: false, identifier: :index)
+      if strict
+        items.map {|item| transactions_of(item, identifier: identifier)}.array_intersection
+      else
+        items.map {|item| transactions_of(item, identifier: identifier)}.array_union
+      end
+    end
+
+    # Return the list of items that have changed
+    # with at least one item from the query
+    def relevant_unchanged_items(query)
+      transactions_of_list(query).map {|id| get_tx(id: id).items - query}.array_union
+    end
+
+    def size
+      @txes.size
+    end
+
+
+    ##
+    # read in a JSON file of transactions
+    #
+    # TRANSACTIONS ARE LOADED IN REVERSE ORDER!
+    #
+    # this implies that the oldest transaction gets index 0 in the txes array
+    # and the newest has index txes.size-1
+    # (givent that the json file is sorted from newest/top to oldest/bottom)
+    #
+    # @param [String] path the path to the json history file
+    # @param [Symbol] granularity one of :mixed,:file or :method
+    def load_transactions(path: nil, before: nil, after: nil, granularity: 'mixed')
+      if !path.nil?
+        json = nil
+        if File.extname(path) == '.gz'
+          Zlib::GzipReader.open(path) {|gz|
+            json = gz.read
+          }
+        else
+          json = File.read(path,external_encoding: 'iso-8859-1',internal_encoding: 'utf-8')
+        end
+
+        STDERR.puts "Loading transactions using strategy: #{granularity}"
+        json.lines.reverse.each do |json_line|
+          begin
+            json_object = JSON.parse(json_line)
+            if valid_date?(json_object,before,after)
+              tx = nil
+              id = json_object["sha"]
+              date = json_object["date"]
+              if items = json_object["changes"]
+                if !items.compact.empty?
+                  case granularity
+                  when 'mixed'
+                    tx = Evoc::Tx.new(id: id,date: date,items: items.compact)
+                  when 'file'
+                    # group all items by parsable files, and return only the unique set of filenames
+                    items = items.group_by {|i| /^(?<parsable_file>.+?):/.match(i).to_s }.keys.reject(&:empty?)
+                    tx = Evoc::Tx.new(id: id,date: date,items: items)
+                  when 'method'
+                    # group all items by parsable files, return only the methods and @residuals
+                    items = items.group_by {|i| /^(?<parsable_file>.+?):/.match(i).to_s }                   # group items by parsable files
+                                                                                  .select {|k,v| !k.empty?} # filter out the non-parsable files 
+                                                                                  .values                   # get the methods
+                                                                                  .flatten                  # flatten the list of list of methods 
+                    tx = Evoc::Tx.new(id: id,date: date,items: items)
+                  when 'file_all'
+                    items = items.group_by {|i| /^(?<file>[^:]+?)(?::|\z)/.match(i)[:file].to_s } # group items by file name
+                                                                                    .keys # get the set of files
+                    tx = Evoc::Tx.new(id: id,date: date,items: items)
+                  else
+                    raise ArgumentError.new, "Granularity level must be one of 'mixed', 'file', 'method' or 'file_all', was called with #{granularity}"
+                  end
+                else
+                  logger.warn "#{json["sha"]} \"changes\" field only contained nil value(s)"
+                  next
+                end
+              else
+                logger.warn "#{json["sha"]} did not have a \"changes\" field"
+                next
+              end
+              if tx.nil?
+                logger.warn "#{json["sha"]} could not be converted to a tx"
+                next
+              end
+              if tx.items.empty?
+                logger.warn "#{json["sha"]} with granularity #{granularity} filtered out all artifacts"
+                next
+              end
+              self << tx
+            end
+          rescue JSON::ParserError => e
+            logger.warn e.message
+            next # skip to next line
+          rescue Evoc::Exceptions::NoDateInJsonObject => e
+            logger.warn e.message
+            next
+          end
+        end
+        STDERR.puts "Loaded #{self.size} transactions from #{path}"
+      end
+    end
+
+    ##
+    # a looser version of #between?
+    # we also allow nil comparisons
+    # if both <after> and <before> are nil we consider the date valid
+    def valid_date?(json_object,after,before)
+      if date = json_object["date"]
+        if after.nil? & before.nil?
+          return true
+        elsif !after.nil? & !before.nil?
+          if date.between?(after, before)
+            return true
+          end
+        elsif !after.nil?
+          if date > after
+            return true
+          end
+        elsif !before.nil?
+          if date < before
+            return true
+          end
+        end
+      else
+        raise Evoc::Exceptions::NoDateInJsonObject.new, "#{json_object["sha"]} had no \"date\" field."
+      end
+      return false
+    end
+
+    ##
+    # #get_cloned_subset
+    #
+    # Returns a clone of <self> with transactions equal to the index range defined
+    # by
+    #  from and including <start_index> to and including <stop_index>
+    # also exclude transactions with size larger than <max_size>
+    def clone_with_subset(start_index,stop_index,max_size = nil)
+      clone = TxStore.new
+      clone.name_2_int = self.name_2_int
+      clone.int_2_name = self.int_2_name
+      if start_index.nil? & stop_index.nil? & max_size.nil? then return self end
+      # if only one of start_index and stop_index is provided, raise exception
+      if !start_index.nil? ^ !stop_index.nil?
+        raise ArgumentError.new "You must provide both a start and end index"
+      end
+      # check that its a valid range
+      if range = self.txes[start_index..stop_index]
+        if max_size.nil?
+          range.each do |tx|
+            clone << tx
+          end
+        else
+          range.select {|tx| tx.size <= max_size}.each do |tx|
+            clone << tx
+          end
+        end
+      else
+        raise ArgumentError.new, "#{start_index}..#{stop_index} was not a valid range on tx_store with size #{self.size}"
+      end
+      clone
+    end
+
+    ##
+    # Return a new tx_store containing the specified tx ids
+    def clone_by_indexes(tx_indexes)
+        subset = TxStore.new
+        self.each do |tx|
+            if tx_indexes.include?(tx.index)
+                subset << tx
+            end
+        end
+        return subset
+    end
+
+    ############################################
+    # HELPERS                                  #
+    #                                          #
+    # These are meant to be used by all        #
+    # algorithm implementations if needed.     #
+    # Having these methods here eases testing. #
+    ############################################
+
+
+
+
+    ##
+    # return a (string) json representation of the tx_store
+    def to_json 
+      commits = Hash.new {|h,k| h[k] = Hash.new(&h.default_proc) }
+      self.each do |tx|
+        sha = tx.id
+        commits[sha][:sha]            = sha
+        commits[sha][:date]           = tx.date
+        commits[sha][:index]          = tx.index
+        commits[sha][:changes][:all]  = []
+        tx.items.each {|item| commits[sha][:changes][:all] << item}
+      end
+      # print the commits sorted by index 
+      # but dont include the index in the json as there might be "holes" (after filtering etc)
+      JSON.pretty_generate(commits.sort_by {|id,commit| commit[:index]}.reverse.map {|(_,commit)| commit.tap {|c| c.delete(:index)}})
+    end
+
+    def pretty_print
+      self.txes.reverse.each {|tx| CSV {|row| row << tx.items}}
+    end
+
+    private
+    def parse_date date
+      if !date.nil?
+        begin
+          Time.parse date
+        rescue TypeError => e
+          # something else than string was given as input
+          $stderr.puts "Unable to parse #{date}, error: " + e
+        rescue ArgumentError => e
+          # unable to parse the string for a date
+          $stderr.puts "Unable to parse #{date} for a date, error: " + e
+        end
+      end
+    end
+  end
+end

data/lib/evoc/version.rb

@@ -0,0 +1,3 @@
+module Evoc
+  VERSION = "3.5.0"
+end

data/lib/evoc.rb

@@ -0,0 +1,4 @@
+require 'evoc_helper'
+# dummy module to require other gems to import evoc as: require 'evoc'
+module Evoc
+end

data/lib/evoc_cli/analyze.rb

@@ -0,0 +1,198 @@
+require_relative 'cli_helper'
+
+module EvocCLI
+  class Analyze < Thor
+    class_option :transactions, :aliases => '-t', :type => :string, :required => false, :desc => "Path to change-history"
+    class_option :logger_level, type: :string, default: 'error', desc: "The severity level to output to debug.log"
+    class_option :granularity, type: :string, default: 'mixed', desc: "The granularity level of imported items from history"
+
+
+    method_option :number, aliases: '-n', type: :numeric, default: 1000, desc: "The number of rules to calculate measures for"
+    desc "measure_values","Empirically investigate the range of interestingness measures"
+    def measure_values
+      a = Evoc::Analyze.new(options)
+      a.measure_values
+    end
+
+    desc "measure_ranges","Empirically investigate the range of interestingness measures"
+    def measure_ranges
+      a = Evoc::Analyze.new(options)
+      a.measure_ranges
+    end
+
+    desc "aggregator_range","Generate example output of aggregators given a list of values to be aggregated"
+    def aggregator_range
+      a = Evoc::Analyze.new(options)
+      a.aggregator_range
+    end
+
+    desc "uniqueness [options]","Run uniqueness analysis on TRANSACTIONS"
+    long_desc <<-LONGDESC
+    Run an analysises of how many unique transactions there are in TRANSACTIONS.
+
+    The analysis is based on the order present in TRANSACTIONS. A transaction
+    is considered unique if no previous transactions are a superset of the
+    given transaction.
+    LONGDESC
+    def uniqueness
+      a = Evoc::Analyze.new(options)
+      $stdout.puts a.uniqueness
+    end
+
+    desc "all", "Outputs a collection of stats about the input history"
+    def all
+      Evoc::Analyze.new(options).all
+    end
+
+    desc "avg_changes_per_file", "Output the average number of commits that each file is present in"
+    def avg_changes_per_file
+      a = Evoc::Analyze.new(options)
+      $stdout.puts __method__
+      $stdout.puts a.average_changes_per_file
+    end
+
+    desc "num_commits", "Output the number of commits in the input history"
+    def num_commits
+      a = Evoc::Analyze.new(options)
+      $stdout.puts __method__
+      $stdout.puts a.num_commits
+    end
+
+    desc "time_span [options]","Dump the time span between first and last commits in history"
+    def time_span
+      a = Evoc::Analyze.new(options)
+      unit = 'years'
+      $stdout.puts "#{__method__}_#{unit}"
+      $stdout.puts a.time_span(unit: unit)
+    end
+
+    desc "avg_time_between_commits [options]","Dump the average time between each commit"
+    def avg_time_between_commits
+      a = Evoc::Analyze.new(options)
+      unit = 'hours'
+      $stdout.puts "#{__method__}_#{unit}"
+      $stdout.puts a.average_time_between_commits(unit: unit)
+    end
+
+    desc "num_unique_files [options]","Dump the number of unique files mentioned in the history"
+    def num_unique_files
+      a = Evoc::Analyze.new(options)
+      $stdout.puts __method__
+      $stdout.puts a.num_unique_files
+    end
+
+    method_option :top, :aliases => '-n', :type => :numeric, default: 10, :desc => "How many files to return"
+    desc "file_frequency [options]","Return the X most frequent files"
+    def file_frequency
+      a = Evoc::Analyze.new(options)
+      a.file_frequency
+    end
+
+
+    method_option :group, :aliases => '-g', :type => :boolean, default: false, :desc => "If the commits should be grouped by size"
+    desc "commit_size", "Dumps the commits sizes to stdout"
+    def commit_size
+      a = Evoc::Analyze.new(options)
+      a.commit_size
+    end
+
+    desc "avg_commit_size", "Dumps the average commits size of history"
+    def avg_commit_size
+      a = Evoc::Analyze.new(options)
+      $stdout.puts __method__
+      $stdout.puts a.average_commit_size
+    end
+
+    desc "create_dict", ""
+    def create_dict
+      a = Evoc::Analyze.new(options)
+      a.create_dict
+    end
+
+    desc "recommendation_meta",""
+    def recommendation_meta
+
+      int i = 1
+      ARGF.each_line do |line|
+
+        rec = JSON.parse(line)
+
+        tx_id = rec["tx_id"]
+        algorithm = rec["algorithm"]
+        ap = rec["average_precision1000"]
+        rank = nil
+        if !ap.nil?
+          if ap > 0
+            rat = ap.rationalize
+            # the rank is only valid if we obtained a 1 in the numerator
+            if rat.numerator == 1
+              rank = rat.denominator
+            end
+          end
+        else
+          $stderr.puts "AP was nil for #{tx_id}"
+        end
+        applicable = !rec["rules"].empty?
+        positive_recommendation = !rank.nil?
+        # find the confidence of the correct item
+        expected_outcome = nil
+        confidence_correct = nil
+        rank_one = nil
+        confidence_incorrect = nil
+        discernibility = nil
+        ordered_rec = unique_strongest_consequents(rec["rules"])
+        if ordered_rec.size > 0
+          if positive_recommendation
+            expected_outcome = ordered_rec[rank-1][0]
+            confidence_correct = ordered_rec[rank-1][1]
+          else
+            rank_one = ordered_rec[0][0]
+            confidence_incorrect = ordered_rec[0][1]
+          end
+          discernibility = (ordered_rec.group_by{|(_,confidence)| confidence}.size).to_f/ordered_rec.size
+        end
+
+
+        # build output hash
+        hash = {tx_id: tx_id,
+          algorithm: algorithm,
+          ap: ap,
+          rank: rank,
+          applicable: applicable,
+          positive_recommendation: positive_recommendation,
+          expected_outcome: expected_outcome,
+          confidence_correct: confidence_correct,
+          rank_one: rank_one,
+          confidence_incorrect: confidence_incorrect,
+          discernibility: discernibility}
+        begin
+          puts hash.to_json
+        rescue JSON::GeneratorError => e
+          $stderr.puts "Failed to convert hash to JSON, error was #{e}\n the hash was \n#{hash}\n\n"
+          next
+        end
+      $stderr.print "#{i} lines processed"
+      i += 1
+      end
+    end
+
+    private
+    def unique_strongest_consequents(rules)
+      selected_consequents = Hash.new{|h,k| h[k] = 0}
+      rules.each do |rule|
+        lhs = rule["lhs"]
+        rhs = rule["rhs"]
+        confidence = rule["measures"]["m_confidence"]
+        if !confidence.nil?
+          if confidence.to_r.to_f > selected_consequents[rhs]
+            selected_consequents[rhs] = confidence.to_r.to_f
+          end
+        else
+          $stderr.puts "Confidence was nil for rule #{lhs} -> #{rhs}"
+        end
+      end
+      return selected_consequents.sort_by {|k,v| -v}
+    end
+  end
+end
+

data/lib/evoc_cli/cli_helper.rb

@@ -0,0 +1 @@
+require_relative '../evoc_helper'

data/lib/evoc_cli/experiment.rb

@@ -0,0 +1,78 @@
+require_relative 'cli_helper'
+
+module EvocCLI
+  class Experiment < Thor
+    class_option :case_id, type: :string, desc: "Specify case identifier."
+    class_option :logger_level, type: :string, default: 'error', desc: "The severity level to output to debug.log"
+    class_option :transactions, :aliases => '-t', :type => :string, :required => true, :desc => "Path to change-history"
+    class_option :granularity, type: :string, default: 'mixed', desc: "The granularity level of imported items from history"
+
+    ##
+    # sample_transactions
+    method_option :sample_groups, aliases: "-g", type: :array, required: true, desc: "What tx size groups to sample from"
+    method_option :sample_size, aliases: "-s", type: :numeric, required: true, desc: "Number of transactions to sample from each group"
+    method_option :minimum_history, :aliases => '-m', type: :numeric, desc: "Filter out transactions which has less previous history than this"
+    method_option :maximum_commit_size, type: :numeric, desc: "Filter out transactions which are larger than this before sampling"
+    method_option :after, :aliases => '-a', :desc => "Only include commits after this date"
+    method_option :before, :aliases => '-b', :desc => "Only include commits before this date"
+    desc "sample_transactions [OPTIONS]","Make a sample of transactions (from JSON format)"
+    def sample_transactions
+      e = Evoc::Experiment.new(options)
+      STDOUT.puts 'tx_id'
+      STDOUT.puts e.sample_transactions
+    end
+
+    ##
+    # generate_queries
+    #
+    method_option :transaction_ids_path, :aliases => '-i', :type => :string, :required => true, :desc => "Path to file with \\n separated list of transaction ids"
+    method_option :random_select, type: :boolean, default: false, desc: "Randomly select a query from each given transaction"
+    method_option :select, aliases: '-s', type: :array, default: [],
+      :desc => "Number of items to select for each query"
+    method_option :reverse_select, aliases: '-r', type: :array,
+      desc: "Reverse version of --select (select \"all but\" X)"
+    method_option :percentage, aliases: '-e', type: :array,
+      desc: "Percentage of items to select for each query"
+    method_option :filter_duplicates, aliases: '-d', type: :boolean, desc: "Remove identical queries (same id/algorithm/items/model_size/max_size)"
+    method_option :filter_expected_outcome, aliases: '-n', type: :boolean, desc: "Remove new files from the expected outcome"
+    desc "generate_queries [options]", "Generate queries from <transactions>"
+    def generate_queries
+      #MemoryProfiler.start('create_queries',30)
+      e = Evoc::Experiment.new(options)
+      e.generate_queries
+      #MemoryProfiler.stop
+    end
+
+    ##
+    # execute_scenarios
+    #
+    # input: csv of queries from #create_queries
+    # output: query_id, algorithm, average_precision
+    method_option :algorithms, type: :array, default: ['tarmaq0','rose','co_change'], desc: "Which algorithms to use"
+    method_option :measures, type: :array, default: ['support','confidence','support,confidence'], desc: "Which measures to calculate for generated rules."
+    method_option :aggregators, aliases: '-a', type: :array, desc: "Which aggregators to use"
+    method_option :model_size, type: :array, desc: "How many previous transactions to include. 0 = all previous transactions."
+    method_option :model_age, type: :array, desc: "The number of commits between history and query."
+    method_option :max_size, :aliases => '-m', type: :array, desc: "Transactions of size larger than this will be excluded for rule mining"
+    method_option :queries, aliases: '-q', type: :string, required: true, desc: "Path to queries"
+    method_option :permutation, aliases: '-p', type: :numeric,
+      desc: "DEPRECATED WILL HAVE NO EFFECT    Number of query permutations/replications to produce."
+    method_option :fail_safe, type: :string, desc: "If the fail safe file exists, safely exit."
+    method_option :evaluators, aliases: '-e', type: :array, enum: ['average_precision'], required: false, desc: "Methods for evaluating the recommendations"
+    desc "execute_scenarios [options]",""
+    def execute_scenarios
+      if !options[:permutation].nil?
+        STDERR.puts "Permutation option has been set, but the option is currently disabled and will have no effect"
+      end
+      #MemoryProfiler.start('execute_scenarios',30)
+      e = Evoc::Experiment.new(options)
+      e.execute_scenarios
+      #MemoryProfiler.stop
+    end
+
+
+    desc "util SUBCOMMAND [options]", "Various helper functions"
+    subcommand "util", Util
+
+  end
+end

data/lib/evoc_cli/info.rb

@@ -0,0 +1,22 @@
+require_relative 'cli_helper'
+
+module EvocCLI
+  class Info < Thor
+
+    desc "measures","Prints the currently implemented interestingness measures"
+    def measures
+      STDOUT.puts Evoc::InterestingnessMeasures.measures.map {|m| m.to_s.sub("m_","")}.join(" ") 
+    end
+
+    desc "measure_range","Prints the range of available interestingness measures"
+    def measure_range
+      $stdout.puts "measures,range"
+      Evoc::InterestingnessMeasures.measures.sort.each do |m|
+        min      = Evoc::InterestingnessMeasures.get_min(m)
+        max      = Evoc::InterestingnessMeasures.get_max(m)
+        range = "[#{min},#{max}]"
+        $stdout.puts "#{m},\"#{range}\""
+      end
+    end
+  end
+end