evoc 3.8.1 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 085ad4c083e2ac19df50dbecd0a6279d0b5e010c
-  data.tar.gz: 57f950592bad1814599e2394c028ab8c47df8e50
+  metadata.gz: 88a68050e4c0549ae496b8aa53cfdf00c452a596
+  data.tar.gz: 9f51e810f2402c4238f0fd0df127de5df813ea3b
 SHA512:
-  metadata.gz: d8e4b6e718744121e6c50c3c49111c1f679c47f57376d9cca287697e2e04efa22bad6a55f6c29c7beba21e6a10b9f15134f1f539de9249d60a84e38846595a8e
-  data.tar.gz: 05f581eb1a4c8838c0258e920aca701db8400648b7dd8c2ab4a3184327732ad4e67b87bc46c2b860a055f8cece51f81509fea885d2c340d28abe0ece26ba3234
+  metadata.gz: 2d8cb5d438e8299ffb5ad7a8eabf0cd99c3a3dec8951a32e1e7eea476b5b471a7fa02afd04320d56b5d539c20315bfaaf1422436b545dbbbcc32125ee70d0d2a
+  data.tar.gz: bbaee8dcd9dee18adeb8bb070e7aaf33b19225d237026546d6924684518006f400e527d3758f3f9a8a23cb9bc929a6d56a3d371ee9ac46ec2125f3f86cc61419

data/lib/evoc/algorithms/closed_rules.rb

@@ -1,51 +1,59 @@
 module Evoc
   class ClosedRules
+    extend Logging
+
     def self.closed_rules(tx_store:,query:)
       # @@store = tx_store
       # create initial trees, one tree per consequent
       tree = self.initialize_tree(tx_store,query)
-      # puts "INIT TREE:"
-      # tree.print_tree(1,nil,lambda {|node,pre| puts "#{pre} #{@@store.ints2names(node.name.map(&:to_i))}"})
+      logger.debug "INIT TREE:"
+      tree.print_tree(1,nil,lambda {|node,pre| logger.debug "#{pre} #{node.name}"})
       closed_rules = Evoc::RuleStore.new(query: query)
-      tree.children.each do |consequent|
-        self.extend_nodes(consequent).each do |frequency, closed_sets|
+      # tree.children.each do |consequent|
+        self.extend_nodes(tree).each do |frequency, closed_sets|
           closed_sets.each do |closed_set| 
             antecedent = closed_set - consequent.name
             closed_rules << Evoc::Rule.new(lhs: antecedent.map(&:to_i), rhs: consequent.name.map(&:to_i),tx_store: tx_store, m_support: frequency.to_r/tx_store.size)
           end
         end
-      end
+      # end
       return closed_rules
     end
 
     private
     def self.initialize_tree(tx_store, query)
+      rules = Evoc::Algorithm.co_change(tx_store: tx_store, query: query)
       tree = Tree::TreeNode.new([])
-      # find all items that changed with something in the query
-      query_changed_in = tx_store.transactions_of_list(query)
-      # store all items from the query that have changed with each consequent
-      query_changed_in.each do |tx_id|
-        tx = tx_store.get_tx(id:tx_id,id_type: :index)
-        antecedent = (query & tx.items)
-        consequents = (tx.items - antecedent)
-        if consequents.size != 0
-          consequents.each do |consequent|
-            consequent_key = [consequent.to_s]
-            if tree[consequent_key].nil?
-              # initialize candidates
-              tree << Tree::TreeNode.new([consequent.to_s],tx_store.transactions_of(consequent))
-            end
-            txes_consequent = tree[consequent_key].content
-            antecedent.each do |item|
-              union = [item.to_s,consequent.to_s]
-              if tree[consequent_key][union].nil?
-                txes_union = tx_store.transactions_of(item) & txes_consequent
-                tree[consequent_key] << Tree::TreeNode.new(union,txes_union)
-              end
-            end
-          end
-        end
+      rules.each do |rule|
+        txes_union = tx_store.transactions_of(rule.lhs.first) & tx_store.transactions_of(rule.rhs.first)
+        union = [rule.lhs.first.to_s,rule.rhs.first.to_s]
+        tree << Tree::TreeNode.new(union,txes_union)
       end
+      # # find all items that changed with something in the query
+      # query_changed_in = tx_store.transactions_of_list(query)
+      # # store all items from the query that have changed with each consequent
+      # query_changed_in.each do |tx_id|
+      #   tx = tx_store.get_tx(id:tx_id,id_type: :index)
+      #   antecedent = (query & tx.items)
+      #   consequents = (tx.items - antecedent)
+      #   if consequents.size != 0
+      #     consequents.each do |consequent|
+      #       consequent_key = [consequent.to_s]
+      #       if tree[consequent_key].nil?
+      #         # initialize candidates
+      #         tree << Tree::TreeNode.new([consequent.to_s],tx_store.transactions_of(consequent))
+      #       end
+      #       txes_consequent = tree[consequent_key].content
+      #       antecedent.each do |item|
+      #         union = [item.to_s,consequent.to_s]
+      #         if tree[consequent_key][union].nil?
+      #           txes_union = tx_store.transactions_of(item) & txes_consequent
+      #           tree[consequent_key] << Tree::TreeNode.new(union,txes_union)
+      #         end
+      #       end
+      #     end
+      #   end
+      # end
       return(tree)
     end
 
@@ -55,7 +63,7 @@ module Evoc
         a = current_node
         b = a.next_sibling
         while(!b.nil?) do
-          # print "Checking #{@@store.ints2names(a.name.map(&:to_i))}:{#{a.content}} against #{@@store.ints2names(b.name.map(&:to_i))}:{#{b.content}}"
+          logger.debug "Checking #{a.name}:{#{a.content}} against #{b.name}:{#{b.content}}"
           ab = a.name | b.name
           a_txes = a.content
           b_txes = b.content
@@ -67,54 +75,53 @@ module Evoc
           if ab_txes.size > 0
             case self.compare(a_txes,b_txes)
             when 'EQUAL'
-              # puts "  EQUAL"
-              # puts "    removing #{@@store.ints2names(b.name.map(&:to_i))}"
-              # puts "    renaming #{@@store.ints2names(a.name.map(&:to_i))} to #{@@store.ints2names(ab.map(&:to_i))}"
+              logger.debug "  EQUAL"
+              logger.debug "    removing #{b.name}"
+              logger.debug "    renaming #{a.name} to #{ab}"
               temp = b.previous_sibling
               root.remove!(b)
               b = temp
               a.each {|n| n.rename(ab | n.name)}
             when 'A_IN_B'
-              # puts "  A in B"
-              # puts "    renaming #{@@store.ints2names(a.name.map(&:to_i))} to #{@@store.ints2names(ab.map(&:to_i))}"
+              logger.debug "  A in B"
+              logger.debug "    renaming #{a.name} to #{ab}"
               a.each {|n| n.rename(ab | n.name)}
             when 'B_IN_A'
-              # puts "  B in A"
-              # puts "    removing #{@@store.ints2names(b.name.map(&:to_i))}"
-              # puts "    adding child #{@@store.ints2names(ab.map(&:to_i))} to #{@@store.ints2names(a.name.map(&:to_i))}"
+              logger.debug "  B in A"
+              logger.debug "    removing #{b.name}"
+              logger.debug "    adding child #{ab} to #{a.name}"
               temp = b.previous_sibling
               root.remove!(b)
               b = temp
               a << Tree::TreeNode.new(ab,ab_txes)
             when 'NOT_EQUAL'
-              # puts "  NOT EQUAL"
-              # puts "    adding child #{@@store.ints2names(ab.map(&:to_i))} to #{@@store.ints2names(a.name.map(&:to_i))}"
+              logger.debug "  NOT EQUAL"
+              logger.debug "    adding child #{ab} to #{a.name}"
               a << Tree::TreeNode.new(ab,ab_txes)
             end
           end
-          # puts "NEW TREE:"
-          # root.print_tree(1,nil,lambda {|node,pre| puts "#{pre} #{@@store.ints2names(node.name.map(&:to_i))}:#{node.content.size}"})
+          logger.debug "NEW TREE:"
+          root.print_tree(1,nil,lambda {|node,pre| logger.debug "#{pre} #{node.name}:#{node.content.size}"})
           b = b.next_sibling
-          # puts "A siblings #{a.right_siblings.map(&:name).map {|n| @@store.ints2names(n.map(&:to_i))}}"
-          # puts "A next sibling #{@@store.ints2names(a.next_sibling.name.map(&:to_i))}}"
-          # puts "A:#{@@store.ints2names(a.name.map(&:to_i))}, B:#{b.nil? ? nil : @@store.ints2names(b.name.map(&:to_i))}"
+          logger.debug "A next sibling #{b}}"
+          logger.debug "A:#{a.name}, B:#{b.nil? ? nil : b.name}"
         end # siblings.each
         if !a.children.empty?
-          # puts "TRAVERSING DOWN"
+          logger.debug "TRAVERSING DOWN"
           self.extend_nodes(a, closed_rules: closed_rules)
         end
         # add node as closed rule if not subsumed by another rule already added
         rule_frequency = a.content.size
         rule = a.name
         if closed_rules[rule_frequency].nil?
-          # puts "ADDING NEW CLOSED RULE: #{@@store.ints2names(rule.map(&:to_i))}:#{rule_frequency}"
+          logger.debug "ADDING NEW CLOSED RULE: #{rule}:#{rule_frequency}"
           closed_rules[rule_frequency] = [rule] 
         else
           if !closed_rules[rule_frequency].any? {|closed| (rule - closed).empty? }
-            # puts "ADDING NEW CLOSED RULE: #{@@store.ints2names(rule.map(&:to_i))}:#{rule_frequency}"
+            logger.debug "ADDING NEW CLOSED RULE: #{rule}:#{rule_frequency}"
             closed_rules[rule_frequency] << rule
           else
-            # puts "RULE SUBSUMED, NOT ADDING: #{@@store.ints2names(rule.map(&:to_i))}:#{rule_frequency}"
+            logger.debug "RULE SUBSUMED, NOT ADDING: #{rule}:#{rule_frequency}"
           end
         end
         current_node = current_node.next_sibling

data/lib/evoc/experiment.rb

@@ -17,59 +17,78 @@ module Evoc
     end
 
     def sample_transactions
-      STDERR.puts "Sampling transactions.."
       # by default we can sample from the whole history
       sampling_history = Evoc::HistoryStore.base_history
+      STDERR.puts "Sampling transactions from a pool of #{sampling_history.size}.."
       sample = []
+
+      #################################################################################
+      # performing filtering steps on min/max commits size and minimum previous history
+      #################################################################################
+
+      if !self.opts[:recent].nil?
+        size = sampling_history.size
+        sampling_history = sampling_history[[0,size-self.opts[:recent]].max..-1]
+        STDERR.puts "    Filtering to the #{self.opts[:recent]} most recent transactions (new pool size: #{sampling_history.size})"
+      end
       # filter out transactions larger than X
+      if !self.opts[:minimum_commit_size].nil?
+        sampling_history = sampling_history.select {|tx| tx.size >= self.opts[:minimum_commit_size]}
+        STDERR.puts "    Filtering to txes larger than or equal to #{self.opts[:minimum_commit_size]} (new pool size: #{sampling_history.size})"
+      end
       if !self.opts[:maximum_commit_size].nil?
-        STDERR.puts "Only sampling txes smaller than #{self.opts[:maximum_commit_size]}"
-        sampling_history = sampling_history.clone_with_subset(0,sampling_history.size-1,self.opts[:maximum_commit_size])
+        sampling_history = sampling_history.select {|tx| tx.size <= self.opts[:maximum_commit_size]}
+        STDERR.puts "    Filtering to txes smaller than or equal to #{self.opts[:maximum_commit_size]} (new pool size: #{sampling_history.size})"
       end
       # only sample transactions that have at least 'minimum_history' previous history
       if !self.opts[:minimum_history].nil?
-        STDERR.puts "Only sampling txes with at least #{self.opts[:minimum_history]} previous txes (history)"
-        if self.opts[:minimum_history] >= sampling_history.size-1
-          raise ArgumentError, "The history you provided (#{self.opts[:transactions]}), only contains #{sampling_history.size}, not enough to sample with a minimum history set to #{self.opts[:minimum_history]}. Perhaps also #{self.opts[:maximum_commit_size]} must be increased."
+        sampling_history = sampling_history.select {|tx| tx.index >= self.opts[:minimum_history]}
+        STDERR.puts "    Filtering to txes with at least #{self.opts[:minimum_history]} previous txes (new pool size: #{sampling_history.size})"
+      end
+      filtering_switches = [:recent,:minimum_commit_size,:maximum_commit_size,:minimum_history]
+      if filtering_switches.any? {|s| !self.opts[s].nil?}
+        if sampling_history.size == 0
+          STDERR.puts "WARNING: All transactions were filtered out, unable to sample"
+          return []
         end
-        sampling_history = sampling_history.clone_with_subset(self.opts[:minimum_history],sampling_history.size-1)
       end
-      if self.opts[:recent]
-        STDERR.puts "Taking the #{self.opts[:sample_size]} most recent transactions, this overrides any other sampling params apart from maximum_commit_size"
-        txes_larger_than_one = sampling_history.select {|tx| tx.size > 2}
-        sample = txes_larger_than_one.sort_by {|tx| -tx.index}.take(self.opts[:sample_size]).map(&:id)
-      else
-        # group the txes by size
-        groups = sampling_history.group_by {|tx| tx.size}
-        # sort the sample_groups option to reduce the need for maintaining control over which txes that have been sampled
-        # i.e., random sampling is done first, then the sampled txes are removed from the sampling
-        tx_sizes_to_sample_from = self.opts[:sample_groups].sort_by(&:to_s)
-        tx_sizes_to_sample_from.each do |group_size|
-          if group_size == '*'
-            # TODO: > 2 should be generalized to > X
-            txes_larger_than_one = sampling_history.select {|tx| tx.size > 2}.map(&:id)
-            sampled_ids = txes_larger_than_one.sample(self.opts[:sample_size])
-            sample << sampled_ids
-            STDERR.puts "Sampled #{sampled_ids.size} txes from the whole history"
-            # remove sampled txes from sampling_history
-            filtered_hist = sampling_history.reject {|tx| sampled_ids.include? tx.id}
-            sampling_history.clear
-            filtered_hist.each {|tx| sampling_history << tx}
-          elsif group_size.to_i
-            # check if there were any txes of this size
-            if group = groups[group_size.to_i]
-              if group.size < self.opts[:sample_size]
-                logger.warn "Only #{group.size} transactions found of size #{group_size}, asked for #{self.opts[:sample_size]}"
-              end
-              sampled_ids = group.sample(self.opts[:sample_size]).map(&:id)
-              sample << sampled_ids
-              STDERR.puts "Sampled #{sampled_ids.size} txes of size #{group_size}"
-            else
-              logger.warn "No transactions found of size #{group_size}, asked for #{self.opts[:sample_size]} (minimum history: #{self.opts[:minimum_history]})"
+
+      if self.opts[:sample_size] > sampling_history.size 
+        STDERR.puts "WARNING: The sample size is larger than the available transactions"
+      end
+
+      ######################
+      # performing sampling
+      ######################
+
+      # group the txes by size
+      groups = sampling_history.group_by {|tx| tx.size}
+      # sort the sample_groups option to reduce the need for maintaining control over which txes that have been sampled
+      # i.e., random sampling is done first, then the sampled txes are removed from the sampling
+      tx_sizes_to_sample_from = self.opts[:sample_groups].sort_by(&:to_s)
+      tx_sizes_to_sample_from.each do |group_size|
+        if group_size == '*'
+          sampled_ids = sampling_history.map(&:id).sample(self.opts[:sample_size])
+          sample << sampled_ids
+          STDERR.puts "Sampled #{sampled_ids.size} txes"
+          # remove sampled txes from sampling_history
+          filtered_hist = sampling_history.reject {|tx| sampled_ids.include? tx.id}
+          sampling_history.clear
+          filtered_hist.each {|tx| sampling_history << tx}
+        elsif group_size.to_i
+          # check if there were any txes of this size
+          if group = groups[group_size.to_i]
+            if group.size < self.opts[:sample_size]
+              logger.warn "Only #{group.size} transactions found of size #{group_size}, asked for #{self.opts[:sample_size]}"
             end
+            sampled_ids = group.sample(self.opts[:sample_size]).map(&:id)
+            sample << sampled_ids
+            STDERR.puts "Sampled #{sampled_ids.size} txes of size #{group_size}"
           else
-            raise ArgumentError.new, "Tx size for sampling must either be specified by an Integer or '*' (was #{group_size}:#{group_size.class})"
+            logger.warn "No transactions found of size #{group_size}, asked for #{self.opts[:sample_size]} (minimum history: #{self.opts[:minimum_history]})"
           end
+        else
+          raise ArgumentError.new, "Tx size for sampling must either be specified by an Integer or '*' (was #{group_size}:#{group_size.class})"
         end
       end
       sample.flatten.uniq
@@ -135,17 +154,17 @@ module Evoc
           # only specified sizes
           if random_sizes.empty? & !specified_sizes.empty?
             sampled_queries = specified_sizes.map {|s| items.sample(s)}
-            # only random sizes
+          # only random sizes
           elsif !random_sizes.empty? & specified_sizes.empty?
             sampled_queries = random_sizes.map {|s| items.sample(s)}
-            # random + specified = randomly sample in range defined by specified
-            # ex:
-            # specified = [1,3,10,20]
-            # tx size = 4
-            #
-            # 1. remove X in specified that are larger than or equal to 4
-            # 2. randomly select X in specified = Y
-            # 3. randomly select Y in tx
+          # random + specified = randomly sample in range defined by specified
+          # ex:
+          # specified = [1,3,10,20]
+          # tx size = 4
+          #
+          # 1. remove X in specified that are larger than or equal to 4
+          # 2. randomly select X in specified = Y
+          # 3. randomly select Y in tx
           elsif !random_sizes.empty? & !specified_sizes.empty?
             specified_sizes.select! {|s| (s < tx_size) & (s > 0)} #1.
             if randomly_sampled_size = specified_sizes.sample #2.
@@ -231,6 +250,7 @@ module Evoc
       factors = [factor_model_size,factor_max_size,factor_model_age,factor_algorithms,factor_measures,factor_permutation,factor_aggregators].compact
       num_of_scenarios = factors.inject(1) {|product,f| product * f.size}
       invalid_configuration = 0 
+      last_error = 'no errors'
       CSV.foreach(self.opts[:queries], headers: true) do |query|
         # abort if the failsafe file is present 
         if !self.opts[:fail_safe].nil?
@@ -254,7 +274,6 @@ module Evoc
         end
 
         current_scenario = 1
-        last_error = 'no errors'
         # - compact removes nil values (not used factors)
         # - the splat operator '*' turns the array into parameters for #product
         # - the block form of #product makes it lazy (i.e., the whole cartesian product isn't generated at once)

data/lib/evoc/tx_store.rb

@@ -95,8 +95,8 @@ module Evoc
       @txes.last
     end
 
-    def [] index
-      @txes[index]
+    def [] *indexes
+      @txes[*indexes]
     end
 
     def to_s

data/lib/evoc/version.rb

@@ -1,3 +1,3 @@
 module Evoc
-  VERSION = "3.8.1"
+  VERSION = "3.9.0"
 end

data/lib/evoc_cli/experiment.rb

@@ -23,7 +23,8 @@ module EvocCLI
     method_option :sample_size, aliases: "-s", type: :numeric, required: true, desc: "Number of transactions to sample from each group"
     method_option :minimum_history, :aliases => '-m', type: :numeric, desc: "Filter out transactions which has less previous history than this"
     method_option :maximum_commit_size, type: :numeric, desc: "Filter out transactions which are larger than this before sampling"
-    method_option :recent, type: :boolean, desc: "If transactions should be the most recent"
+    method_option :minimum_commit_size, type: :numeric, default: 2, desc: "Filter out transactions which are smaller than this before sampling"
+    method_option :recent, type: :numeric, desc: "Sample in the X most recent transactions"
     method_option :after, :aliases => '-a', :desc => "Only include commits after this date"
     method_option :before, :aliases => '-b', :desc => "Only include commits before this date"
     desc "sample_transactions [OPTIONS]","Make a sample of transactions (from JSON format)"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: evoc
 version: !ruby/object:Gem::Version
-  version: 3.8.1
+  version: 3.9.0
 platform: ruby
 authors:
 - Thomas Rolfsnes
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-15 00:00:00.000000000 Z
+date: 2017-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler