evoc 3.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dba4bd171ddbca345a90203454afd6285122db56
+  data.tar.gz: 987093521af4703fcc540f665a70125fe9e39716
+SHA512:
+  metadata.gz: 9d3c6d229df7deb55a7bff7afea8dd953e656e4382be8fa6e5a16f4248996fc6401f51ba962466e090742e6c07039b92f7675ca10cf709b79a212e51b824a011
+  data.tar.gz: e1bf42491834eab8d90eb5ed9cac1af0e54f30f74bdaa70c2f518d2eea19e42d23d9ee231767da10a3795038c4549bfe200b6f94f32d9625d369493a83b2d7c0

data/.gitignore

@@ -0,0 +1,15 @@
+evoc.log
+.DS_Store
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*/*.html
+/.history
+/GRTAGS
+*TAGS

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.3
+before_install: gem install bundler -v 1.10.4

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tarma.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Thomas Rolfsnes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Makefile

@@ -0,0 +1,4 @@
+install:
+	bundle install
+	rake install
+

data/README.md

@@ -0,0 +1,61 @@
+# Evoc
+
+Evoc is now using [semantic versioning](http://semver.org/)
+
+## Instructions for installing Ruby
+
+1. install ruby version manager (rvm)
+
+```
+    gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+    curl -sSL https://get.rvm.io | bash -s stable
+    source /home/ubuntu/.rvm/scripts/rvm
+```
+
+2. install ruby
+
+```
+    rvm install ruby-2.2.0
+    rvm --default use ruby-2.2.0
+```
+
+## Instructions for installing Evoc
+
+1. clone the evoc repo
+
+```
+    git clone git@bitbucket.org:evolveit/evoc.git 
+```
+
+2. (linux) install atlas (used for svd, can be skipped)
+
+```
+    sudo apt-get install libatlas-base-dev
+```
+2. (mac) install gcc49 (used for svd, can be skipped)
+
+```
+    brew install gcc49
+```
+
+3. install bundler
+
+```
+    gem install bundler
+```
+
+4. install gem dependencies for evoc (using bundler)
+
+```
+    cd 'evoc folder'
+    bundle install
+```
+
+
+
+5. install evoc
+
+```
+    cd 'evoc folder'
+    rake install
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "evoc_helper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/evoc

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require_relative '../lib/evoc_helper'
+EvocCLI::Main.start(ARGV)

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/evoc.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'evoc/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "evoc"
+  spec.version       = Evoc::VERSION
+  spec.authors       = ["Thomas Rolfsnes"]
+  spec.email         = ["mail@thomasrolfsnes.com"]
+
+  spec.summary       = %q{A collection of algorithms for doing Targeted Association Rule Mining}
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = ["evoc",]
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "ruby-prof"
+  spec.add_runtime_dependency "require_all"
+  spec.add_runtime_dependency "thor"
+  spec.add_runtime_dependency "time_difference"
+  spec.add_runtime_dependency "ruby-progressbar"
+  spec.add_runtime_dependency "rubyzip"
+  spec.add_runtime_dependency "algorithms"
+end

data/lib/evoc/algorithm.rb

@@ -0,0 +1,147 @@
+#file algorithm.rb
+module Evoc
+    class Algorithm
+        extend Logging
+
+        def self.execute(tx_store:,query:,algorithm:)
+          if algorithm.nil?
+            raise ArgumentError.new, "Algorithm was equal to nil" 
+          end
+            # tarmaq2 => call tarmaq(2)
+            if match = /tarmaq(?<num>\d+)/.match(algorithm)
+                Evoc::Algorithm.tarmaq(match[:num].to_i,tx_store:tx_store,query:query)
+            elsif match = /topk_(?<k>\d+)_(?<min>\d+)_(?<max>\d+)/.match(algorithm)
+              Evoc::TopK.topk(k: match[:k].to_i,
+                              min: match[:min].to_i,
+                              max: match[:max].to_i,
+                              tx_store: tx_store,
+                              query: query)
+            elsif match = /crule_range_(?<min>\d+)_(?<max>\d+)/.match(algorithm)
+              Evoc::Algorithm.cached_rule_range(match[:min].to_i,match[:max].to_i,tx_store:tx_store,query:query)
+            elsif match = /rule_range_(?<min>\d+)_(?<max>\d+)/.match(algorithm)
+              Evoc::Algorithm.rule_range(match[:min].to_i,match[:max].to_i,tx_store:tx_store,query:query)
+            elsif Evoc::Algorithm.respond_to?(algorithm+'_algorithm')
+                Evoc::Algorithm.method(algorithm+'_algorithm').call(tx_store:tx_store,query:query)
+            else raise ArgumentError.new, "#{algorithm} is not an available algorithm"
+            end
+        end
+
+        ##
+        # Targeted Association Rule Mining implementations
+        #
+        #
+        
+        def self.rule_cache
+          return @@rule_range_cache
+        end
+        
+        @@rule_range_cache = Hash.new {|h,k| h[k] = Hash.new(&h.default_proc) }
+        def self.cached_rule_range(min,max,tx_store:,query:)
+          tag = tx_store.hash+query.hash
+          rule_store = Evoc::RuleStore.new(query: query)
+          if @@rule_range_cache[tag].empty?
+            @@rule_range_cache.clear
+            Evoc::Algorithm.rule_range(min,max,tx_store:tx_store,query:query).group_by {|r| r.lhs.size}.each do |size,rules|
+              @@rule_range_cache[tag][size] = rules 
+            end
+          else
+            # calculate missing range
+            missing_range = (min..max).to_a - @@rule_range_cache[tag].keys
+            if !missing_range.empty?
+              Evoc::Algorithm.rule_range(missing_range.min,missing_range.max,tx_store:tx_store,query:query).group_by {|r| r.lhs.size}.each do |size,rules|
+                @@rule_range_cache[tag][size] = rules 
+              end
+            end
+          end
+          (min..max).each do |antecedent_size|
+            @@rule_range_cache[tag][antecedent_size].each do |rule|
+              rule_store << rule
+            end
+          end
+          logger.debug  "Algorithm (#{min},#{max}) generated #{rule_store.size} rules for query of size #{query.size}"
+          return rule_store
+        end
+
+        def self.rule_range(min,max,tx_store:,query:)
+          rule_store = Evoc::RuleStore.new(query: query)
+          unique_rules = Set.new
+          # min must be equal or smaller to max
+          if min <= max
+            # can only create rules where the antecedent is equal to or smaller in size than the query
+            if min <= query.size
+              #initial filter, we consider all txes where something in the query changed
+              query_changed_in = tx_store.transactions_of_list(query)
+              # create rules 
+              query_changed_in.each do |tx_id| 
+                tx = tx_store.get_tx(id:tx_id,id_type: :index)
+                # the tx size must be larger than the min size
+                # (so that we have items left for the consequent)
+                if (tx.size > min) 
+                  # CONSTRUCT RULES in the desired range
+
+                  # 1. Find overlap
+                  query_overlap = (query & tx.items)
+                  consequents = (tx.items - query_overlap)
+                  # 2. Ignore exact overlaps (no items left for consequent)
+                  if consequents.size != 0
+                    # 3. Construct rules for overlaps that are at least min in size
+                    if query_overlap.size >= min
+                      # 4. Generate all permutations of query_overlap in the desired range
+                      antecedents = (min..max).map {|size| query_overlap.combination(size).to_a}.flatten(1)
+                      # 5. Add one rule for each antecedent/consequent combination
+                      antecedents.each do |antecedent|
+                        consequents.each do |consequent|
+                          unique_rules << [antecedent,[consequent]]
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+          unique_rules.each {|lhs,rhs| rule_store << Evoc::Rule.new(lhs: lhs,rhs: rhs,tx_store:tx_store)}
+          return rule_store
+        end
+
+
+        ##
+        # TARMAQ
+        # find largest subsets in @query with evidence in @tx_store version
+        def self.tarmaq(depth,tx_store:,query:)
+            largest_match = 0
+            #initial filter, we consider all txes where something in the query changed
+            query_changed_in = tx_store.transactions_of_list(query)
+            # now find what subsets of the query changed in each tx
+            rules = Hash.new
+            query_changed_in.each do |tx_id|
+                tx = tx_store.get_tx(id:tx_id,id_type: :index)
+                largest_match_in_query = (query & tx.items)
+                match_size = largest_match_in_query.size
+                remainder_in_tx = tx.items - largest_match_in_query
+                if remainder_in_tx.size > 0
+                  if match_size > largest_match
+                    largest_match = match_size
+                  end
+                end
+            end
+            # now generate rules 
+            max = largest_match
+            min = ((max - depth) <= 0) ? 1 : max - depth
+            self.cached_rule_range(min,max,tx_store: tx_store,query: query)
+        end
+
+        ###
+        ## rose
+        ###
+        def self.rose_algorithm(tx_store:,query:)
+          qs = query.size
+          self.cached_rule_range(qs,qs,tx_store: tx_store, query: query)
+        end
+
+        def self.co_change_algorithm(tx_store:, query:)
+          self.cached_rule_range(1,1,tx_store: tx_store, query: query)
+        end
+
+    end # Algorithm
+end

data/lib/evoc/algorithms/top_k.rb

@@ -0,0 +1,86 @@
+module Evoc
+  class TopK
+
+    def self.topk(k:,min:,max:,tx_store:,query:)
+      @@k = k
+      @@top_k_rules = Containers::CRBTreeMap.new
+      @@expansion_candidates = Containers::CRBTreeMap.new
+      @@min_support = 0
+      # start with all subsets of size @min in @query
+      query.combination(min).each do |antecedent|
+        # check if this antecedent has min_support > 0
+        changed_in = tx_store.transactions_of_list(antecedent, strict: true)
+        if changed_in.size > 0
+          consequents = Set.new
+          # find all consequents for this antecedent
+          changed_in.each do |tx_id| 
+            tx = tx_store.get_tx(id:tx_id,id_type: :index)
+            # the tx size must be larger than the antecedent size
+            # (so that we have items left for the consequent)
+            if (tx.size > antecedent.size) 
+              (tx.items - query).each {|c| consequents << c}
+            end
+          end
+          consequents.each do |consequent|
+            r = Evoc::Rule.new(lhs:antecedent,rhs: [consequent], tx_store: tx_store)
+            if r.m_support.value.to_f >= @@min_support
+              self.save_rule(r)
+            end
+          end
+        end
+      end
+
+      # now expand antecedents
+      while !@@expansion_candidates.empty? && (@@expansion_candidates.min_key.to_f >= @@min_support)
+        max_candidate = @@expansion_candidates.delete_max
+        #puts "expanding #{max_candidate.name} | #{max_candidate.m_support.value}"
+        expansions = query - max_candidate.lhs
+        # don't make antecedents larger than @max
+        #if (most_frequent_rule.lhs.size < max) & !expansions.empty?
+        if (max_candidate.lhs.size < max) 
+          self.expand(max_candidate,expansions,tx_store)
+        end
+        # remove all rules with sup < min_sup from candidates
+        while !@@expansion_candidates.empty? && (@@expansion_candidates.min_key.to_f < @@min_support)
+          @@expansion_candidates.delete_min
+        end
+      end
+      return Evoc::RuleStore.new(@@top_k_rules.map {|k,rule| rule})
+    end
+
+    def self.expand(rule,expansions,tx_store)
+      expansions.each do |e|
+        # only expand rules with items lexically larger than the lhs
+        if rule.lhs.all? {|i| e > i}
+          expanded_rule = Evoc::Rule.new(lhs: rule.lhs+[e],rhs: rule.rhs,tx_store: tx_store)
+          if expanded_rule.m_support.value > @@min_support
+            self.save_rule(expanded_rule)
+          end
+        end
+      end
+    end
+
+    def self.save_rule(r)
+      key = "#{r.m_support.value.to_f}|#{r.name}"
+        #puts "adding rule #{key}"
+      @@top_k_rules.push(key,r)
+      @@expansion_candidates.push(key,r)
+      if @@top_k_rules.size >= @@k
+        ##puts "K is #{@@k} rules in #{@@top_k_rules.size}"
+        #puts "new rule sup #{r.m_support.value.to_f} min sup is #{@@min_support}"
+        if r.m_support.value > @@min_support
+          #puts "new rule larger than min sup"
+          # remove min sup rules until size is K
+          while (@@top_k_rules.size > @@k) && (@@top_k_rules.min_key.to_f == @@min_support)
+            min_rule = @@top_k_rules.delete_min
+            #puts "ksize #{@@top_k_rules.size} after removing rule #{min_rule}"
+          end
+          @@min_support = @@top_k_rules.min_key.to_f
+          #puts "set min sup to #{@@min_support}"
+        else
+         #puts "#{"%f" % r.m_support.value.to_f.to_s} not larger than #{@@min_support}"
+        end
+      end
+    end
+  end
+end