tyler-collaborative_filter 0.5.0

data/README.textile

@@ -0,0 +1,109 @@
+h1. CollaborativeFilter
+
+Introduction goes here.
+
+h2. Example
+
+<pre>
+  <code>
+    CollaborativeFilter.filter(:logger => LOGGER) do |cf|
+      # We're using a low threshold settings because we're using a content booster
+      # for post processing.  We keep it low so the content booster has plenty of
+      # data to work with.  There is a second threshold in the content booster that
+      # is set high.
+      #
+      # The threshold of the content booster is set at 4.2, with a factor of 0.3
+      # and a single gene.  So the lowest recommendation which could theoretically
+      # make it through the content booster would be:
+      #   
+      #   4.2 - (5 - crossover) * factor =
+      #   4.2 - (5 - 2.5) * 0.3 = 3.45
+      #
+      # So we set it to 3.45.  Anything lower than that will never be recommended.
+      
+      cf.recommender CollaborativeFilter::SimplestRecommender, :threshold => 3.45, :max_per_user => 90
+
+
+      cf.output :type => :sql, :options => { :table_name => 'recommendations',
+                :mapping => { :user_id => :customer_id, :score => :relevance } }
+
+      cf.content_booster do |cb|
+        cb.booster CollaborativeFilter::SimpleBooster
+
+        # The crossover determines what we consider to be positive or negative.
+        # With a crossover of 2.5, a rating of 3 counts as +0.5, whereas a 
+        # rating of 1 counts as -1.5.
+        cb.crossover 3
+
+        # The threshold is a quality control that determines what we allow to be
+        # output.  A threshold of 4.2 means we won't recommend anything that we
+        # think the user will rate lower than 4.2.
+        cb.threshold 4.2
+
+        # The factor determines how much weight we give to content.  A factor of
+        # 1 would be "full weight".  In other words, if your average Superhero
+        # rating is -2, we will subtract 2 from all Superhero recommendations
+        # before thresholding them again.  Whereas a factor of 0.5 would mean
+        # the same person would only have 1 substracted from the Superhero 
+        # recommendations.
+        cb.factor 0.8
+
+        # Genes determine content.  You can have multiple of these blocks.  Please
+        # be sure to knock the factor down for each gene, as they all count
+        # independently.
+        cb.gene :genres do |items|
+          items.map do |(id,type)|
+            i = type.constantize.find(id)
+            next i.genres.map(&:id) if i.respond_to?(:genres)
+            next [i.genre.id] if i.respond_to?(:genre) && i.genre
+            []
+          end
+        end
+      end
+ 
+
+      cf.dataset :ratings do |ds|
+        ds.correlator CollaborativeFilter::SimpleSvd
+        
+        # The cosine_similarity option determines what the minimum cosine similarity
+        # should be between two users to consider them similar.  The
+        # max_similar_users option determines the maximum number of users we'll
+        # user for Determining recommendations.  20 seems like a decent number for
+        # this.  Be aware that changing this will significantly impact how long it
+        # takes to run.
+        ds.options :cosine_similarity => 0.96, :max_similar_users => 20
+        
+        ds.users Rating.find(:all).map(&:customer).uniq
+        ds.items Rating.find(:all).map(&:rateable).uniq
+        ds.nodes do |m|
+          Rating.find(:all).each do |r|
+            score = r.not_interested ? 0.1 : r.score
+            m[ds.item_index(r.rateable_id, r.rateable_type), ds.user_index(r.customer_id)] = score
+          end
+        end
+      end
+
+      cf.dataset :purchases do |ds|
+        ds.correlator CollaborativeFilter::SimpleSvd
+        ds.options :cosine_similarity => 0.985, :max_similar_users => 20
+
+        ds.users Order.find(:all).map(&:customer).uniq
+        ds.items LineItem.find(:all).map(&:product).uniq
+        ds.nodes do |m|
+          Order.find(:all).each do |o|
+            o.line_items.each do |li|
+              next if li.cancelled?
+              m[ds.item_index(li.product_id), ds.user_index(o.customer_id)] = 5
+            end
+          end
+        end
+      end
+    end
+  </code>
+</pre>
+
+
+
+Many thanks to SmartFlix for letting me spend an inordinate amount of time on this.
+
+Copyright (c) 2008 Tyler McMullen, released under the GPL license

data/Rakefile

@@ -0,0 +1 @@
+require 'rake'

data/lib/boosters/simple_booster.rb

@@ -0,0 +1,137 @@
+# SimpleBooster - a content booster for CollaborativeFiltration
+#
+# The purpose of a content booster is to improve the purely collaborative output
+# from a recommender. Collaborative filtration relies on the idea that if you
+# have similar ratings/purchases as someone else in the past, you are likely to
+# continue to in the future.  
+#
+# The fallacy of this is obvious when you consider that Bob may really enjoy 
+# Superhero comics and Horror comics.  Joe really enjoys Superhero comics and 
+# Humor comics.  Depending on how many things they rate and other factors, Joe 
+# and Bob may still have a high correlation. However, Bob's love of Horror 
+# comics may infiltrate Joe's ratings despite the fact that Joe really dislikes 
+# them.
+#
+# So, a content booster allows us to nudge the value of your recommendations up
+# or down. One strategy for using a content booster is to set the threshold on
+# your recommender low (say... 2.5 or 3), but the threshold on the content
+# booster high.
+class CollaborativeFilter
+  class SimpleBooster
+    def run(recs,datasets,genes,options)
+      @recs, @datasets, @genes, @options = recs, datasets, genes, options
+      generate_profiles
+    end
+
+    def generate_profiles
+      all_items = @datasets.inject([]) { |o,(dn,ds)| o.concat ds.items; o }.uniq
+      gene_lists = @genes.map { |gn,gene| gene.all(all_items) }
+
+      # Generating a profile for each user.
+      # In essence ...
+      # user_id => [ {'superhero' => -1, 'horror' => +2 },
+      #              {'spiderman' => 2, 'atomic robo' => 1 } ]
+      #
+      # Iterate through each dataset, as we take all of them into account.
+      CollaborativeFilter.log "  Generating user_profs: #{Time.now}"
+      user_profs = @datasets.inject({}) { |profiles,(ds_name,ds)|
+        CollaborativeFilter.log "   Starting new dataset: #{Time.now}"
+
+        ds.users.each_with_index do |user_id,user_idx|
+          profiles[user_id] ||= []
+
+          # Grab the User's ratings from their column in the input matrix
+          user_ratings = ds.m.col(user_idx).to_a
+          user_ratings.each_index do |item_idx|
+
+            # user_ratings is an array with an entry for each item for the user
+            score = user_ratings[item_idx]
+
+            next if score == 0
+
+            # we have a master list of all items in all datasets which we need
+            # for the profiles to span datasets.  Find this item's index in there.
+            all_items_idx = all_items.index(ds.items[item_idx])
+
+            # iterate through each gene type (genres, franchises, etc)
+            gene_lists.each_index do |gene_type_idx|
+
+              # find the value of the gene for this particular item (e.g. this item's genre is horror)
+              # this value is always an array and can contain more than one value
+              gis = gene_lists[gene_type_idx][all_items_idx]
+
+              profiles[user_id][gene_type_idx] ||= {}
+
+              adj = (score - @options[:crossover]) / gis.size
+
+              gis.each do |gi|
+                # we keep a tuple for each gene value (genre => horror) the first element
+                # is the count of how many items we've noted and last is the total adjustment
+                # they are later used to make an average
+                profiles[user_id][gene_type_idx][gi] ||= [0,0]
+                profiles[user_id][gene_type_idx][gi][0] += 1
+                profiles[user_id][gene_type_idx][gi][1] += adj
+              end
+            end
+          end
+        end
+        profiles
+      }.to_a.map { |user_id,genes| 
+        # Grab each of those tuples we made above ([count, total]) and turn each one into an
+        # average multiplied by the 'factor' option.  Meaning...  If you rated Superman 2 points
+        # above the crossover, and Spiderman 1 point above the threshold, we have a tuple that
+        # looks like [2,3].  3 / 2 = 1.5.  Then we multiply by the factor (say 0.5) meaning we
+        # only want to half weight on the factors.  So, on average you've rated Superheros 1.5
+        # above the crossover, however, since our factor is 0.5, we're going to record 0.75 as
+        # the modifier.  This limits the power of the content booster.
+        #
+        # The more genes you have the lower you'll want to set the factor, as each of them modify
+        # the recommendations in turn.  I should probably change the factor to be configurable
+        # per gene.
+        [user_id, genes.map { |m| m.to_a.map { |gi,(qty,tot)| [gi, (tot/qty) * @options[:factor] ] } } ]
+      }
+
+      CollaborativeFilter.log "  Boosting recommendations: #{Time.now}"
+
+      new_recs = []
+      @recs.each_index do |user_idx|
+        # Grab a user's raw recs and their profile which we generated above
+        user_id, user_recs = @recs[user_idx]
+        user_id, user_profile = user_profs[user_idx]
+
+        # Iterate through each of the individual items in the recommendations
+        new_user_recs = user_recs.map { |item_id, score| 
+
+
+          user_profile.each_index do |gene_type_idx|
+            # Grab this item's genes for this particular gene type from the master list
+            item_gene = gene_lists[gene_type_idx][all_items.index(item_id)]
+
+            # item_gene will always be an array, if it's empty we can move on
+            next if item_gene.empty?
+
+            # an item can have multiple genes for a gene type, we just use the average
+            item_mod = item_gene.inject([0,0]) { |o,g| 
+
+              # find the user's modifier for this gene
+              mod = user_profile[gene_type_idx].detect { |ig| ig.first == g }
+              next o unless mod
+              [o[0] + mod.last, o[1] + 1]
+            }
+            # move on unless we have at least modifier
+            next unless item_mod[1] > 0
+            score += item_mod[0] / item_mod[1]
+          end
+
+          # if the score is at or above the threshold, add it to our new recs list
+          next if score < @options[:threshold]
+          [item_id, score > 5 ? 5 : score]
+        }.compact
+        new_recs << [user_id, new_user_recs]
+      end
+
+      new_recs
+    end
+  end
+end
+

data/lib/collaborative_filter.rb

@@ -0,0 +1,43 @@
+require 'gsl'
+require 'collaborative_filter/data_set'
+require 'collaborative_filter/config'
+require 'collaborative_filter/content_booster'
+require 'collaborative_filter/output'
+require 'collaborative_filter/output/mysql_adapter'
+require 'collaborative_filter/output/yaml_adapter'
+require 'correlators/simple_svd'
+require 'recommenders/simplest_recommender'
+require 'boosters/simple_booster'
+
+class CollaborativeFilter
+  def self.filter(options={})
+    @@logger = options[:logger]
+
+    log "Starting configuration: #{Time.now}"
+    raise '#setup must be sent a block' unless block_given?
+
+    yield config
+
+    log "Starting correlations: #{Time.now}"
+    config.datasets.each do |name,ds|
+      log "  Correlating '#{name}': #{Time.now}"
+      ds.run
+    end
+    log "Starting recommender: #{Time.now}"
+    recommendations = config.recommender.new.run(config.datasets, config.recommender_options)
+
+    log "Starting booster: #{Time.now}"
+    recommendations = config.content_booster.run(recommendations, config.datasets)
+
+    log "Output: #{Time.now}"
+    Output.store config.output, recommendations
+
+    log "Done: #{Time.now}"
+  end
+
+  def self.log(msg)
+    @@logger.info(msg) if @@logger
+  end
+end
+
+

data/lib/collaborative_filter/config.rb

@@ -0,0 +1,58 @@
+class CollaborativeFilter
+
+  def self.config
+    @@config ||= Config.new
+  end
+
+  class Config
+    def self.config_option(option, default=nil)
+      @@config_defaults ||= {}
+      @@config_defaults[option] = default
+
+      define_method(:setup_defaults) {
+        @@config_defaults.each { |k,v| instance_variable_set("@#{k}".to_sym, v) }
+      } unless instance_methods.include?('setup_defaults')
+
+      class_eval <<-END
+        def #{option}(input=nil)
+          return @#{option} unless input
+          @#{option} = input
+        end
+      END
+    end
+
+    config_option :item_genes, []
+    config_option :output, { :type => :yaml }
+    attr_reader :datasets, :recommender_options
+
+    def dataset(name)
+      raise Error.new('#dataset requires a block') unless block_given?
+
+      ds = CollaborativeFilter::DataSet.new
+      yield ds
+      @datasets[name] = ds
+    end
+
+    def content_booster
+      return @content_booster unless block_given?
+
+      cb = CollaborativeFilter::ContentBooster.new
+      yield cb
+      @content_booster = cb
+    end
+
+    def recommender(input=nil, options={})
+      return @recommender unless input
+      @recommender = input
+      @recommender_options = options
+    end
+
+    def initialize
+      setup_defaults
+      @datasets = {}
+    end
+
+    class Error < StandardError; end
+  end
+end
+

data/lib/collaborative_filter/content_booster.rb

@@ -0,0 +1,51 @@
+class CollaborativeFilter
+  class ContentBooster
+    attr_reader :genes
+    attr_reader :booster
+    
+    def initialize
+      @genes = {}
+    end
+
+    def booster(booster_class=nil)
+      return @booster unless booster_class
+      @booster = booster_class
+    end
+
+    def gene(name, &block)
+      gene = Gene.new
+      gene.finder = block
+      @genes[name] = gene
+    end
+
+    def crossover(point=nil)
+      return @crossover unless point
+      @crossover = point
+    end
+
+    def threshold(input=nil)
+      return @threshold unless input
+      @threshold = input
+    end
+
+    def factor(input=nil)
+      return @factor unless input
+      @factor = input
+    end
+
+
+    def run(recommendations,datasets)
+      options = { :crossover => @crossover, :threshold => @threshold, :factor => @factor }
+      @booster.new.run(recommendations,datasets,@genes,options)
+    end
+
+
+    class Gene
+      attr_accessor :finder
+      def all(items)
+        @finder[items]
+      end
+    end
+  end
+end
+

data/lib/collaborative_filter/data_set.rb

@@ -0,0 +1,75 @@
+class CollaborativeFilter
+  class DataSet
+    attr_accessor :users
+    attr_accessor :items
+    attr_accessor :m
+    attr_accessor :similarities
+
+    def users(input=nil)
+      return @users unless input
+
+      raise Error.new("all users must be unique") if input.size != input.uniq.size
+      raise Error.new("must have at least two users") if input.size < 2
+
+      if input.map(&:class).uniq.size == 1
+        @users = input.map(&:id)
+      else
+        @users = input.map { |u| [u.id,u.class.to_s] }
+      end
+    end
+
+    def items(input=nil)
+      return @items unless input
+
+      raise Error.new("all items must be unique") if input.size != input.uniq.size
+      raise Error.new("must have at least two items") if input.size < 2
+
+      if input.map(&:class).uniq.size == 1
+        @items = input.map(&:id)
+      else
+        @items = input.map { |i| [i.id,i.class.to_s] }
+      end
+    end
+
+    def nodes(input=nil)
+      if block_given?
+        @m = GSL::Matrix[@items.size,@users.size]
+        yield @m
+      else
+        @m = input.is_a?(GSL::Matrix) ? input : GSL::Matrix[*input]
+      end
+    end
+
+    def options(opts=nil)
+      return @options unless opts
+      @options = opts
+    end
+
+    def correlator(input=nil)
+      return @correlator unless input
+      @correlator = input
+    end
+
+
+
+    def item_index(id,type=nil)
+      find_index(id,type,@items)
+    end
+
+    def user_index(id,type=nil)
+      find_index(id,type,@users)
+    end
+
+    def run
+      @similarities = @correlator.new.run(@m, @users, @items, @options)
+    end
+    
+    private
+
+    def find_index(id,type,collection)
+      collection.index(type ? [id,type] : id)
+    end
+
+    class Error < StandardError; end
+  end
+end