recommendify 0.0.1

data/lib/recommendify.rb

@@ -0,0 +1,9 @@
+require "recommendify/recommendify"
+require "recommendify/sparse_matrix"
+require "recommendify/cc_matrix"
+require "recommendify/similarity_matrix"
+require "recommendify/input_matrix"
+require "recommendify/jaccard_input_matrix"
+require "recommendify/cosine_input_matrix"
+require "recommendify/base"
+require "recommendify/neighbor"

data/lib/recommendify/base.rb

@@ -0,0 +1,80 @@
+class Recommendify::Base
+
+  attr_reader :similarity_matrix, :input_matrices
+
+  @@max_neighbors = nil
+  @@input_matrices = {}
+
+  def self.max_neighbors(n=nil)    
+    return @@max_neighbors unless n
+    @@max_neighbors = n
+  end
+
+  def self.input_matrix(key, opts)
+    @@input_matrices[key] = opts
+  end
+
+  def self.input_matrices
+    @@input_matrices
+  end
+
+  def initialize    
+    @input_matrices = Hash[self.class.input_matrices.map{ |key, opts| 
+      opts.merge!(:key => key, :redis_prefix => redis_prefix)
+      [ key, Recommendify::InputMatrix.create(opts) ]
+    }]
+    @similarity_matrix = Recommendify::SimilarityMatrix.new(
+      :max_neighbors => max_neighbors,
+      :key => :similarities,
+      :redis_prefix => redis_prefix
+    )
+  end
+
+  def redis_prefix
+    "recommendify"
+  end
+
+  def max_neighbors
+    self.class.max_neighbors || Recommendify::DEFAULT_MAX_NEIGHBORS
+  end
+
+  def method_missing(method, *args)
+    if @input_matrices.has_key?(method)
+      @input_matrices[method]
+    else
+      raise NoMethodError.new(method.to_s)
+    end
+  end
+
+  def respond_to?(method)
+    @input_matrices.has_key?(method) ? true : super
+  end
+
+  def all_items
+    @input_matrices.map{ |k,m| m.all_items }.flatten.uniq
+  end
+
+  def for(item_id)
+    similarity_matrix[item_id].map do |item_id, similarity|
+      Recommendify::Neighbor.new(
+        :item_id => item_id, 
+        :similarity => similarity
+      )
+    end.sort
+  end
+
+  def process!
+    all_items.each{ |item_id,n| process_item!(item_id) }
+  end
+
+  def process_item!(item_id)
+    input_matrices.map do |k,m|
+      neighbors = m.similarities_for(item_id).map do |i,w|        
+        [i,w*m.weight]
+      end
+      similarity_matrix.update(item_id, neighbors)
+    end
+    similarity_matrix.commit_item!(item_id)
+  end
+
+end

data/lib/recommendify/cc_matrix.rb

@@ -0,0 +1,46 @@
+module Recommendify::CCMatrix
+
+  def ccmatrix
+    @ccmatrix ||= Recommendify::SparseMatrix.new(
+      :redis_prefix => @opts.fetch(:redis_prefix),
+      :key => [@opts.fetch(:key), :ccmatrix].join(":")
+    )
+  end
+
+  def add_set(set_id, item_ids)
+    item_ids.each do |item_id|
+      item_count_incr(item_id)
+    end
+    all_pairs(item_ids).map do |pair| 
+      i1, i2 = pair.split(":") 
+      ccmatrix.incr(i1, i2)
+    end
+  end
+
+  def add_single(set_id, item_id, other_item_ids)
+  # todo: add single item to set after set was added (incrementally)
+  #   item_count_incr(item_id)
+  #   other_item_ids.each do |other_item|
+  #     @ccmatrix.incr(item_id, other_idem)
+  #   end
+  end
+
+  def all_items
+    Recommendify.redis.hkeys(redis_key(:items))
+  end
+
+private
+
+  def all_pairs(keys)
+    keys.map{ |k1| (keys-[k1]).map{ |k2| [k1,k2].sort.join(":") } }.flatten.uniq
+  end
+
+  def item_count_incr(key)
+    Recommendify.redis.hincrby(redis_key(:items), key, 1)
+  end
+
+  def item_count(key)
+    Recommendify.redis.hget(redis_key(:items), key).to_i
+  end
+
+end

data/lib/recommendify/cosine_input_matrix.rb

@@ -0,0 +1,7 @@
+class Recommendify::CosineInputMatrix < Recommendify::InputMatrix
+
+  include Recommendify::CCMatrix
+  
+  # here be dragons ;)
+
+end

data/lib/recommendify/input_matrix.rb

@@ -0,0 +1,47 @@
+class Recommendify::InputMatrix
+
+  def self.create(opts)
+    klass = "#{Recommendify.capitalize(opts[:similarity_func])}InputMatrix"
+    Recommendify.constantize(klass.intern).new(opts)
+  end
+
+  def initialize(opts)
+    @opts = opts
+  end
+
+  def redis_key(append=nil)
+    [@opts.fetch(:redis_prefix), @opts.fetch(:key), append].flatten.compact.join(":")
+  end
+
+  def weight
+    (@opts[:weight] || 1).to_f
+  end
+
+  # add a set of item_ids to the matrix
+  def add_set(set_id, item_ids)
+    raise "implemented in subclass"
+  end
+
+  # add a single item to a set of item_ids to the matrix
+  def add_single(set_id, item_id, other_item_ids)
+    raise "implemented in subclass"
+  end
+
+  # calculate the similarity between item1 and item1 (0.0-1.0)
+  def similarity(item1, item2)
+    raise "implemented in subclass"
+  end
+
+  # calculate all similarities to other items in the matrix for item1
+  def similarities_for(item1)
+    # return => [ ["item23", 0.6], ["item42", 0.23], (...) ]
+    raise "implemented in subclass"
+  end
+
+  # retrieve all item_ids in the matrix
+  def all_items
+    # retzrb => [ "item23", "item42", "item17", (...) ]
+    raise "implemented in subclass"
+  end
+
+end

data/lib/recommendify/jaccard_input_matrix.rb

@@ -0,0 +1,35 @@
+class Recommendify::JaccardInputMatrix < Recommendify::InputMatrix
+
+  include Recommendify::CCMatrix
+
+  def initialize(opts={})
+    super(opts)
+  end
+
+  def similarity(item1, item2)
+    calculate_jaccard_cached(item1, item2)
+  end
+
+  # optimize: get all item-counts and the cc-row with 2 redis hmgets.
+  # optimize: don't return more than sm.max_neighbors items (truncate set while collecting)
+  def similarities_for(item1)
+    # todo: optimize native. execute with own redis conn and write top K to stdout
+    # native_ouput = %x{recommendify_native jaccard "#{redis_key}" "#{item1}"}
+    # return native_output.split("\n").map{ |l| l.split(",") }
+    (all_items - [item1]).map do |item2|
+      [item2, similarity(item1, item2)]
+    end
+  end
+
+private
+
+  def calculate_jaccard_cached(item1, item2)
+    val = ccmatrix[item1, item2]
+    val.to_f / (item_count(item1)+item_count(item2)-val).to_f
+  end
+
+  def calculate_jaccard(set1, set2)
+    (set1&set2).length.to_f / (set1 + set2).uniq.length.to_f
+  end
+
+end

data/lib/recommendify/neighbor.rb

@@ -0,0 +1,19 @@
+class Recommendify::Neighbor
+
+  def initialize(data)
+    @data = data
+  end
+
+  def item_id
+    @data.fetch(:item_id).to_s
+  end
+
+  def similarity
+    @data.fetch(:similarity)
+  end
+
+  def <=>(other)
+    other.similarity <=> self.similarity
+  end
+
+end

data/lib/recommendify/recommendify.rb

@@ -0,0 +1,25 @@
+module Recommendify
+
+  DEFAULT_MAX_NEIGHBORS = 50
+
+  @@redis = nil
+
+  def self.redis=(redis)
+    @@redis = redis
+  end
+
+  def self.redis
+    return @@redis unless @@redis.nil?
+    raise "redis not configured! - Recommendify.redis = Redis.new"
+  end
+
+  def self.capitalize(str_or_sym)
+  	str = str_or_sym.to_s.each_char.to_a
+  	str.first.upcase + str[1..-1].join("").downcase
+  end
+
+  def self.constantize(klass)
+    Object.module_eval("Recommendify::#{klass}", __FILE__, __LINE__)
+  end
+
+end

data/lib/recommendify/similarity_matrix.rb

@@ -0,0 +1,61 @@
+class Recommendify::SimilarityMatrix
+
+  attr_reader :write_queue
+
+  def initialize(opts={})
+    @opts = opts
+    @write_queue = Hash.new{ |h,k| h[k] = {} }
+  end
+
+  def redis_key(append=nil)
+    [@opts.fetch(:redis_prefix), @opts.fetch(:key), append].flatten.compact.join(":")
+  end
+
+  def max_neighbors
+    @opts[:max_neighbors] || Recommendify::DEFAULT_MAX_NEIGHBORS
+  end
+
+  def update(item_id, neighbors)
+    neighbors.each do |neighbor_id, score|
+      if @write_queue[item_id].has_key?(neighbor_id)
+        @write_queue[item_id][neighbor_id] += score
+      else
+        @write_queue[item_id][neighbor_id] = score
+      end
+    end
+  end
+
+  def [](item_id)
+    if @write_queue.has_key?(item_id)
+      @write_queue[item_id]
+    else
+      retrieve_item(item_id)
+    end
+  end
+
+  def commit_item!(item_id)
+    serialized = serialize_item(item_id)
+    Recommendify.redis.hset(redis_key, item_id, serialized)
+    @write_queue.delete(item_id)    
+  end
+
+  # optimize: the items are already stored in a sorted fashion. we shouldn't 
+  # throw away this info by storing them in a hash (and re-sorting later). maybe 
+  # use activesupport's orderedhash?
+  def retrieve_item(item_id)
+    data = Recommendify.redis.hget(redis_key, item_id)
+    Hash[data.split("|").map{ |i| (k,s=i.split(":")) && [k,s.to_f] }]
+  end
+
+private
+
+  # optimize: implement a better sort. never add more than 50 items the the array
+  def serialize_item(item_id, max_precision=5)
+    items = @write_queue[item_id].to_a
+    items.sort!{ |a,b| b[1] <=> a[1] }
+    items = items[0..max_neighbors-1]
+    items = items.map{ |i,s| s>0 ? "#{i}:#{s.to_s[0..max_precision]}" : nil } 
+    items.compact * "|"
+  end
+
+end

data/lib/recommendify/sparse_matrix.rb

@@ -0,0 +1,45 @@
+class Recommendify::SparseMatrix
+
+  def initialize(opts={})
+    @opts = opts
+  end
+
+  def redis_key
+    [@opts.fetch(:redis_prefix), @opts.fetch(:key)].join(":")
+  end
+
+  def [](x,y)
+    k_get(key(x,y))
+  end
+
+  def []=(x,y,v)
+    v == 0 ? k_del(key(x,y)) : k_set(key(x,y), v)
+  end
+
+  def incr(x,y)
+    k_incr(key(x,y))
+  end
+
+private
+
+  def key(x,y)
+    [x,y].sort.join(":")
+  end
+
+  def k_set(key, val)
+    Recommendify.redis.hset(redis_key, key, val)
+  end
+
+  def k_del(key)
+    Recommendify.redis.hdel(redis_key, key)
+  end
+
+  def k_get(key)
+    Recommendify.redis.hget(redis_key, key).to_f
+  end
+
+  def k_incr(key)
+    Recommendify.redis.hincrby(redis_key, key, 1)
+  end
+
+end

data/spec/base_spec.rb

@@ -0,0 +1,161 @@
+require ::File.expand_path('../spec_helper', __FILE__)
+
+describe Recommendify::Base do
+
+  before(:each) do
+    flush_redis!
+    Recommendify::Base.class_variable_set(:@@max_neighbors, nil)
+    Recommendify::Base.class_variable_set(:@@input_matrices, {})
+  end
+
+  describe "configuration" do
+
+    it "should return default max_neighbors if not configured" do
+      Recommendify::DEFAULT_MAX_NEIGHBORS.should == 50      
+      sm = Recommendify::Base.new
+      sm.max_neighbors.should == 50
+    end
+      
+    it "should remember max_neighbors if configured" do
+      Recommendify::Base.max_neighbors(23)
+      sm = Recommendify::Base.new      
+      sm.max_neighbors.should == 23
+    end
+
+    it "should add an input_matrix by 'key'" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      Recommendify::Base.class_variable_get(:@@input_matrices).keys.should == [:myinput]
+    end
+
+    it "should retrieve an input_matrix on a new instance" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      lambda{ sm.myinput }.should_not raise_error
+    end
+
+    it "should retrieve an input_matrix on a new instance and correctly overload respond_to?" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.respond_to?(:process!).should be_true
+      sm.respond_to?(:myinput).should be_true
+      sm.respond_to?(:fnord).should be_false
+    end
+
+    it "should retrieve an input_matrix on a new instance and intialize the correct class" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myinput.should be_a(Recommendify::JaccardInputMatrix)
+    end
+
+  end
+
+  describe "process_item!" do
+
+    it "should call similarities_for on each input_matrix" do
+      Recommendify::Base.input_matrix(:myfirstinput, :similarity_func => :jaccard)
+      Recommendify::Base.input_matrix(:mysecondinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myfirstinput.should_receive(:similarities_for).with("fnorditem").and_return([["fooitem",0.5]])
+      sm.mysecondinput.should_receive(:similarities_for).with("fnorditem").and_return([["fooitem",0.5]])
+      sm.similarity_matrix.stub!(:update)
+      sm.process_item!("fnorditem")
+    end
+
+    it "should call similarities_for on each input_matrix and add all outputs to the similarity matrix" do
+      Recommendify::Base.input_matrix(:myfirstinput, :similarity_func => :jaccard)
+      Recommendify::Base.input_matrix(:mysecondinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myfirstinput.should_receive(:similarities_for).and_return([["fooitem",0.5]])      
+      sm.mysecondinput.should_receive(:similarities_for).and_return([["fooitem",0.75], ["baritem", 1.0]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",0.5]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",0.75], ["baritem", 1.0]])
+      sm.process_item!("fnorditem")
+    end
+
+    it "should call similarities_for on each input_matrix and add all outputs to the similarity matrix with weight" do
+      Recommendify::Base.input_matrix(:myfirstinput, :similarity_func => :jaccard, :weight => 4.0)
+      Recommendify::Base.input_matrix(:mysecondinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myfirstinput.should_receive(:similarities_for).and_return([["fooitem",0.5]])      
+      sm.mysecondinput.should_receive(:similarities_for).and_return([["fooitem",0.75], ["baritem", 1.0]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",2.0]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",0.75], ["baritem", 1.0]])
+      sm.process_item!("fnorditem")
+    end
+
+    it "should retrieve all items from all input matrices" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "shmoo"])
+      sm = Recommendify::Base.new    
+      sm.all_items.length.should == 4
+      sm.all_items.should include("foo")
+      sm.all_items.should include("bar")
+      sm.all_items.should include("fnord")
+      sm.all_items.should include("shmoo")
+    end
+
+    it "should retrieve all items from all input matrices (uniquely)" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "bar"])
+      sm = Recommendify::Base.new    
+      sm.all_items.length.should == 3
+      sm.all_items.should include("foo")
+      sm.all_items.should include("bar")
+      sm.all_items.should include("fnord")
+    end
+
+  end
+
+  describe "process!" do
+
+    it "should call process_item for all input_matrix.all_items's" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "shmoo"])
+      sm = Recommendify::Base.new    
+      sm.should_receive(:process_item!).exactly(4).times
+      sm.process!
+    end
+
+    it "should call process_item for all input_matrix.all_items's (uniquely)" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "bar"])
+      sm = Recommendify::Base.new    
+      sm.should_receive(:process_item!).exactly(3).times
+      sm.process!
+    end
+
+  end
+
+  describe "for(item_id)" do
+
+    it "should retrieve the n-most similar neighbors" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).with("fnorditem").and_return({:fooitem => 0.4, :baritem => 1.5})
+      sm.for("fnorditem").length.should == 2
+    end
+
+    it "should retrieve the n-most similar neighbors as Recommendify::Neighbor objects" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).exactly(2).times.with("fnorditem").and_return({:fooitem => 0.4, :baritem => 1.5})
+      sm.for("fnorditem").first.should be_a(Recommendify::Neighbor)
+      sm.for("fnorditem").last.should be_a(Recommendify::Neighbor)
+    end
+
+    it "should retrieve the n-most similar neighbors in the correct order" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).exactly(4).times.with("fnorditem").and_return({:fooitem => 0.4, :baritem => 1.5})
+      sm.for("fnorditem").first.similarity.should == 1.5
+      sm.for("fnorditem").first.item_id.should == "baritem"
+      sm.for("fnorditem").last.similarity.should == 0.4
+      sm.for("fnorditem").last.item_id.should == "fooitem"
+    end
+
+    it "should return an empty array if the item if no neighbors were found" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).with("fnorditem").and_return({})
+      sm.for("fnorditem").should == []
+    end
+
+  end
+
+end