recommendify-ruby 0.3.8

data/lib/recommendify/cc_matrix.rb

@@ -0,0 +1,51 @@
+module Recommendify::CCMatrix
+
+  def ccmatrix
+    @ccmatrix ||= Recommendify::SparseMatrix.new(
+      :redis_prefix => @opts.fetch(:redis_prefix),
+      :key => [@opts.fetch(:key), :ccmatrix].join(":")
+    )
+  end
+
+  def add_set(set_id, item_ids)
+    # FIXPAUL: forbid | and : in item_ids
+    item_ids.each do |item_id|
+      item_count_incr(item_id)
+    end
+    all_pairs(item_ids).map do |pair| 
+      i1, i2 = pair.split(":") 
+      ccmatrix.incr(i1, i2)
+    end
+  end
+
+  def add_single(set_id, item_id, other_item_ids)
+    item_count_incr(item_id)
+    other_item_ids.each do |other_item|
+      ccmatrix.incr(item_id, other_item)
+    end
+  end
+
+  def all_items
+    Recommendify.redis.hkeys(redis_key(:items))
+  end
+
+  def delete_item(item_id)    
+    Recommendify.redis.hdel(redis_key(:items), item_id)
+    ccmatrix.send(:k_delall, item_id)
+  end
+
+private
+
+  def all_pairs(keys)
+    keys.map{ |k1| (keys-[k1]).map{ |k2| [k1,k2].sort.join(":") } }.flatten.uniq
+  end
+
+  def item_count_incr(key)
+    Recommendify.redis.hincrby(redis_key(:items), key, 1)
+  end
+
+  def item_count(key)
+    Recommendify.redis.hget(redis_key(:items), key).to_i
+  end
+
+end

data/lib/recommendify/cosine_input_matrix.rb

@@ -0,0 +1,7 @@
+class Recommendify::CosineInputMatrix < Recommendify::InputMatrix
+
+  include Recommendify::CCMatrix
+  
+  # here be dragons ;)
+
+end

data/lib/recommendify/input_matrix.rb

@@ -0,0 +1,52 @@
+class Recommendify::InputMatrix
+
+  def self.create(opts)
+    klass = "#{Recommendify.capitalize(opts[:similarity_func])}InputMatrix"
+    Recommendify.constantize(klass.intern).new(opts)
+  end
+
+  def initialize(opts)
+    @opts = opts
+  end
+
+  def redis_key(append=nil)
+    [@opts.fetch(:redis_prefix), @opts.fetch(:key), append].flatten.compact.join(":")
+  end
+
+  def weight
+    (@opts[:weight] || 1).to_f
+  end
+
+  # add a set of item_ids to the matrix
+  def add_set(set_id, item_ids)
+    raise "implemented in subclass"
+  end
+
+  # add a single item to a set of item_ids to the matrix
+  def add_single(set_id, item_id, other_item_ids)
+    raise "implemented in subclass"
+  end
+
+  # calculate the similarity between item1 and item1 (0.0-1.0)
+  def similarity(item1, item2)
+    raise "implemented in subclass"
+  end
+
+  # calculate all similarities to other items in the matrix for item1
+  def similarities_for(item1)
+    # return => [ ["item23", 0.6], ["item42", 0.23], (...) ]
+    raise "implemented in subclass"
+  end
+
+  # retrieve all item_ids in the matrix
+  def all_items
+    # retzrb => [ "item23", "item42", "item17", (...) ]
+    raise "implemented in subclass"
+  end
+
+  # delete item_id from the matrix
+  def delete_item(item_id)
+    raise "implemented in subclass"
+  end
+
+end

data/lib/recommendify/jaccard_input_matrix.rb

@@ -0,0 +1,62 @@
+class Recommendify::JaccardInputMatrix < Recommendify::InputMatrix
+
+  include Recommendify::CCMatrix
+
+  def initialize(opts={})
+    check_native if opts[:native]
+    super(opts)
+  end
+
+  def similarity(item1, item2)
+    calculate_jaccard_cached(item1, item2)
+  end
+
+  def similarities_for(item1)
+    return run_native(item1) if @opts[:native]
+    calculate_similarities(item1)
+  end
+
+private
+
+  def calculate_similarities(item1)
+    (all_items - [item1]).map do |item2|
+      [item2, similarity(item1, item2)]
+    end
+  end
+
+  def calculate_jaccard_cached(item1, item2)
+    val = ccmatrix[item1, item2]
+    val.to_f / (item_count(item1)+item_count(item2)-val).to_f
+  end
+
+  def calculate_jaccard(set1, set2)
+    (set1&set2).length.to_f / (set1 + set2).uniq.length.to_f
+  end
+
+  def run_native(item_id)
+    res = %x{#{native_path} --jaccard "#{redis_key}" "#{item_id}" "#{redis_url}"}
+    raise "error: dirty exit (#{$?})" if $? != 0
+    res.split("\n").map do |line|
+      sim = line.match(/OUT: \(([^\)]*)\) \(([^\)]*)\)/)
+      unless sim
+        raise "error: #{res}" unless (res||"").include?('exit:')
+      else
+        [sim[1], sim[2].to_f]
+      end
+    end.compact
+  end
+
+  def check_native
+    return true if ::File.exists?(native_path)
+    raise "recommendify_native not found - you need to run rake build_native first"
+  end
+
+  def native_path
+    ::File.expand_path('../../../bin/recommendify', __FILE__)
+  end
+
+  def redis_url
+    Recommendify.redis.client.location
+  end
+
+end

data/lib/recommendify/neighbor.rb

@@ -0,0 +1,19 @@
+class Recommendify::Neighbor
+
+  def initialize(data)
+    @data = data
+  end
+
+  def item_id
+    @data.fetch(:item_id).to_s
+  end
+
+  def similarity
+    @data.fetch(:similarity)
+  end
+
+  def <=>(other)
+    other.similarity <=> self.similarity
+  end
+
+end

data/lib/recommendify/recommendify.rb

@@ -0,0 +1,25 @@
+module Recommendify
+
+  DEFAULT_MAX_NEIGHBORS = 50
+
+  @@redis = nil
+
+  def self.redis=(redis)
+    @@redis = redis
+  end
+
+  def self.redis
+    return @@redis unless @@redis.nil?
+    raise "redis not configured! - Recommendify.redis = Redis.new"
+  end
+
+  def self.capitalize(str_or_sym)
+  	str = str_or_sym.to_s.each_char.to_a
+  	str.first.upcase + str[1..-1].join("").downcase
+  end
+
+  def self.constantize(klass)
+    Object.module_eval("Recommendify::#{klass}", __FILE__, __LINE__)
+  end
+
+end

data/lib/recommendify/similarity_matrix.rb

@@ -0,0 +1,62 @@
+class Recommendify::SimilarityMatrix
+
+  attr_reader :write_queue
+
+  def initialize(opts={})
+    @opts = opts
+    @write_queue = Hash.new{ |h,k| h[k] = {} }
+  end
+
+  def redis_key(append=nil)
+    [@opts.fetch(:redis_prefix), @opts.fetch(:key), append].flatten.compact.join(":")
+  end
+
+  def max_neighbors
+    @opts[:max_neighbors] || Recommendify::DEFAULT_MAX_NEIGHBORS
+  end
+
+  def update(item_id, neighbors)
+    neighbors.each do |neighbor_id, score|
+      if @write_queue[item_id].has_key?(neighbor_id)
+        @write_queue[item_id][neighbor_id] += score
+      else
+        @write_queue[item_id][neighbor_id] = score
+      end
+    end
+  end
+
+  def [](item_id)
+    if @write_queue.has_key?(item_id)
+      @write_queue[item_id]
+    else
+      retrieve_item(item_id)
+    end
+  end
+
+  def commit_item!(item_id)
+    serialized = serialize_item(item_id)
+    Recommendify.redis.hset(redis_key, item_id, serialized)
+    @write_queue.delete(item_id)    
+  end
+
+  # optimize: the items are already stored in a sorted fashion. we shouldn't 
+  # throw away this info by storing them in a hash (and re-sorting later). maybe 
+  # use activesupport's orderedhash?
+  def retrieve_item(item_id)
+    data = Recommendify.redis.hget(redis_key, item_id)
+    return {} if data.nil?
+    Hash[data.split("|").map{ |i| (k,s=i.split(":")) && [k,s.to_f] }]
+  end
+
+private
+
+  # optimize: implement a better sort. never add more than 50 items the the array
+  def serialize_item(item_id, max_precision=5)
+    items = @write_queue[item_id].to_a
+    items.sort!{ |a,b| b[1] <=> a[1] }
+    items = items[0..max_neighbors-1]
+    items = items.map{ |i,s| s>0 ? "#{i}:#{s.to_s[0..max_precision]}" : nil } 
+    items.compact * "|"
+  end
+
+end

data/lib/recommendify/sparse_matrix.rb

@@ -0,0 +1,53 @@
+class Recommendify::SparseMatrix
+
+  def initialize(opts={})
+    @opts = opts
+  end
+
+  def redis_key
+    [@opts.fetch(:redis_prefix), @opts.fetch(:key)].join(":")
+  end
+
+  def [](x,y)
+    k_get(key(x,y))
+  end
+
+  def []=(x,y,v)
+    v == 0 ? k_del(key(x,y)) : k_set(key(x,y), v)
+  end
+
+  def incr(x,y)
+    k_incr(key(x,y))
+  end
+
+private
+
+  def key(x,y)
+    [x,y].sort.join(":")
+  end
+
+  def k_set(key, val)
+    Recommendify.redis.hset(redis_key, key, val)
+  end
+
+  def k_del(key)
+    Recommendify.redis.hdel(redis_key, key)
+  end
+
+  def k_get(key)
+    Recommendify.redis.hget(redis_key, key).to_f
+  end
+
+  def k_incr(key)
+    Recommendify.redis.hincrby(redis_key, key, 1)
+  end
+
+  # OPTIMIZE: use scripting/lua in redis 2.6
+  def k_delall(*keys)
+    Recommendify.redis.hkeys(redis_key).each do |iikey|
+      next unless (iikey.split(":") & keys).size > 0
+      Recommendify.redis.hdel(redis_key, iikey)
+    end
+  end
+
+end

data/recommendify.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "recommendify-ruby"
+  s.version     = "0.3.8"
+  s.date        = Date.today.to_s
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Paul Asmuth", "Raj Shah"]
+  s.email       = ["paul@paulasmuth.com", "brainix@gmail.com"]
+  s.homepage    = "http://github.com/brainix/recommendify"
+  s.summary     = %q{ruby/redis based recommendation engine (collaborative filtering)}
+  s.description = %q{Recommendify is a distributed, incremental item-based recommendation engine for binary input ratings. It's based on ruby and redis and uses an approach called "Collaborative Filtering"}
+  s.licenses    = ["MIT"]
+
+  # s.extensions = ['ext/extconf.rb']
+
+  s.add_dependency "redis", ">= 2.2.2"
+
+  s.add_development_dependency "rspec", "~> 2.8.0"
+
+  s.files         = `git ls-files`.split("\n") - [".gitignore", ".rspec", ".travis.yml"]
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ["lib"]
+end

data/spec/base_spec.rb

@@ -0,0 +1,188 @@
+require ::File.expand_path('../spec_helper', __FILE__)
+
+describe Recommendify::Base do
+
+  before(:each) do
+    flush_redis!
+    Recommendify::Base.send(:class_variable_set, :@@max_neighbors, nil)
+    Recommendify::Base.send(:class_variable_set, :@@input_matrices, {})
+  end
+
+  describe "configuration" do
+
+    it "should return default max_neighbors if not configured" do
+      Recommendify::DEFAULT_MAX_NEIGHBORS.should == 50      
+      sm = Recommendify::Base.new
+      sm.max_neighbors.should == 50
+    end
+      
+    it "should remember max_neighbors if configured" do
+      Recommendify::Base.max_neighbors(23)
+      sm = Recommendify::Base.new      
+      sm.max_neighbors.should == 23
+    end
+
+    it "should add an input_matrix by 'key'" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      Recommendify::Base.send(:class_variable_get, :@@input_matrices).keys.should == [:myinput]
+    end
+
+    it "should retrieve an input_matrix on a new instance" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      lambda{ sm.myinput }.should_not raise_error
+    end
+
+    it "should retrieve an input_matrix on a new instance and correctly overload respond_to?" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.respond_to?(:process!).should be_true
+      sm.respond_to?(:myinput).should be_true
+      sm.respond_to?(:fnord).should be_false
+    end
+
+    it "should retrieve an input_matrix on a new instance and intialize the correct class" do
+      Recommendify::Base.input_matrix(:myinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myinput.should be_a(Recommendify::JaccardInputMatrix)
+    end
+
+  end
+
+  describe "process_item!" do
+
+    it "should call similarities_for on each input_matrix" do
+      Recommendify::Base.input_matrix(:myfirstinput, :similarity_func => :jaccard)
+      Recommendify::Base.input_matrix(:mysecondinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myfirstinput.should_receive(:similarities_for).with("fnorditem").and_return([["fooitem",0.5]])
+      sm.mysecondinput.should_receive(:similarities_for).with("fnorditem").and_return([["fooitem",0.5]])
+      sm.similarity_matrix.stub!(:update)
+      sm.process_item!("fnorditem")
+    end
+
+    it "should call similarities_for on each input_matrix and add all outputs to the similarity matrix" do
+      Recommendify::Base.input_matrix(:myfirstinput, :similarity_func => :jaccard)
+      Recommendify::Base.input_matrix(:mysecondinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myfirstinput.should_receive(:similarities_for).and_return([["fooitem",0.5]])      
+      sm.mysecondinput.should_receive(:similarities_for).and_return([["fooitem",0.75], ["baritem", 1.0]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",0.5]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",0.75], ["baritem", 1.0]])
+      sm.process_item!("fnorditem")
+    end
+
+    it "should call similarities_for on each input_matrix and add all outputs to the similarity matrix with weight" do
+      Recommendify::Base.input_matrix(:myfirstinput, :similarity_func => :jaccard, :weight => 4.0)
+      Recommendify::Base.input_matrix(:mysecondinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myfirstinput.should_receive(:similarities_for).and_return([["fooitem",0.5]])      
+      sm.mysecondinput.should_receive(:similarities_for).and_return([["fooitem",0.75], ["baritem", 1.0]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",2.0]])
+      sm.similarity_matrix.should_receive(:update).with("fnorditem", [["fooitem",0.75], ["baritem", 1.0]])
+      sm.process_item!("fnorditem")
+    end
+
+    it "should retrieve all items from all input matrices" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "shmoo"])
+      sm = Recommendify::Base.new    
+      sm.all_items.length.should == 4
+      sm.all_items.should include("foo")
+      sm.all_items.should include("bar")
+      sm.all_items.should include("fnord")
+      sm.all_items.should include("shmoo")
+    end
+
+    it "should retrieve all items from all input matrices (uniquely)" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "bar"])
+      sm = Recommendify::Base.new    
+      sm.all_items.length.should == 3
+      sm.all_items.should include("foo")
+      sm.all_items.should include("bar")
+      sm.all_items.should include("fnord")
+    end
+
+  end
+
+  describe "process!" do
+
+    it "should call process_item for all input_matrix.all_items's" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "shmoo"])
+      sm = Recommendify::Base.new    
+      sm.should_receive(:process_item!).exactly(4).times
+      sm.process!
+    end
+
+    it "should call process_item for all input_matrix.all_items's (uniquely)" do
+      Recommendify::Base.input_matrix(:anotherinput, :similarity_func => :test, :all_items => ["foo", "bar"])
+      Recommendify::Base.input_matrix(:yetanotherinput, :similarity_func => :test, :all_items => ["fnord", "bar"])
+      sm = Recommendify::Base.new    
+      sm.should_receive(:process_item!).exactly(3).times
+      sm.process!
+    end
+
+  end
+
+  describe "for(item_id)" do
+
+    it "should retrieve the n-most similar neighbors" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).with("fnorditem").and_return({:fooitem => 0.4, :baritem => 1.5})
+      sm.for("fnorditem").length.should == 2
+    end
+    
+    it "should not throw exception for non existing items" do
+      sm = Recommendify::Base.new
+      sm.for("not_existing_item").length.should == 0
+    end
+
+    it "should retrieve the n-most similar neighbors as Recommendify::Neighbor objects" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).exactly(2).times.with("fnorditem").and_return({:fooitem => 0.4, :baritem => 1.5})
+      sm.for("fnorditem").first.should be_a(Recommendify::Neighbor)
+      sm.for("fnorditem").last.should be_a(Recommendify::Neighbor)
+    end
+
+    it "should retrieve the n-most similar neighbors in the correct order" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).exactly(4).times.with("fnorditem").and_return({:fooitem => 0.4, :baritem => 1.5})
+      sm.for("fnorditem").first.similarity.should == 1.5
+      sm.for("fnorditem").first.item_id.should == "baritem"
+      sm.for("fnorditem").last.similarity.should == 0.4
+      sm.for("fnorditem").last.item_id.should == "fooitem"
+    end
+
+    it "should return an empty array if the item if no neighbors were found" do
+      sm = Recommendify::Base.new
+      sm.similarity_matrix.should_receive(:[]).with("fnorditem").and_return({})
+      sm.for("fnorditem").should == []
+    end
+
+    it "should not call split on nil when retrieving a non-existent item (return an empty array)" do
+      sm = Recommendify::Base.new
+      sm.for("NONEXISTENT").should == []
+    end
+
+  end
+
+  describe "delete_item!" do
+
+    it "should call delete_item on each input_matrix" do
+      Recommendify::Base.input_matrix(:myfirstinput, :similarity_func => :jaccard)
+      Recommendify::Base.input_matrix(:mysecondinput, :similarity_func => :jaccard)
+      sm = Recommendify::Base.new
+      sm.myfirstinput.should_receive(:delete_item).with("fnorditem")
+      sm.mysecondinput.should_receive(:delete_item).with("fnorditem")
+      sm.delete_item!("fnorditem")
+    end
+
+    it "should delete the item from the similarity matrix"
+
+    it "should delete all occurences of the item in other similarity sets from the similarity matrix"
+
+  end
+
+end