recommendify_whosv 0.5.8

data/ext/cc_item.h

@@ -0,0 +1,8 @@
+#define ITEM_ID_SIZE 64
+
+struct cc_item {         
+  char  item_id[ITEM_ID_SIZE];
+  int   coconcurrency_count;
+  int   total_count;
+  float similarity;  
+};         

data/ext/cosine.c

@@ -0,0 +1,3 @@
+void calculate_cosine(char *item_id, int itemCount, struct cc_item *cc_items, int cc_items_size){
+  /* here be dragons */
+}

data/ext/extconf.rb

@@ -0,0 +1,18 @@
+makefile = <<-MAKEFILE
+all: prepare build
+
+build:
+	gcc -Wall recommendify.c -lhiredis -o ../bin/recommendify
+
+prepare:
+	mkdir -p ../bin
+
+clean:
+	rm -f *.o
+
+install: prepare
+MAKEFILE
+  
+File.open(::File.expand_path("../Makefile", __FILE__), "w+") do |f|   
+  f.write(makefile)
+end

data/ext/iikey.c

@@ -0,0 +1,18 @@
+char* item_item_key(char *item1, char *item2){
+  int keylen = strlen(item1) + strlen(item2) + 2;
+  char *key = (char *)malloc(keylen * sizeof(char));
+
+  if(!key){
+    printf("cannot allocate\n");
+    return 0;
+  }  
+
+  // FIXPAUL: make shure this does exactly the same as ruby sort  
+  if(rb_strcmp(item1, item2) <= 0){
+    snprintf(key, keylen, "%s:%s", item1, item2);
+  } else {
+    snprintf(key, keylen, "%s:%s", item2, item1);
+  }
+
+  return key;
+}

data/ext/jaccard.c

@@ -0,0 +1,19 @@
+void calculate_jaccard(char *item_id, int itemCount, struct cc_item *cc_items, int cc_items_size){
+  int j, n;
+
+  for(j = 0; j < cc_items_size; j++){
+    n = cc_items[j].coconcurrency_count;
+    if(n>0){
+      cc_items[j].similarity = (
+        (float)n / (
+          (float)itemCount + 
+          (float)cc_items[j].total_count - 
+          (float)n
+        )
+      );
+    } else {
+      cc_items[j].similarity = 0.0;
+    }
+  }  
+
+}

data/ext/output.c

@@ -0,0 +1,22 @@
+int print_version(){
+  printf(
+    VERSION_STRING, 
+    VERSION_MAJOR, 
+    VERSION_MINOR, 
+    VERSION_MICRO
+  );
+  return 0;
+}
+
+int print_usage(char *bin){
+  printf(USAGE_STRING, bin);
+  return 1;
+}
+
+void print_item(struct cc_item item){
+  printf(
+    "OUT: (%s) (%.4f)\n", 
+    item.item_id, 
+    item.similarity
+  );
+}

data/ext/recommendify.c

@@ -0,0 +1,214 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <hiredis/hiredis.h>
+
+#include "version.h"
+#include "cc_item.h" 
+#include "jaccard.c" 
+#include "cosine.c" 
+#include "output.c" 
+#include "sort.c" 
+#include "iikey.c" 
+
+
+int main(int argc, char **argv){
+  int i, j, n, similarityFunc = 0;    
+  int itemCount = 0;  
+  char *itemID;  
+  char *redisPrefix;
+  redisContext *c;
+  redisReply *all_items;
+  redisReply *reply;
+  int cur_batch_size;
+  char* cur_batch;
+  char *iikey;
+
+  int batch_size = 200; /* FIXPAUL: make option */
+  int maxItems = 50; /* FIXPAUL: make option */
+  
+  struct {
+    char  host[1024];
+    int   port;
+  } redis_addr;
+
+  /* option parsing */
+  if(argc < 2)
+    return print_usage(argv[0]);
+
+  if(!strcmp(argv[1], "--version"))
+    return print_version();
+
+  if(!strcmp(argv[1], "--jaccard")) 
+    similarityFunc = 1;
+
+  if(!strcmp(argv[1], "--cosine"))  
+    similarityFunc = 2;
+
+  if(!similarityFunc){
+    printf("invalid option: %s\n", argv[1]);
+    return 1;
+  } else if(argc < 4 || argc > 5){
+    printf("wrong number of arguments\n");
+    print_usage(argv[0]);
+    return 1;
+  }
+
+  redisPrefix = argv[2];
+  itemID = argv[3];
+  redis_addr.host[0] = 0;
+  redis_addr.port = 0;
+
+  /* configure redis location */
+  if(argc > 4){
+    char* has_port = strchr(argv[4], ':');
+    if(has_port){
+      strncpy(redis_addr.host, argv[4], strlen(argv[4]) - strlen(has_port));
+      redis_addr.host[strlen(argv[4]) - strlen(has_port)] = 0;
+      redis_addr.port = atoi(has_port + 1);
+    } else {
+      strncpy(redis_addr.host, argv[4], sizeof(redis_addr.host));
+    }
+  }
+
+  /* default redis location */
+  if(strlen(redis_addr.host) == 0)
+    strcpy(redis_addr.host, "localhost");
+
+  if(!redis_addr.port)
+    redis_addr.port = 6379;
+
+  /* connect to redis */
+  struct timeval timeout = { 1, 500000 }; 
+  c = redisConnectWithTimeout(redis_addr.host, redis_addr.port, timeout);
+
+  if(c->err){
+    printf("connection to redis failed: %s\n", c->errstr);
+    return 1;
+  }
+
+
+  /* get item count */
+  reply = redisCommand(c,"HGET %s:items %s", redisPrefix, itemID);    
+
+  if(reply->str){
+    itemCount = atoi(reply->str);
+  } else {
+    itemCount = 0;
+  }
+
+  freeReplyObject(reply);
+
+  if(itemCount < 2){
+    printf("exit: item count is zero or one\n");
+    return 0;
+  }
+  
+
+  /* get all items_ids and the total counts */ 
+  all_items = redisCommand(c,"HGETALL %s:items", redisPrefix);
+
+  if(all_items->type != REDIS_REPLY_ARRAY)
+    return 1;
+
+
+  /* populate the cc_items array */ 
+  int cc_items_size = all_items->elements / 2;  
+  int cc_items_mem = cc_items_size * sizeof(struct cc_item);
+  struct cc_item *cc_items = malloc(cc_items_mem);
+  cc_items_size--;
+
+  if(!cc_items){    
+    printf("cannot allocate memory: %i", cc_items_mem);
+    return 1;
+  }
+  
+  i = 0;
+  for (j = 0; j < all_items->elements/2; j++){                   
+    if(strcmp(itemID, all_items->element[j*2]->str) != 0){      
+      strncpy(cc_items[i].item_id, all_items->element[j*2]->str, ITEM_ID_SIZE);
+      cc_items[i].total_count = atoi(all_items->element[j*2+1]->str);
+      i++;
+    }
+  }
+
+  freeReplyObject(all_items);
+
+
+  // batched redis hmgets on the ccmatrix
+  cur_batch = (char *)malloc(((batch_size * (ITEM_ID_SIZE + 4) * 2) + 100) * sizeof(char));
+
+  if(!cur_batch){    
+    printf("cannot allocate memory");
+    return 1;
+  }
+
+  n = cc_items_size;
+  while(n >= 0){
+    cur_batch_size = ((n-1 < batch_size) ? n-1 : batch_size);
+    sprintf(cur_batch, "HMGET %s:ccmatrix ", redisPrefix);
+
+    for(i = 0; i < cur_batch_size; i++){
+      iikey = item_item_key(itemID, cc_items[n-i].item_id);
+
+      strcat(cur_batch, iikey);
+      strcat(cur_batch, " ");
+
+      if(iikey)
+        free(iikey);
+    }    
+
+    redisAppendCommand(c, cur_batch);  
+    redisGetReply(c, (void**)&reply);
+      
+    for(j = 0; j < reply->elements; j++){
+      if(reply->element[j]->str){
+        cc_items[n-j].coconcurrency_count = atoi(reply->element[j]->str);
+      } else {
+        cc_items[n-j].coconcurrency_count = 0;
+      }   
+    }
+
+    freeReplyObject(reply);
+    n -= batch_size;
+  }
+
+  free(cur_batch);
+
+
+
+  /* calculate similarities */
+  if(similarityFunc == 1)
+    calculate_jaccard(itemID, itemCount, cc_items, cc_items_size);
+
+  if(similarityFunc == 2)
+    calculate_cosine(itemID, itemCount, cc_items, cc_items_size);
+
+  
+  /* find the top x items with simple bubble sort */
+  for(i = 0; i < maxItems - 1; ++i){
+    for (j = 0; j < cc_items_size - i - 1; ++j){
+      if (cc_items[j].similarity > cc_items[j + 1].similarity){
+        struct cc_item tmp = cc_items[j];
+        cc_items[j] = cc_items[j + 1];
+        cc_items[j + 1] = tmp;
+      }
+    }
+  }
+
+
+  /* print top k items */
+  n = ((cc_items_size < maxItems) ? cc_items_size : maxItems);
+  for(j = 0; j < n; j++){
+    i = cc_items_size-j-1;
+    if(cc_items[i].similarity > 0){
+      print_item(cc_items[i]);      
+    }    
+  }
+
+
+  free(cc_items); 
+  return 0;
+}
+
+

data/ext/sort.c

@@ -0,0 +1,23 @@
+int lesser(int i1, int i2){
+  if(i1 > i2){
+    return i2;
+  } else {
+    return i1;
+  }
+}
+
+int rb_strcmp(char *str1, char *str2){
+  long len;
+  int retval;
+  len = lesser(strlen(str1), strlen(str2));
+  retval = memcmp(str1, str2, len);
+  if (retval == 0){
+    if (strlen(str1) == strlen(str2)) {      
+      return 0;
+    }
+    if (strlen(str1) > strlen(str2)) return 1;
+    return -1;
+  }
+  if (retval > 0) return 1;
+  return -1;
+}

data/ext/version.h

@@ -0,0 +1,17 @@
+#ifndef VERSION_H
+#define VERSION_H
+
+#define VERSION_MAJOR 0
+#define VERSION_MINOR 0
+#define VERSION_MICRO 1
+
+#define VERSION_STRING "recommendify_native %i.%i.%i\n" \
+ "\n" \
+ "Copyright © 2012\n" \
+ "  Paul Asmuth <paul@paulasmuth.com>\n" 
+
+#define USAGE_STRING "usage: %s " \
+ "{--version|--jaccard|--cosine} " \
+ "[redis_key] [item_id]\n"
+
+#endif

data/lib/recommendify/base.rb

@@ -0,0 +1,88 @@
+class Recommendify::Base
+
+  attr_reader :similarity_matrix, :input_matrices
+	attr_accessor :redis_prefix_self
+
+  @@max_neighbors = nil
+  @@input_matrices = {}
+
+  def self.max_neighbors(n=nil)    
+    return @@max_neighbors unless n
+    @@max_neighbors = n
+  end
+
+  def self.input_matrix(key, opts)
+    @@input_matrices[key] = opts
+  end
+
+  def self.input_matrices
+    @@input_matrices
+  end
+
+  def initialize(opts = nil)
+		@redis_prefix_self = opts[:redis_prefix] if opts && opts[:redis_prefix]
+    @input_matrices = Hash[self.class.input_matrices.map{ |key, opts| 
+      opts.merge!(:key => key, :redis_prefix => redis_prefix)
+      [ key, Recommendify::InputMatrix.create(opts) ]
+    }]
+    @similarity_matrix = Recommendify::SimilarityMatrix.new(
+      :max_neighbors => max_neighbors,
+      :key => :similarities,
+      :redis_prefix => redis_prefix
+    )
+  end
+
+  def redis_prefix
+    @redis_prefix_self || "recommendify"
+  end
+
+  def max_neighbors
+    self.class.max_neighbors || Recommendify::DEFAULT_MAX_NEIGHBORS
+  end
+
+  def method_missing(method, *args)
+    if @input_matrices.has_key?(method)
+      @input_matrices[method]
+    else
+      raise NoMethodError.new(method.to_s)
+    end
+  end
+
+  def respond_to?(method)
+    @input_matrices.has_key?(method) ? true : super
+  end
+
+  def all_items
+    @input_matrices.map{ |k,m| m.all_items }.flatten.uniq
+  end
+
+  def for(item_id)
+    similarity_matrix[item_id].map do |item_id, similarity|
+      Recommendify::Neighbor.new(
+        :item_id => item_id, 
+        :similarity => similarity
+      )
+    end.sort
+  end
+
+  def process!
+    all_items.each{ |item_id,n| process_item!(item_id) }
+  end
+
+  def process_item!(item_id)
+    input_matrices.map do |k,m|
+      neighbors = m.similarities_for(item_id).map do |i,w|        
+        [i,w*m.weight]
+      end
+      similarity_matrix.update(item_id, neighbors)
+    end
+    similarity_matrix.commit_item!(item_id)
+  end
+
+  def delete_item!(item_id)
+    input_matrices.map do |k,m|
+      m.delete_item(item_id)
+    end
+  end
+
+end

data/lib/recommendify/cc_matrix.rb

@@ -0,0 +1,51 @@
+module Recommendify::CCMatrix
+
+  def ccmatrix
+    @ccmatrix ||= Recommendify::SparseMatrix.new(
+      :redis_prefix => @opts.fetch(:redis_prefix),
+      :key => [@opts.fetch(:key), :ccmatrix].join(":")
+    )
+  end
+
+  def add_set(set_id, item_ids)
+    # FIXPAUL: forbid | and : in item_ids
+    item_ids.each do |item_id|
+      item_count_incr(item_id)
+    end
+    all_pairs(item_ids).map do |pair| 
+      i1, i2 = pair.split(":") 
+      ccmatrix.incr(i1, i2)
+    end
+  end
+
+  def add_single(set_id, item_id, other_item_ids)
+    item_count_incr(item_id)
+    other_item_ids.each do |other_item|
+      ccmatrix.incr(item_id, other_item)
+    end
+  end
+
+  def all_items
+    Recommendify.redis.hkeys(redis_key(:items))
+  end
+
+  def delete_item(item_id)    
+    Recommendify.redis.hdel(redis_key(:items), item_id)
+    ccmatrix.send(:k_delall, item_id)
+  end
+
+private
+
+  def all_pairs(keys)
+    keys.map{ |k1| (keys-[k1]).map{ |k2| [k1,k2].sort.join(":") } }.flatten.uniq
+  end
+
+  def item_count_incr(key)
+    Recommendify.redis.hincrby(redis_key(:items), key, 1)
+  end
+
+  def item_count(key)
+    Recommendify.redis.hget(redis_key(:items), key).to_i
+  end
+
+end

data/lib/recommendify/cosine_input_matrix.rb

@@ -0,0 +1,7 @@
+class Recommendify::CosineInputMatrix < Recommendify::InputMatrix
+
+  include Recommendify::CCMatrix
+  
+  # here be dragons ;)
+
+end

data/lib/recommendify/input_matrix.rb

@@ -0,0 +1,52 @@
+class Recommendify::InputMatrix
+
+  def self.create(opts)
+    klass = "#{Recommendify.capitalize(opts[:similarity_func])}InputMatrix"
+    Recommendify.constantize(klass.intern).new(opts)
+  end
+
+  def initialize(opts)
+    @opts = opts
+  end
+
+  def redis_key(append=nil)
+    [@opts.fetch(:redis_prefix), @opts.fetch(:key), append].flatten.compact.join(":")
+  end
+
+  def weight
+    (@opts[:weight] || 1).to_f
+  end
+
+  # add a set of item_ids to the matrix
+  def add_set(set_id, item_ids)
+    raise "implemented in subclass"
+  end
+
+  # add a single item to a set of item_ids to the matrix
+  def add_single(set_id, item_id, other_item_ids)
+    raise "implemented in subclass"
+  end
+
+  # calculate the similarity between item1 and item1 (0.0-1.0)
+  def similarity(item1, item2)
+    raise "implemented in subclass"
+  end
+
+  # calculate all similarities to other items in the matrix for item1
+  def similarities_for(item1)
+    # return => [ ["item23", 0.6], ["item42", 0.23], (...) ]
+    raise "implemented in subclass"
+  end
+
+  # retrieve all item_ids in the matrix
+  def all_items
+    # retzrb => [ "item23", "item42", "item17", (...) ]
+    raise "implemented in subclass"
+  end
+
+  # delete item_id from the matrix
+  def delete_item(item_id)
+    raise "implemented in subclass"
+  end
+
+end

data/lib/recommendify/jaccard_input_matrix.rb

@@ -0,0 +1,62 @@
+class Recommendify::JaccardInputMatrix < Recommendify::InputMatrix
+
+  include Recommendify::CCMatrix
+
+  def initialize(opts={})
+    check_native if opts[:native]
+    super(opts)
+  end
+
+  def similarity(item1, item2)
+    calculate_jaccard_cached(item1, item2)
+  end
+
+  def similarities_for(item1)
+    return run_native(item1) if @opts[:native]
+    calculate_similarities(item1)
+  end
+
+private
+
+  def calculate_similarities(item1)
+    (all_items - [item1]).map do |item2|
+      [item2, similarity(item1, item2)]
+    end
+  end
+
+  def calculate_jaccard_cached(item1, item2)
+    val = ccmatrix[item1, item2]
+    val.to_f / (item_count(item1)+item_count(item2)-val).to_f
+  end
+
+  def calculate_jaccard(set1, set2)
+    (set1&set2).length.to_f / (set1 + set2).uniq.length.to_f
+  end
+
+  def run_native(item_id)
+    res = %x{#{native_path} --jaccard "#{redis_key}" "#{item_id}" "#{redis_url}"}
+    raise "error: dirty exit (#{$?})" if $? != 0
+    res.split("\n").map do |line|
+      sim = line.match(/OUT: \(([^\)]*)\) \(([^\)]*)\)/)
+      unless sim
+        raise "error: #{res}" unless (res||"").include?('exit:')
+      else
+        [sim[1], sim[2].to_f]
+      end
+    end.compact
+  end
+
+  def check_native
+    return true if ::File.exists?(native_path)
+    raise "recommendify_native not found - you need to run rake build_native first"
+  end
+
+  def native_path
+    ::File.expand_path('../../../bin/recommendify', __FILE__)
+  end
+
+  def redis_url
+    Recommendify.redis.client.location
+  end
+
+end

data/lib/recommendify/neighbor.rb

@@ -0,0 +1,19 @@
+class Recommendify::Neighbor
+
+  def initialize(data)
+    @data = data
+  end
+
+  def item_id
+    @data.fetch(:item_id).to_s
+  end
+
+  def similarity
+    @data.fetch(:similarity)
+  end
+
+  def <=>(other)
+    other.similarity <=> self.similarity
+  end
+
+end

data/lib/recommendify/recommendify.rb

@@ -0,0 +1,25 @@
+module Recommendify
+
+  DEFAULT_MAX_NEIGHBORS = 50
+
+  @@redis = nil
+
+  def self.redis=(redis)
+    @@redis = redis
+  end
+
+  def self.redis
+    return @@redis unless @@redis.nil?
+    raise "redis not configured! - Recommendify.redis = Redis.new"
+  end
+
+  def self.capitalize(str_or_sym)
+  	str = str_or_sym.to_s.each_char.to_a
+  	str.first.upcase + str[1..-1].join("").downcase
+  end
+
+  def self.constantize(klass)
+    Object.module_eval("Recommendify::#{klass}", __FILE__, __LINE__)
+  end
+
+end