recommendify-ruby 0.3.8

data/ext/cc_item.h

@@ -0,0 +1,8 @@
+#define ITEM_ID_SIZE 64
+
+struct cc_item {         
+  char  item_id[ITEM_ID_SIZE];
+  int   coconcurrency_count;
+  int   total_count;
+  float similarity;  
+};         

data/ext/cosine.c

@@ -0,0 +1,3 @@
+void calculate_cosine(char *item_id, int itemCount, struct cc_item *cc_items, int cc_items_size){
+  /* here be dragons */
+}

data/ext/extconf.rb

@@ -0,0 +1,18 @@
+makefile = <<-MAKEFILE
+all: prepare build
+
+build:
+	gcc -Wall recommendify.c -lhiredis -o ../bin/recommendify
+
+prepare:
+	mkdir -p ../bin
+
+clean:
+	rm -f *.o
+
+install: prepare
+MAKEFILE
+  
+File.open(::File.expand_path("../Makefile", __FILE__), "w+") do |f|   
+  f.write(makefile)
+end

data/ext/iikey.c

@@ -0,0 +1,18 @@
+char* item_item_key(char *item1, char *item2){
+  int keylen = strlen(item1) + strlen(item2) + 2;
+  char *key = (char *)malloc(keylen * sizeof(char));
+
+  if(!key){
+    printf("cannot allocate\n");
+    return 0;
+  }  
+
+  // FIXPAUL: make shure this does exactly the same as ruby sort  
+  if(rb_strcmp(item1, item2) <= 0){
+    snprintf(key, keylen, "%s:%s", item1, item2);
+  } else {
+    snprintf(key, keylen, "%s:%s", item2, item1);
+  }
+
+  return key;
+}

data/ext/jaccard.c

@@ -0,0 +1,19 @@
+void calculate_jaccard(char *item_id, int itemCount, struct cc_item *cc_items, int cc_items_size){
+  int j, n;
+
+  for(j = 0; j < cc_items_size; j++){
+    n = cc_items[j].coconcurrency_count;
+    if(n>0){
+      cc_items[j].similarity = (
+        (float)n / (
+          (float)itemCount + 
+          (float)cc_items[j].total_count - 
+          (float)n
+        )
+      );
+    } else {
+      cc_items[j].similarity = 0.0;
+    }
+  }  
+
+}

data/ext/output.c

@@ -0,0 +1,22 @@
+int print_version(){
+  printf(
+    VERSION_STRING, 
+    VERSION_MAJOR, 
+    VERSION_MINOR, 
+    VERSION_MICRO
+  );
+  return 0;
+}
+
+int print_usage(char *bin){
+  printf(USAGE_STRING, bin);
+  return 1;
+}
+
+void print_item(struct cc_item item){
+  printf(
+    "OUT: (%s) (%.4f)\n", 
+    item.item_id, 
+    item.similarity
+  );
+}

data/ext/recommendify.c

@@ -0,0 +1,214 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <hiredis/hiredis.h>
+
+#include "version.h"
+#include "cc_item.h" 
+#include "jaccard.c" 
+#include "cosine.c" 
+#include "output.c" 
+#include "sort.c" 
+#include "iikey.c" 
+
+
+int main(int argc, char **argv){
+  int i, j, n, similarityFunc = 0;    
+  int itemCount = 0;  
+  char *itemID;  
+  char *redisPrefix;
+  redisContext *c;
+  redisReply *all_items;
+  redisReply *reply;
+  int cur_batch_size;
+  char* cur_batch;
+  char *iikey;
+
+  int batch_size = 200; /* FIXPAUL: make option */
+  int maxItems = 50; /* FIXPAUL: make option */
+  
+  struct {
+    char  host[1024];
+    int   port;
+  } redis_addr;
+
+  /* option parsing */
+  if(argc < 2)
+    return print_usage(argv[0]);
+
+  if(!strcmp(argv[1], "--version"))
+    return print_version();
+
+  if(!strcmp(argv[1], "--jaccard")) 
+    similarityFunc = 1;
+
+  if(!strcmp(argv[1], "--cosine"))  
+    similarityFunc = 2;
+
+  if(!similarityFunc){
+    printf("invalid option: %s\n", argv[1]);
+    return 1;
+  } else if(argc < 4 || argc > 5){
+    printf("wrong number of arguments\n");
+    print_usage(argv[0]);
+    return 1;
+  }
+
+  redisPrefix = argv[2];
+  itemID = argv[3];
+  redis_addr.host[0] = 0;
+  redis_addr.port = 0;
+
+  /* configure redis location */
+  if(argc > 4){
+    char* has_port = strchr(argv[4], ':');
+    if(has_port){
+      strncpy(redis_addr.host, argv[4], strlen(argv[4]) - strlen(has_port));
+      redis_addr.host[strlen(argv[4]) - strlen(has_port)] = 0;
+      redis_addr.port = atoi(has_port + 1);
+    } else {
+      strncpy(redis_addr.host, argv[4], sizeof(redis_addr.host));
+    }
+  }
+
+  /* default redis location */
+  if(strlen(redis_addr.host) == 0)
+    strcpy(redis_addr.host, "localhost");
+
+  if(!redis_addr.port)
+    redis_addr.port = 6379;
+
+  /* connect to redis */
+  struct timeval timeout = { 1, 500000 }; 
+  c = redisConnectWithTimeout(redis_addr.host, redis_addr.port, timeout);
+
+  if(c->err){
+    printf("connection to redis failed: %s\n", c->errstr);
+    return 1;
+  }
+
+
+  /* get item count */
+  reply = redisCommand(c,"HGET %s:items %s", redisPrefix, itemID);    
+
+  if(reply->str){
+    itemCount = atoi(reply->str);
+  } else {
+    itemCount = 0;
+  }
+
+  freeReplyObject(reply);
+
+  if(itemCount < 2){
+    printf("exit: item count is zero or one\n");
+    return 0;
+  }
+  
+
+  /* get all items_ids and the total counts */ 
+  all_items = redisCommand(c,"HGETALL %s:items", redisPrefix);
+
+  if(all_items->type != REDIS_REPLY_ARRAY)
+    return 1;
+
+
+  /* populate the cc_items array */ 
+  int cc_items_size = all_items->elements / 2;  
+  int cc_items_mem = cc_items_size * sizeof(struct cc_item);
+  struct cc_item *cc_items = malloc(cc_items_mem);
+  cc_items_size--;
+
+  if(!cc_items){    
+    printf("cannot allocate memory: %i", cc_items_mem);
+    return 1;
+  }
+  
+  i = 0;
+  for (j = 0; j < all_items->elements/2; j++){                   
+    if(strcmp(itemID, all_items->element[j*2]->str) != 0){      
+      strncpy(cc_items[i].item_id, all_items->element[j*2]->str, ITEM_ID_SIZE);
+      cc_items[i].total_count = atoi(all_items->element[j*2+1]->str);
+      i++;
+    }
+  }
+
+  freeReplyObject(all_items);
+
+
+  // batched redis hmgets on the ccmatrix
+  cur_batch = (char *)malloc(((batch_size * (ITEM_ID_SIZE + 4) * 2) + 100) * sizeof(char));
+
+  if(!cur_batch){    
+    printf("cannot allocate memory");
+    return 1;
+  }
+
+  n = cc_items_size;
+  while(n >= 0){
+    cur_batch_size = ((n-1 < batch_size) ? n-1 : batch_size);
+    sprintf(cur_batch, "HMGET %s:ccmatrix ", redisPrefix);
+
+    for(i = 0; i < cur_batch_size; i++){
+      iikey = item_item_key(itemID, cc_items[n-i].item_id);
+
+      strcat(cur_batch, iikey);
+      strcat(cur_batch, " ");
+
+      if(iikey)
+        free(iikey);
+    }    
+
+    redisAppendCommand(c, cur_batch);  
+    redisGetReply(c, (void**)&reply);
+      
+    for(j = 0; j < reply->elements; j++){
+      if(reply->element[j]->str){
+        cc_items[n-j].coconcurrency_count = atoi(reply->element[j]->str);
+      } else {
+        cc_items[n-j].coconcurrency_count = 0;
+      }   
+    }
+
+    freeReplyObject(reply);
+    n -= batch_size;
+  }
+
+  free(cur_batch);
+
+
+
+  /* calculate similarities */
+  if(similarityFunc == 1)
+    calculate_jaccard(itemID, itemCount, cc_items, cc_items_size);
+
+  if(similarityFunc == 2)
+    calculate_cosine(itemID, itemCount, cc_items, cc_items_size);
+
+  
+  /* find the top x items with simple bubble sort */
+  for(i = 0; i < maxItems - 1; ++i){
+    for (j = 0; j < cc_items_size - i - 1; ++j){
+      if (cc_items[j].similarity > cc_items[j + 1].similarity){
+        struct cc_item tmp = cc_items[j];
+        cc_items[j] = cc_items[j + 1];
+        cc_items[j + 1] = tmp;
+      }
+    }
+  }
+
+
+  /* print top k items */
+  n = ((cc_items_size < maxItems) ? cc_items_size : maxItems);
+  for(j = 0; j < n; j++){
+    i = cc_items_size-j-1;
+    if(cc_items[i].similarity > 0){
+      print_item(cc_items[i]);      
+    }    
+  }
+
+
+  free(cc_items); 
+  return 0;
+}
+
+

data/ext/sort.c

@@ -0,0 +1,23 @@
+int lesser(int i1, int i2){
+  if(i1 > i2){
+    return i2;
+  } else {
+    return i1;
+  }
+}
+
+int rb_strcmp(char *str1, char *str2){
+  long len;
+  int retval;
+  len = lesser(strlen(str1), strlen(str2));
+  retval = memcmp(str1, str2, len);
+  if (retval == 0){
+    if (strlen(str1) == strlen(str2)) {      
+      return 0;
+    }
+    if (strlen(str1) > strlen(str2)) return 1;
+    return -1;
+  }
+  if (retval > 0) return 1;
+  return -1;
+}

data/ext/version.h

@@ -0,0 +1,17 @@
+#ifndef VERSION_H
+#define VERSION_H
+
+#define VERSION_MAJOR 0
+#define VERSION_MINOR 0
+#define VERSION_MICRO 1
+
+#define VERSION_STRING "recommendify_native %i.%i.%i\n" \
+ "\n" \
+ "Copyright © 2012\n" \
+ "  Paul Asmuth <paul@paulasmuth.com>\n" 
+
+#define USAGE_STRING "usage: %s " \
+ "{--version|--jaccard|--cosine} " \
+ "[redis_key] [item_id]\n"
+
+#endif

data/lib/recommendify.rb

@@ -0,0 +1,9 @@
+require "recommendify/recommendify"
+require "recommendify/sparse_matrix"
+require "recommendify/cc_matrix"
+require "recommendify/similarity_matrix"
+require "recommendify/input_matrix"
+require "recommendify/jaccard_input_matrix"
+require "recommendify/cosine_input_matrix"
+require "recommendify/base"
+require "recommendify/neighbor"

data/lib/recommendify/base.rb

@@ -0,0 +1,86 @@
+class Recommendify::Base
+
+  attr_reader :similarity_matrix, :input_matrices
+
+  @@max_neighbors = nil
+  @@input_matrices = {}
+
+  def self.max_neighbors(n=nil)    
+    return @@max_neighbors unless n
+    @@max_neighbors = n
+  end
+
+  def self.input_matrix(key, opts)
+    @@input_matrices[key] = opts
+  end
+
+  def self.input_matrices
+    @@input_matrices
+  end
+
+  def initialize    
+    @input_matrices = Hash[self.class.input_matrices.map{ |key, opts| 
+      opts.merge!(:key => key, :redis_prefix => redis_prefix)
+      [ key, Recommendify::InputMatrix.create(opts) ]
+    }]
+    @similarity_matrix = Recommendify::SimilarityMatrix.new(
+      :max_neighbors => max_neighbors,
+      :key => :similarities,
+      :redis_prefix => redis_prefix
+    )
+  end
+
+  def redis_prefix
+    "recommendify"
+  end
+
+  def max_neighbors
+    self.class.max_neighbors || Recommendify::DEFAULT_MAX_NEIGHBORS
+  end
+
+  def method_missing(method, *args)
+    if @input_matrices.has_key?(method)
+      @input_matrices[method]
+    else
+      raise NoMethodError.new(method.to_s)
+    end
+  end
+
+  def respond_to?(method)
+    @input_matrices.has_key?(method) ? true : super
+  end
+
+  def all_items
+    @input_matrices.map{ |k,m| m.all_items }.flatten.uniq
+  end
+
+  def for(item_id)
+    similarity_matrix[item_id].map do |item_id, similarity|
+      Recommendify::Neighbor.new(
+        :item_id => item_id, 
+        :similarity => similarity
+      )
+    end.sort
+  end
+
+  def process!
+    all_items.each{ |item_id,n| process_item!(item_id) }
+  end
+
+  def process_item!(item_id)
+    input_matrices.map do |k,m|
+      neighbors = m.similarities_for(item_id).map do |i,w|        
+        [i,w*m.weight]
+      end
+      similarity_matrix.update(item_id, neighbors)
+    end
+    similarity_matrix.commit_item!(item_id)
+  end
+
+  def delete_item!(item_id)
+    input_matrices.map do |k,m|
+      m.delete_item(item_id)
+    end
+  end
+
+end