libsmatrix 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5ed4aa20301e6e40e3a9b6b422c081c9b4b00e07
+  data.tar.gz: 4b43c976b08df58841171aaa39919a885f7650f1
+SHA512:
+  metadata.gz: 7d647cd4bdfd5159f38e31955471418ec7a32df1e2d674de13a6c60a2f62ab2857becc4918fbe5d47a1b7f61bc8dcf4e15f6e11729f125f361c6a91e9de4a5dc
+  data.tar.gz: 0700cc4e0b013a5a86be04590e581165e7c8cd3ac4789e16e6f690ed422253308ec6368a8933a2d10009692b04962ce399922831c2e169cbed0ddfca00d9a863

data/Makefile

@@ -0,0 +1,52 @@
+# This file is part of the "libsmatrix" project
+#   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+#
+# Licensed under the MIT License (the "License"); you may not use this
+# file except in compliance with the License. You may obtain a copy of
+# the License at: http://opensource.org/licenses/MIT
+
+include src/Makefile.in
+
+SHELL        = /bin/sh
+CC           = clang
+CFLAGS_      = $(CFLAGS) -Wall -Wextra -O3 -march=native -mtune=native -D NDEBUG -fPIC
+LDFLAGS      = -lpthread -lm -lruby
+PREFIX       = $(DESTDIR)/usr/local
+LIBDIR       = $(PREFIX)/lib
+UNAME        = $(shell uname)
+SOURCES      = src/smatrix.c src/smatrix_jni.c src/smatrix_ruby.c
+
+all: src/smatrix.$(LIBEXT)
+
+src/smatrix.$(LIBEXT):
+	cd src && make
+
+install:
+	cp src/smatrix.$(LIBEXT) $(LIBDIR)
+
+clean:
+	find . -name "*.o" -o -name "*.class" -o -name "*.so" -o -name "*.dylib" -o -name "*.bundle" | xargs rm
+	rm -rf src/java/target src/config.h src/smatrix_benchmark *.gem
+
+ruby:
+	cd src/ruby && ruby extconf.rb
+	cd src/ruby && make
+
+publish_ruby:
+	gem build src/ruby/libsmatrix.gemspec
+	mv *.gem src/ruby/
+
+java:
+	cd src/java && make
+
+publish_java: java
+	cd src/java && mvn deploy
+
+benchmark: src/smatrix_benchmark
+	src/smatrix_benchmark full
+
+src/smatrix_benchmark:
+	cd src && make smatrix_benchmark
+
+test:
+	cd src/java && make test

data/README.md

@@ -0,0 +1,188 @@
+libsmatrix
+==========
+
+A thread-safe two dimensional sparse matrix data structure with C, Java and Ruby bindings.
+It was created to make loading and accessing medium sized (10GB+) matrices in boxed languages
+like Java/Scala or Ruby easier.
+
+While the chosen internal storage format (nested hashmaps) is neither the most memory-efficient
+nor extremely fast in terms of access/insert time it seems to be a good tradeoff between these
+two goals.
+
+A libsmatrix sparse matrix features two modes of operation; a memory-only mode in which all data
+is kept in main memory and a mode in which the data is stored on disk and only a pool of recently
+used rows is kept in memory. In this mode the data is persisted across program restarts. It also
+allows you to handle datasets larger than your available main memory.
+
+#### Documentation
+
++ [Getting Started](#getting-started)
++ [C API](#c-api)
++ [Java/Scala API](#fnord)
++ [Ruby API](#ruby-api)
++ [Internals](#internals)
++ [Benchmarks](#benchmarks)
++ [Examples](#examples)
++ [License](#license)
+
+
+Getting Started (Building)
+--------------------------
+
+There are multiple ways to install libsmatrix:
+
+### Compile from source
+
+This will produce a single shared object "smatrix.so" file that exports all calls documented
+in "C API".
+
+    $ make
+    $ make install
+
+To run the tests/benchmarks (optional, requires java and ruby)
+
+    $ make test
+    $ make benchmark
+
+To build the MRI ruby and Java JNI bindings (optional), run:
+
+    $ make ruby
+    $ make java
+
+This will produce the respective shared objects and bundles in:
+
+    src/ruby/smatrix_ruby.so
+    src/ruby/smatrix_X.X.X.gem
+
+    src/java/smatrix_java.so
+    src/java/target/libsmatrix-X.X-SNAPSHOT.jar
+
+### Import artifact via Maven/sbt (java/scala)
+
+Currently the maven artifact only contains the binding glue code and doesn't actually build
+the native shared object. You need to compile & install "libsmatrix.so" yourself on the target
+host, otherwise you'll get a "UnsatisfiedLinkError".
+
+Import artifact via sbt:
+
+    resolvers += "sbt-libsmatrix-repo" at "https://raw.github.com/paulasmuth/libsmatrix/mvn-repo/"
+
+    libraryDependencies += "com.paulasmuth.libsmatrix" % "libsmatrix" % "0.2-SNAPSHOT"
+
+Import artifact via Maven2 (put this into your pom.xml):
+
+    <repository>
+      <id>libsmatrix-mvn-repo</id>
+      <url>https://raw.github.com/paulsmuth/libsmatrix/mvn-repo/</url>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>always</updatePolicy>
+      </snapshots>
+    </repository>
+
+To publish the maven artifact from source, check out libsmatrix and run this:
+
+    $ make publish_java
+
+
+### Import gem via rubygems (ruby only)
+
+This will install the ruby bindings and compile the native shared object:
+
+    $ gem install libsmatrix
+
+To use libsmatrix in your project, require it like this:
+
+    require "libsmatrix"
+
+To build and publish the ruby gem run:
+
+    $ make publish_ruby
+
+C API
+-----
+
+Open a smatrix (if filename is NULL, use in memory only mode; otherwise open or create file)
+
+    smatrix_t* smatrix_open(const char* fname);
+
+Close a smatrix:
+
+    void smatrix_close(smatrix_t* self);
+
+Get, Set, Increment, Decrement a (x,y) position. _All of the methods are threadsafe_
+
+    uint32_t smatrix_get(smatrix_t* self, uint32_t x, uint32_t y);
+    uint32_t smatrix_set(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value);
+    uint32_t smatrix_incr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value);
+    uint32_t smatrix_decr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value);
+
+Get a whole "row" of the matrix by row coordinate x. _All of the methods are threadsafe_
+
+    uint32_t smatrix_rowlen(smatrix_t* self, uint32_t x);
+    uint32_t smatrix_getrow(smatrix_t* self, uint32_t x, uint32_t* ret, size_t ret_len);
+
+
+Java / Scala API
+----------------
+
+here be dragons
+
+
+Ruby API
+--------
+
+
+Require the gem:
+
+	$ require 'libsmatrix'
+
+
+Create a new smatrix instance:
+
+	$ smatrix = SparseMatrix.new("/path/to/smatrix.smx")
+	
+Get, Set, Increment, Decrement a (x,y) position
+
+	$ smatrix.set(x, y, 5)
+	=> 5
+	$ smatrix.get(x, y)
+	=> 5
+	$ smatrix.incr(x, y, 1)
+	=> 6
+	$ smatrix.decr(x, y, 1)
+	=> 5
+	
+Close and free the matrix (data is persisted to disk):
+
+	$ smatrix = nil
+
+
+Benchmarks
+----------
+
+**No big-data disclaimer:** We are using this code to run a Collaborative Filtering
+recommendation engine for one of Germany's largest ecommerce sites. It is tested on "small-data"
+datasets with up to 40GB per matrix (1.5 billion values in 13 million rows). If your data is
+actually much bigger (measured in terrabytes, not gigabytes) this library is not for you.
+
+here be dragons
+
+
+Examples
+-------
+
++ There is a simple example in src/smatrix_example.c
++ There is a simple Collaborative Filtering based recommendation engine in src/smatrix_example_recommender.c
+
+
+License
+-------
+
+Copyright (c) 2011 Paul Asmuth
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to use, copy and modify copies of the Software, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/examples/.gitignore

@@ -0,0 +1,2 @@
+*.o
+*.class

data/examples/CFRecommender.java

@@ -0,0 +1,24 @@
+/**
+ * This file is part of the "libsmatrix" project
+ *   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+ *
+ * Licensed under the MIT License (the "License"); you may not use this
+ * file except in compliance with the License. You may obtain a copy of
+ * the License at: http://opensource.org/licenses/MIT
+ */
+import com.paulasmuth.libsmatrix.SparseMatrix;
+
+/**
+ * Compile & run this example:
+ *
+ *  $ javac CFRecommender.java && java CFRecommeder
+ *
+ */
+class CFRecommender {
+
+  public static void main(String[] opts) {
+    SparseMatrix smx = new SparseMatrix();
+    smx.test();
+  }
+
+}

data/examples/cf_recommender.c

@@ -0,0 +1,87 @@
+// This file is part of the "libsmatrix" project
+//   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+//
+// Licensed under the MIT License (the "License"); you may not use this
+// file except in compliance with the License. You may obtain a copy of
+// the License at: http://opensource.org/licenses/MIT
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pthread.h>
+
+#include "smatrix.h"
+
+smatrix_t* my_smatrix;
+
+// libsmatrix example: simple CF based recommendation engine
+int main(int argc, char **argv) {
+  my_smatrix = smatrix_open(NULL);
+
+  // one preference set = list of items in one session
+  // e.g. list of viewed items by the same user
+  // e.g. list of bought items in the same checkout
+  uint32_t input_ids[5] = {12,52,63,76,43};
+  import_preference_set(input_ids, 5);
+
+  // generate recommendations (similar items) for item #76
+  void neighbors_for_item(76);
+
+  smatrix_close(my_smatrix);
+  return 0;
+}
+
+// train / add a preference set (list of items in one session)
+void import_preference_set(uint32_t* ids, uint32_t num_ids) {
+  uint32_t i, n;
+
+  for (n = 0; n < num_ids; n++) {
+    smatrix_incr(my_smatrix, ids[n], 0, 1);
+
+    for (i = 0; i < pset->len; i++) {
+      if (i != n) {
+        smatrix_incr(my_smatrix, ids[n], ids[i], 1);
+      }
+    }
+  }
+}
+
+// get recommendations for item with id "item_id"
+void neighbors_for_item(uint32_t item_id)
+  uint32_t neighbors, *row, total;
+
+  total = smatrix_get(my_smatrix, item_id, 0);
+  neighbors = smatrix_getrow(my_smatrix, item_id, row, 8192);
+
+  for (pos = 0; pos < neighbors; pos++) {
+    uint32_t cur_id = row[pos * 2];
+
+    printf("found neighbor for item %u: item %u with distance %f\n",
+      item_id, cf_cosine(smatrix, cur_id, row[pos * 2 + 1], total));
+  }
+
+  free(row);
+}
+
+// calculates the cosine vector distance between two items
+double cf_cosine(smatrix_t* smatrix, uint32_t b_id, uint32_t cc_count, uint32_t a_total) {
+  uint32_t b_total;
+  double num, den;
+
+  b_total = smatrix_get(smatrix, b_id, 0);
+
+  if (b_total == 0)
+    b_total = 1;
+
+  num = cc_count;
+  den = sqrt((double) a_total) * sqrt((double) b_total);
+
+  if (den == 0.0)
+    return 0.0;
+
+  if (num > den)
+    return 0.0;
+
+  return (num / den);
+}
+

data/examples/smatrix_example.c

@@ -0,0 +1,75 @@
+// This file is part of the "libsmatrix" project
+//   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+//
+// Licensed under the MIT License (the "License"); you may not use this
+// file except in compliance with the License. You may obtain a copy of
+// the License at: http://opensource.org/licenses/MIT
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pthread.h>
+
+#include "smatrix.h"
+
+smatrix_t* db;
+
+void* test(void* fnord) {
+  uint64_t i, n, m;
+
+  for (m = 0; m < 100; m++) {
+    for (n = 1; n < 30; n++) {
+      for (i = 1; i < 50; i++) {
+        smatrix_incr(db, n, i, 1);
+      }
+    }
+  }
+
+  return NULL;
+}
+
+int main(int argc, char **argv) {
+  int i,n,m,l,x=0, num_threads = 4;
+  pthread_t threads[num_threads];
+
+  printf("\nloading\n");
+  db = smatrix_open("/var/tmp/reco.db");
+  //db = smatrix_open(NULL); // in-memory only mode
+
+  if (db == NULL)
+    abort();
+
+  printf("\nstarting\n");
+
+  for (n = 0; n < num_threads; n++)
+    pthread_create(&threads[n], NULL, test, NULL);
+
+  for (n = 0; n < num_threads; n++)
+    pthread_join(threads[n], NULL);
+
+  printf("\ndone\n");
+
+  for (n = 1; n < 30; n++) {
+    for (i = 1; i < 50; i++) {
+      printf("(%u,%u) => %u, ", n, i, smatrix_get(db, n, i));
+      if (x++ % 5 == 0) printf("\n");
+    }
+  }
+
+  printf("rowlen: %u\n", l = smatrix_rowlen(db, 23));
+  size_t bytes = sizeof(uint32_t) * l * 2;
+  uint32_t* data = malloc(bytes);
+
+  l = smatrix_getrow(db, 23, data, bytes);
+
+  for (i = 0; i < l; i++) {
+    printf("%u => %u, ", data[i * 2], data[i * 2 + 1]);
+  }
+
+  printf("\n");
+
+  smatrix_close(db);
+  printf("in use at exit: %lu\n", db->mem);
+
+  return 0;
+}

data/src/Makefile

@@ -0,0 +1,28 @@
+# This file is part of the "libsmatrix" project
+#   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+#
+# Licensed under the MIT License (the "License"); you may not use this
+# file except in compliance with the License. You may obtain a copy of
+# the License at: http://opensource.org/licenses/MIT
+
+include Makefile.in
+
+TARGET       = smatrix.$(LIBEXT)
+
+all: $(TARGET)
+
+$(TARGET): smatrix.o
+	$(CC) $(LIBFLAGS) smatrix.o -o $(TARGET) $(LDFLAGS)
+
+smatrix.o: config.h smatrix.c smatrix.h smatrix_private.h
+	$(CC) -c $(CFLAGS) smatrix.c -o smatrix.o
+
+config.h:
+	touch config.h
+
+smatrix_jni.h:
+	javac com/paulasmuth/libsmatrix/SparseMatrix.java
+	javah -o smatrix_jni.h -classpath . com.paulasmuth.libsmatrix.SparseMatrix
+
+smatrix_benchmark: smatrix.o smatrix_benchmark.c
+	$(CC) $(CFLAGS) smatrix_benchmark.c smatrix.o -o smatrix_benchmark $(LDFLAGS)