libsmatrix 0.0.1

data/src/smatrix.h

@@ -0,0 +1,96 @@
+// This file is part of the "libsmatrix" project
+//   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+//
+// Licensed under the MIT License (the "License"); you may not use this
+// file except in compliance with the License. You may obtain a copy of
+// the License at: http://opensource.org/licenses/MIT
+
+#include <stdint.h>
+#include <pthread.h>
+#include <stdio.h>
+
+#ifndef SMATRIX_H
+#define SMATRIX_H
+
+#define SMATRIX_META_SIZE 512
+#define SMATRIX_RMAP_FLAG_LOADED 4
+#define SMATRIX_RMAP_FLAG_DIRTY 8
+#define SMATRIX_RMAP_FLAG_RESIZED 16
+#define SMATRIX_RMAP_MAGIC "\x23\x23\x23\x23\x23\x23\x23\x23"
+#define SMATRIX_RMAP_MAGIC_SIZE 8
+#define SMATRIX_RMAP_INITIAL_SIZE 16
+#define SMATRIX_RMAP_SLOT_SIZE 8
+#define SMATRIX_RMAP_HEAD_SIZE 16
+#define SMATRIX_CMAP_INITIAL_SIZE 65536
+#define SMATRIX_CMAP_SLOT_SIZE 12
+#define SMATRIX_CMAP_HEAD_SIZE 16
+#define SMATRIX_CMAP_BLOCK_SIZE 4194304
+#define SMATRIX_CMAP_SLOT_USED 1
+
+typedef struct {
+  volatile uint16_t    count;
+  volatile uint16_t    mutex;
+} smatrix_lock_t;
+
+typedef struct {
+  uint32_t             key;
+  uint32_t             value;
+} smatrix_rmap_slot_t;
+
+typedef struct {
+  uint64_t             fpos;
+  uint64_t             meta_fpos;
+  uint32_t             size;
+  uint32_t             used;
+  uint32_t             key;
+  uint32_t             flags;
+  smatrix_rmap_slot_t* data;
+  smatrix_lock_t       lock;
+} smatrix_rmap_t;
+
+typedef struct {
+  uint32_t             flags;
+  uint32_t             key;
+  smatrix_rmap_t*      rmap;
+} smatrix_cmap_slot_t;
+
+typedef struct {
+  uint64_t             size;
+  uint64_t             used;
+  uint64_t             block_fpos;
+  uint64_t             block_used;
+  uint64_t             block_size;
+  smatrix_cmap_slot_t* data;
+  smatrix_lock_t       lock;
+} smatrix_cmap_t;
+
+typedef struct smatrix_ref_s smatrix_ref_t;
+
+struct smatrix_ref_s {
+  int                  write;
+  smatrix_rmap_t*      rmap;
+  smatrix_rmap_slot_t* slot;
+  smatrix_ref_t*       next;
+};
+
+typedef struct {
+  int                  fd;
+  int                  shutdown;
+  uint64_t             fpos;
+  uint64_t             mem;
+  smatrix_ref_t*       ioqueue;
+  pthread_t            iothread;
+  smatrix_cmap_t       cmap;
+  smatrix_lock_t       lock;
+} smatrix_t;
+
+smatrix_t* smatrix_open(const char* fname);
+uint32_t smatrix_get(smatrix_t* self, uint32_t x, uint32_t y);
+uint32_t smatrix_set(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value);
+uint32_t smatrix_incr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value);
+uint32_t smatrix_decr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value);
+uint32_t smatrix_rowlen(smatrix_t* self, uint32_t x);
+uint32_t smatrix_getrow(smatrix_t* self, uint32_t x, uint32_t* ret, size_t ret_len);
+void smatrix_close(smatrix_t* self);
+
+#endif

data/src/smatrix_benchmark.c

@@ -0,0 +1,236 @@
+// This file is part of the "libsmatrix" project
+//   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+//
+// Licensed under the MIT License (the "License"); you may not use this
+// file except in compliance with the License. You may obtain a copy of
+// the License at: http://opensource.org/licenses/MIT
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <assert.h>
+#include <pthread.h>
+
+#include "smatrix.h"
+
+typedef struct {
+  smatrix_t* smx;
+  int        threadn;
+  int        user1;
+} args_t;
+
+smatrix_t* smx_mem;
+
+void* benchmark_incr_mixed(void* args_) {
+  args_t* args = (args_t*) args_;
+  int i, n, r, o;
+
+  o = 42 + args->threadn;
+  smatrix_t* smx = args->smx;
+
+  for (r = 0; r < args->user1; r++) {
+    for (n = 0; n < 23; n++) {
+      for (i = 0; i < 22; i++) {
+        smatrix_incr(smx, n + o, i + o, 1);
+        smatrix_incr(smx, i + o, n + o, 1);
+      }
+    }
+  }
+
+  return NULL;
+}
+
+void* benchmark_get_mixed(void* args_) {
+  args_t* args = (args_t*) args_;
+  int i, n, r, o;
+
+  o = 42 + args->threadn;
+  smatrix_t* smx = args->smx;
+
+  for (r = 0; r < args->user1; r++) {
+    for (n = 0; n < 23; n++) {
+      for (i = 0; i < 22; i++) {
+        smatrix_get(smx, n + o, i + o);
+        smatrix_get(smx, i + o, n + o);
+      }
+    }
+  }
+
+  return NULL;
+}
+
+void* benchmark_incr_independent(void* args_) {
+  args_t* args = (args_t*) args_;
+  int i, r, o;
+
+  smatrix_t* smx = args->smx;
+  o = 123 + args->threadn;
+
+  for (r = 0; r < args->user1; r++) {
+    for (i = 0; i < 1000; i++) {
+      smatrix_incr(smx, o, 1, 1);
+    }
+  }
+
+  return NULL;
+}
+
+void* benchmark_incr_compete(void* args_) {
+  args_t* args = (args_t*) args_;
+  int i, r;
+
+  smatrix_t* smx = args->smx;
+
+  for (r = 0; r < args->user1; r++) {
+    for (i = 0; i < 1000; i++) {
+      smatrix_incr(smx, 23, 1, 1);
+    }
+  }
+
+  return NULL;
+}
+
+void measure(void* (*cb)(void*), int nthreads, smatrix_t* smx, int user1) {
+  char str[20] = "           ";
+  int n;
+  pthread_t* threads;
+  double elapsed;
+  struct timeval t0, t1;
+  void* retval;
+
+  args_t* args = malloc(sizeof(args_t) * nthreads);
+  threads = malloc(sizeof(pthread_t) * nthreads);
+
+  gettimeofday(&t0, NULL);
+
+  for (n = 0; n < nthreads; n++) {
+    args[n].smx     = smx;
+    args[n].threadn = n;
+    args[n].user1   = user1;
+    pthread_create(threads + n, NULL, cb, &args[n]);
+  }
+
+  for (n = 0; n < nthreads; n++) {
+    pthread_join(threads[n], &retval);
+  }
+
+  gettimeofday(&t1, NULL);
+
+  elapsed  = (double) (t1.tv_sec - t0.tv_sec)   * 1000;
+  elapsed += (double) (t1.tv_usec - t0.tv_usec) / 1000;
+
+  str[snprintf(str, 20, "%.1fms", elapsed)] = ' ';
+
+  free(threads);
+  free(args);
+  printf("%s", str);
+}
+
+void print_header(const char* title) {
+  printf("TEST: %s\n", title);
+  printf("---------------------------------------------------------------\n");
+  printf("T=1        T=2        T=4        T=8        T=16       T=32     \n");
+}
+
+void test_incr(smatrix_t* smx_mem) {
+  int n, max = 10;
+
+  for (n = 0; n < max; n++) {
+    measure(&benchmark_incr_mixed, 1,  smx_mem, 1024);
+    measure(&benchmark_incr_mixed, 2,  smx_mem, 512);
+    measure(&benchmark_incr_mixed, 4,  smx_mem, 256);
+    measure(&benchmark_incr_mixed, 8,  smx_mem, 128);
+    measure(&benchmark_incr_mixed, 16, smx_mem, 64);
+    measure(&benchmark_incr_mixed, 32, smx_mem, 32);
+
+    if (n < max - 1) {
+      printf("\n");
+    }
+  }
+
+  printf("\n\n");
+}
+
+void test_get(smatrix_t* smx_mem) {
+  int n, max = 10;
+
+  for (n = 0; n < max; n++) {
+    measure(&benchmark_get_mixed, 1,  smx_mem, 1024);
+    measure(&benchmark_get_mixed, 2,  smx_mem, 512);
+    measure(&benchmark_get_mixed, 4,  smx_mem, 256);
+    measure(&benchmark_get_mixed, 8,  smx_mem, 128);
+    measure(&benchmark_get_mixed, 16, smx_mem, 64);
+    measure(&benchmark_get_mixed, 32, smx_mem, 32);
+
+    if (n < max - 1) {
+      printf("\n");
+    }
+  }
+
+  printf("\n\n");
+}
+
+int main(int argc, char** argv) {
+  int num = 1024, threads = 4;
+  smatrix_t* smx = NULL;
+
+  if (argc < 2) {
+    printf("usage: smatrix_benchmark [test] [times] [threads] [file]\n\n");
+    printf("  Available Tests:\n");
+    printf("    full      test all methods\n");
+    printf("    incr      test the incr method\n");
+    printf("    get       test the get method\n\n");
+    printf("  Examples:\n");
+    printf("    $ smatrix_benchmark incr 1024 4\n");
+    printf("    $ smatrix_benchmark incr 1024 4 /tmp/test.smx\n");
+    printf("    $ smatrix_benchmark get 10000 2 /tmp/text.smx\n");
+    printf("    $ smatrix_benchmark full\n\n");
+    return 1;
+  }
+
+  if (argc > 2) {
+    num = atoi(argv[2]);
+  }
+
+  if (argc > 3) {
+    threads = atoi(argv[3]);
+  }
+
+  if (argc > 4) {
+    smx = smatrix_open(argv[4]);
+  } else {
+    smx = smatrix_open(NULL);
+  }
+
+  if (!strcmp(argv[1], "incr")) {
+    printf("testing: %ik x incr @ %i threads: ", num, threads);
+    measure(&benchmark_incr_mixed, threads, smx, num / threads);
+    printf("\n");
+    goto exit;
+  }
+
+  if (!strcmp(argv[1], "get")) {
+    printf("testing: %ik x get @ %i threads: ", num, threads);
+    measure(&benchmark_get_mixed, threads, smx, num / threads);
+    printf("\n");
+    goto exit;
+  }
+
+  printf("libsmatrix benchmark [date]\n\n");
+
+  print_header("1 million x incr (memory)");
+  test_incr(smx);
+
+  print_header("1 million x get (memory)");
+  test_get(smx);
+
+exit:
+  smatrix_close(smx);
+
+  return 0;
+}

data/src/smatrix_jni.c

@@ -0,0 +1,161 @@
+/**
+ * This file is part of the "libsmatrix" project
+ *   (c) 2011-2013 Paul Asmuth <paul@paulasmuth.com>
+ *
+ * Licensed under the MIT License (the "License"); you may not use this
+ * file except in compliance with the License. You may obtain a copy of
+ * the License at: http://opensource.org/licenses/MIT
+ */
+#include <stdlib.h>
+#include "smatrix_jni.h"
+#include "smatrix.h"
+
+#define _JM(X) Java_com_paulasmuth_libsmatrix_SparseMatrix_##X
+#define ERR_PTRNOTFOUND "can't find native object. maybe close() was already called"
+
+void throw_exception(JNIEnv* env, const char* error) {
+  jclass exception = (*env)->FindClass(env, "java/lang/IllegalArgumentException");
+  (*env)->ThrowNew(env, exception, error);
+}
+
+void set_ptr(JNIEnv* env, jobject self, void* ptr_) {
+  jclass   cls;
+  jfieldID fid;
+  long     ptr = (long) ptr_;
+
+  cls = (*env)->FindClass(env, "com/paulasmuth/libsmatrix/SparseMatrix");
+  fid = (*env)->GetFieldID(env, cls, "ptr", "J");
+
+  (*env)->SetLongField(env, self, fid, ptr);
+}
+
+int get_ptr(JNIEnv* env, jobject self, void** ptr) {
+  jclass   cls;
+  jfieldID fid;
+  jlong    ptr_;
+
+  cls  = (*env)->FindClass(env, "com/paulasmuth/libsmatrix/SparseMatrix");
+  fid  = (*env)->GetFieldID(env, cls, "ptr", "J");
+  ptr_ = (*env)->GetLongField(env, self, fid);
+
+  if (ptr_ > 0) {
+    *ptr = (void *) ptr_;
+    return 0;
+  } else {
+    throw_exception(env, ERR_PTRNOTFOUND);
+
+    return 1;
+  }
+}
+
+JNIEXPORT void JNICALL _JM(init) (JNIEnv* env, jobject self, jstring file_) {
+  void* ptr;
+  char* file = NULL;
+
+  if (file_ != NULL) {
+    file = (char *) (*env)->GetStringUTFChars(env, file_, 0);
+  }
+
+  ptr = smatrix_open(file);
+
+  if (ptr == NULL) {
+    throw_exception(env, "smatrix_open() failed");
+  } else {
+    set_ptr(env, self, ptr);
+  }
+
+  if (file != NULL) {
+    (*env)->ReleaseStringUTFChars(env, file_, file);
+  }
+}
+
+JNIEXPORT void JNICALL _JM(close) (JNIEnv* env, jobject self) {
+  void* ptr = NULL;
+
+  if (!get_ptr(env, self, &ptr)) {
+    smatrix_close(ptr);
+    set_ptr(env, self, NULL);
+  }
+}
+
+JNIEXPORT jint _JM(get) (JNIEnv* env, jobject self, jint x, jint y) {
+  void* ptr = NULL;
+
+  if (get_ptr(env, self, &ptr)) {
+    return 0;
+  } else {
+    return (jint) smatrix_get(ptr, x, y);
+  }
+}
+
+JNIEXPORT void JNICALL _JM(set) (JNIEnv* env, jobject self, jint x, jint y, jint v) {
+  void* ptr = NULL;
+
+  if (!get_ptr(env, self, &ptr)) {
+    smatrix_set(ptr, (uint32_t) x, (uint32_t) y, (uint32_t) v);
+  }
+}
+
+JNIEXPORT void JNICALL _JM(incr) (JNIEnv* env, jobject self, jint x, jint y, jint v) {
+  void* ptr = NULL;
+
+  if (!get_ptr(env, self, &ptr)) {
+    smatrix_incr(ptr, (uint32_t) x, (uint32_t) y, (uint32_t) v);
+  }
+}
+
+JNIEXPORT void JNICALL _JM(decr) (JNIEnv* env, jobject self, jint x, jint y, jint v) {
+  void* ptr = NULL;
+
+  if (!get_ptr(env, self, &ptr)) {
+    smatrix_decr(ptr, (uint32_t) x, (uint32_t) y, (uint32_t) v);
+  }
+}
+
+JNIEXPORT void JNICALL _JM(getRowNative) (JNIEnv* env, jobject self, jint x, jobject map, jint maxlen) {
+  jclass    cls;
+  jint      i, len;
+  jmethodID mid;
+  uint32_t  *data;
+  void*     ptr = NULL;
+  size_t    bytes;
+
+  cls = (*env)->GetObjectClass(env, map);
+  mid = (*env)->GetMethodID(env, cls, "putIntTuple", "(II)V");
+
+  if (mid == NULL || get_ptr(env, self, &ptr)) {
+    return;
+  }
+
+  len   = smatrix_rowlen(ptr, x);
+  bytes = len * 8;
+  data  = malloc(bytes);
+
+  if (data == NULL) {
+    throw_exception(env, "malloc() failed");
+    return;
+  }
+
+  len = smatrix_getrow(ptr, (uint32_t) x, data, bytes);
+
+  for (i = 0; i < len; i++) {
+    if (maxlen > 0 && i >= maxlen) {
+      break;
+    }
+
+    (*env)->CallVoidMethod(env, map, mid, data[i * 2], data[i * 2 + 1]);
+  }
+
+  free(data);
+}
+
+JNIEXPORT jint JNICALL _JM(getRowLength) (JNIEnv* env, jobject self, jint x) {
+  void* ptr = NULL;
+
+  if (get_ptr(env, self, &ptr)) {
+    return 0;
+  } else {
+    return (jint) smatrix_rowlen(ptr, x);
+  }
+}
+