hyperll 0.2.6 → 0.3.0

data/ext/hyperll/hyperll.c

@@ -0,0 +1,12 @@
+#include "hyperll.h"
+
+VALUE rb_mHyperll;
+
+void Init_hyperll(void) {
+  rb_mHyperll = rb_define_module("Hyperll");
+
+  Init_hyperll_register_set();
+  Init_hyperll_hyper_log_log_plus();
+  Init_hyperll_varint();
+  Init_hyperll_delta_bytes();
+}

data/ext/hyperll/hyperll.h

@@ -0,0 +1,17 @@
+#ifndef __H__HYPERLL
+#define __H__HYPERLL
+
+#include <ruby.h>
+#ifndef HAVE_RUBY_ENCODING_H
+#error "Hyperll requires Ruby 1.9+ to build"
+#else
+#include <ruby/encoding.h>
+#endif
+
+// Initialization functions
+void Init_hyperll_register_set(void);
+void Init_hyperll_hyper_log_log_plus(void);
+void Init_hyperll_varint(void);
+void Init_hyperll_delta_bytes(void);
+
+#endif

data/ext/hyperll/register_set.c

@@ -0,0 +1,196 @@
+#include "hyperll.h"
+#include "register_set.h"
+
+extern VALUE rb_mHyperll;
+VALUE rb_cRegisterSet;
+
+const int LOG2_BITS_PER_WORD = 6;
+const int REGISTER_SIZE = 5;
+
+void register_set_init(register_set *set, int count) {
+  set->count = count;
+
+  int bits = count / LOG2_BITS_PER_WORD;
+  if (bits == 0) {
+    set->size = 1;
+  } else if ((bits % sizeof(int)) == 0) {
+    set->size = bits;
+  } else {
+    set->size = bits + 1;
+  }
+
+  set->values = (uint32_t*)calloc(set->size, sizeof(uint32_t));
+}
+
+void register_set_set(register_set *set, int position, uint32_t value) {
+  int bucket = position / LOG2_BITS_PER_WORD;
+  int shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD));
+
+  set->values[bucket] = (set->values[bucket] & ~(0x1f << shift)) | (value << shift);
+}
+
+uint32_t register_set_get(register_set *set, int position) {
+  int bucket = position / LOG2_BITS_PER_WORD;
+  int shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD));
+  return (uint32_t)(((set->values[bucket] & (0x1f << shift))) >> shift);
+}
+
+int register_set_update_if_greater(register_set *set, int position, uint32_t value) {
+  int bucket = position / LOG2_BITS_PER_WORD;
+  int shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD));
+  uint32_t mask = 0x1f << shift;
+
+  uint64_t cur = set->values[bucket] & mask;
+  uint64_t new = value << shift;
+  if (cur < new) {
+    set->values[bucket] = (uint32_t)((set->values[bucket] & ~mask) | new);
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+void register_set_merge(register_set *set, register_set *other) {
+  int size = set->size;
+  for (int bucket = 0; bucket < size; bucket++) {
+    uint32_t sval = set->values[bucket];
+    uint32_t oval = other->values[bucket];
+    uint32_t word = 0;
+
+    for (int j = 0; j < LOG2_BITS_PER_WORD; j++) {
+      uint32_t mask = 0x1f << (REGISTER_SIZE * j);
+
+      uint32_t thisval = (sval & mask);
+      uint32_t thatval = (oval & mask);
+      word |= (thisval < thatval) ? thatval : thisval;
+    }
+    set->values[bucket] = word;
+  }
+}
+
+void register_set_free(register_set *set) {
+  free(set->values);
+  free(set);
+}
+
+static VALUE rb_register_set_new(int argc, VALUE *argv, VALUE klass) {
+  VALUE count, values;
+  rb_scan_args(argc, argv, "11", &count, &values);
+
+  register_set *set = ALLOC(register_set);
+  register_set_init(set, NUM2INT(count));
+  VALUE setv = Data_Wrap_Struct(klass, 0, register_set_free, set);
+
+  if (!NIL_P(values)) {
+    Check_Type(values, T_ARRAY);
+    if (RARRAY_LEN(values) == set->size) {
+      for (int i = 0; i < set->size; i++) {
+        set->values[i] = NUM2ULONG(rb_ary_entry(values, i));
+      }
+    } else {
+      rb_raise(rb_eArgError, "initial set of values is not of the correct size");
+    }
+  }
+
+  return setv;
+}
+
+static VALUE rb_register_set_index_set(VALUE self, VALUE position, VALUE value) {
+  Check_Type(position, T_FIXNUM);
+  Check_Type(value, T_FIXNUM);
+
+  register_set *set;
+  Data_Get_Struct(self, register_set, set);
+  register_set_set(set, NUM2INT(position), NUM2ULONG(value));
+
+  return Qnil;
+}
+
+static VALUE rb_register_set_index_get(VALUE self, VALUE position) {
+  Check_Type(position, T_FIXNUM);
+
+  register_set *set;
+  Data_Get_Struct(self, register_set, set);
+  return UINT2NUM(register_set_get(set, NUM2INT(position)));
+}
+
+static VALUE rb_register_set_update_if_greater(VALUE self, VALUE position, VALUE value) {
+  Check_Type(position, T_FIXNUM);
+  Check_Type(value, T_FIXNUM);
+
+  register_set *set;
+  Data_Get_Struct(self, register_set, set);
+  int rv = register_set_update_if_greater(set, NUM2INT(position), NUM2ULONG(value));
+
+  return rv ? Qtrue : Qfalse;
+}
+
+static VALUE rb_register_set_merge(VALUE self, VALUE other) {
+  register_set *set;
+  Data_Get_Struct(self, register_set, set);
+
+  register_set *other_set;
+  Data_Get_Struct(other, register_set, other_set);
+  if (other_set == NULL) {
+    rb_raise(rb_eTypeError, "other must be another register set");
+    return Qnil;
+  }
+
+  register_set_merge(set, other_set);
+
+  return self;
+}
+
+static VALUE rb_register_set_each(VALUE self) {
+  register_set *set;
+  Data_Get_Struct(self, register_set, set);
+
+  for (int i = 0; i < set->count; i++) {
+    rb_yield(UINT2NUM(register_set_get(set, i)));
+  }
+
+  return self;
+}
+
+static VALUE rb_register_set_serialize(VALUE self) {
+  register_set *set;
+  Data_Get_Struct(self, register_set, set);
+
+  int strsize = set->size * sizeof(int);
+  char *str = (char*)malloc(strsize + 1);
+  str[strsize] = 0;
+
+  for (int i = 0; i < set->size; i++) {
+    int value = set->values[i];
+    int offset = i * 4;
+
+    str[offset] = (char)(value >> 24);
+    str[offset + 1] = (char)(value >> 16);
+    str[offset + 2] = (char)(value >> 8);
+    str[offset + 3] = (char)value;
+  }
+
+  return rb_str_new(str, strsize);
+}
+
+static VALUE rb_register_set_size(VALUE self) {
+  register_set *set;
+  Data_Get_Struct(self, register_set, set);
+
+  return INT2NUM(set->size);
+}
+
+void Init_hyperll_register_set(void) {
+  rb_cRegisterSet = rb_define_class_under(rb_mHyperll, "RegisterSet", rb_cObject);
+  rb_include_module(rb_cRegisterSet, rb_mEnumerable);
+
+  rb_define_singleton_method(rb_cRegisterSet, "new", rb_register_set_new, -1);
+
+  rb_define_method(rb_cRegisterSet, "[]=", rb_register_set_index_set, 2);
+  rb_define_method(rb_cRegisterSet, "[]", rb_register_set_index_get, 1);
+  rb_define_method(rb_cRegisterSet, "update_if_greater", rb_register_set_update_if_greater, 2);
+  rb_define_method(rb_cRegisterSet, "merge", rb_register_set_merge, 1);
+  rb_define_method(rb_cRegisterSet, "each", rb_register_set_each, 0);
+  rb_define_method(rb_cRegisterSet, "serialize", rb_register_set_serialize, 0);
+  rb_define_method(rb_cRegisterSet, "size", rb_register_set_size, 0);
+}

data/ext/hyperll/register_set.h

@@ -0,0 +1,19 @@
+#ifndef __H_REGISTER_SET
+#define __H_REGISTER_SET
+
+#include <stdint.h>
+
+typedef struct {
+  int count;
+  int size;
+  uint32_t *values;
+} register_set;
+
+void register_set_init(register_set *set, int count);
+void register_set_set(register_set *set, int position, uint32_t value);
+uint32_t register_set_get(register_set *set, int position);
+int register_set_update_if_greater(register_set *set, int position, uint32_t value);
+void register_set_merge(register_set *set, register_set *other);
+void register_set_free(register_set *set);
+
+#endif

data/ext/hyperll/sparse_set.c

@@ -0,0 +1,76 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "sparse_set.h"
+
+void sparse_set_init(sparse_set *set, int sm, int capacity) {
+  set->sm = sm;
+  set->capacity = capacity;
+  set->size = 0;
+  set->values = (uint32_t*)calloc(set->capacity, sizeof(uint32_t));
+}
+
+int sparse_set_cardinality(sparse_set *set) {
+  return (int)round(set->sm * log(((double)set->sm) / (set->sm - set->size)));
+}
+
+uint32_t sparse_set_sparse_index(uint32_t k) {
+  if ((k & 1) == 1) {
+    return k >> 7;
+  } else {
+    return k >> 1;
+  }
+}
+
+int sparse_set_merge(sparse_set *set, sparse_set *other) {
+  uint32_t *svals = set->values;
+  int ssize = set->size;
+  uint32_t *ovals = other->values;
+  int osize = other->size;
+
+  int capacity = set->capacity;
+  uint32_t *values = (uint32_t*)calloc(capacity, sizeof(uint32_t));
+
+  // s tracks svals, o tracks ovals, size tracks values
+  int s = 0, o = 0, size = 0;
+  while (s < ssize || o < osize) {
+    // Check that we don't grow over capacity
+    if (size >= capacity) {
+      free(values);
+      return -1;
+    }
+
+    if (o >= osize) {
+      values[size++] = svals[s++];
+    } else if (s >= ssize) {
+      values[size++] = ovals[o++];
+    } else {
+      uint32_t sval = svals[s];
+      uint32_t sidx = sparse_set_sparse_index(sval);
+      uint32_t oval = ovals[o];
+      uint32_t oidx = sparse_set_sparse_index(oval);
+
+      if (sidx == oidx) {
+        values[size++] = sval < oval ? sval : oval;
+        s++;
+        o++;
+      } else if (sidx < oidx) {
+        values[size++] = sval;
+        s++;
+      } else {
+        values[size++] = oval;
+        o++;
+      }
+    }
+  }
+
+  free(set->values);
+  set->values = values;
+  set->size = size;
+  return 0;
+}
+
+void sparse_set_free(sparse_set *set) {
+  free(set->values);
+  free(set);
+}

data/ext/hyperll/sparse_set.h

@@ -0,0 +1,24 @@
+#ifndef __H_SPARSE_SET
+#define __H_SPARSE_SET
+
+#include <stdint.h>
+
+typedef struct {
+  int sm;
+  int capacity;
+  int size;
+  uint32_t *values;
+} sparse_set;
+
+void sparse_set_init(sparse_set *set, int sm, int capacity);
+int sparse_set_cardinality(sparse_set *set);
+
+// Merges two sparse sets together.
+//
+// Returns 0 on success; -1 if the sparse set would grow too large
+int sparse_set_merge(sparse_set *set, sparse_set *other);
+
+uint32_t sparse_set_sparse_index(uint32_t k);
+void sparse_set_free(sparse_set *set);
+
+#endif

data/ext/hyperll/varint.c

@@ -0,0 +1,83 @@
+#include "hyperll.h"
+#include "varint.h"
+
+extern VALUE rb_mHyperll;
+VALUE rb_cVarint;
+
+int varint_write_unsigned(uint32_t value, uint8_t bytes[]) {
+  int i = 0;
+  while ((value & 0xFFFFFF80) != 0) {
+    bytes[i++] = (uint8_t)((value & 0x7F) | 0x80);
+    value >>= 7;
+  }
+
+  bytes[i++] = (uint8_t)(value & 0x7F);
+  return i;
+}
+
+uint32_t varint_read_unsigned(uint8_t bytes[], int maxlen, int *len) {
+  if (maxlen <= 0) {
+    *len = -1;
+    return 0;
+  }
+
+  uint32_t value, i, b;
+  value = i = b = 0;
+
+  while (((b = bytes[i]) & 0x80) != 0) {
+    value |= (b & 0x7F) << (i * 7);
+    i++;
+
+    if (i >= maxlen) {
+      *len = -1;
+      return 0;
+    }
+  }
+
+  value |= (b << (i * 7));
+  *len = (i + 1);
+  return value;
+}
+
+static VALUE rb_varint_read_unsigned_var_int(VALUE klass, VALUE rbytes) {
+  int rlen = RARRAY_LEN(rbytes);
+  int maxlen = (rlen > 5) ? 5 : rlen;
+
+  uint8_t bytes[5];
+  for (int i = 0; i < maxlen; i++) {
+    bytes[i] = (uint8_t)NUM2INT(rb_ary_entry(rbytes, i));
+  }
+
+  int len = 0;
+  uint32_t value = varint_read_unsigned(bytes, maxlen, &len);
+
+  if (len == -1) {
+    rb_raise(rb_eRuntimeError, "Variable length quantity is too long");
+    return Qnil;
+  }
+
+  // Discard elements that were used to retrieve the value
+  for (int i = 0; i < len; i++) {
+    rb_ary_shift(rbytes);
+  }
+  return ULONG2NUM(value);
+}
+
+static VALUE rb_varint_write_unsigned_var_int(VALUE klass, VALUE value) {
+  VALUE rbytes = rb_ary_new2(5);
+
+  uint8_t bytes[5];
+  int len = varint_write_unsigned(NUM2ULONG(value), bytes);
+  for (int i = 0; i < len; i++) {
+    rb_ary_push(rbytes, INT2NUM(bytes[i]));
+  }
+
+  return rbytes;
+}
+
+void Init_hyperll_varint(void) {
+  rb_cVarint = rb_define_class_under(rb_mHyperll, "Varint", rb_cObject);
+
+  rb_define_singleton_method(rb_cVarint, "read_unsigned_var_int", rb_varint_read_unsigned_var_int, 1);
+  rb_define_singleton_method(rb_cVarint, "write_unsigned_var_int", rb_varint_write_unsigned_var_int, 1);
+}

data/ext/hyperll/varint.h

@@ -0,0 +1,21 @@
+#ifndef __H_VARINT
+#define __H_VARINT
+
+// Writes an unsigned varint.
+//
+// value: value to encode
+// bytes: destination buffer. Caller is responsible for the allocation.
+//        Must be of size 5.
+//
+// Returns the count of the bytes that were actually needed to store the varint
+int varint_write_unsigned(uint32_t value, uint8_t bytes[]);
+
+// Reads an unsigned varint.
+//
+// bytes: encoded value
+// maxlen: the maximum number of bytes to read
+// len: the number of bytes read to reconstruct the varint; -1 if an error
+//      occurred
+uint32_t varint_read_unsigned(uint8_t bytes[], int maxlen, int *len);
+
+#endif