ewah-bitset 0.0.1

data/CHANGELOG

@@ -0,0 +1 @@
+v0.0.1. First version.

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010 Josh Ferguson (josh@besquared.net)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Manifest

@@ -0,0 +1,11 @@
+CHANGELOG
+LICENSE
+Manifest
+README.md
+Rakefile
+ext/boolarray.h
+ext/ewah-bitset.cpp
+ext/ewah.h
+ext/extconf.rb
+spec/ewah_bitset_spec.rb
+spec/spec.opts

data/README.md

@@ -0,0 +1,33 @@
+# EWAH Bitsets in Ruby
+
+Enhanced Word Aligned Hybrid (EWAH) encoding is a compression strategy for bitmaps. The compressed bitmaps can be scanned over and operated on in place which results in extremely fast processing times even while maintaining high compression rates.
+
+This library wraps the original C++ implementation written by one of the original paper authors Daniel Lemire which is hosted here:
+
+https://github.com/lemire/EWAHBoolArray
+
+Examples are in the spec but the general idea goes something like this:
+
+    require 'ewahbitset'
+    
+    bitset = EwahBitset.new
+    0.upto(10) do |i|
+      bitset.set(i * 10)
+    end
+    
+    positions = []
+    bitset.each do |position|
+      positions << position
+    end
+    
+    puts positions.inspect
+    
+    => [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
+    
+    puts bitset.serialize.inspect
+    
+    => "e\000\000\000\000\000\000\000\003\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\001\004\020@\000\001\004\020@\000\001\004\020\000\000\000"
+
+# Tradeoffs
+
+Of course there are a few tradeoffs at work here to mention. The first is that bits must be set in order. This means that you can't set bit position 1000 and then go set bit position 100. The other thing to note is that there is no random access into the bitset. Anytime you want to check a set bit you're required to do a sequential scan. Due to this an implemention of is_set? requires a sequential scan using the each method. This may make sense for some uses and not others but be aware!

data/Rakefile

@@ -0,0 +1,18 @@
+require 'echoe'
+require 'rake'
+require 'rspec/core/rake_task'
+
+task :default => :spec
+
+Echoe.new("ewah-bitset") do |p|
+  p.author = "Josh Ferguson"
+  p.email = "josh@besquared.net"
+  p.project = "ewah-bitset"
+  p.summary = "A wrapper around Lemire's EWAHBoolArray from https://github.com/lemire/EWAHBoolArray"
+  p.url = "http://www.github.com/yammer/ewah-bitset/"
+end
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+  t.rspec_opts = ['--options', "\"spec/spec.opts\""]
+end

data/ewah-bitset.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "ewah-bitset"
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Josh Ferguson"]
+  s.date = "2012-04-21"
+  s.description = "A wrapper around Lemire's EWAHBoolArray from https://github.com/lemire/EWAHBoolArray"
+  s.email = "josh@besquared.net"
+  s.extensions = ["ext/extconf.rb"]
+  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README.md", "ext/boolarray.h", "ext/ewah-bitset.cpp", "ext/ewah.h", "ext/extconf.rb"]
+  s.files = ["CHANGELOG", "LICENSE", "Manifest", "README.md", "Rakefile", "ext/boolarray.h", "ext/ewah-bitset.cpp", "ext/ewah.h", "ext/extconf.rb", "spec/ewah_bitset_spec.rb", "spec/spec.opts", "ewah-bitset.gemspec"]
+  s.homepage = "http://www.github.com/yammer/ewah-bitset/"
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Ewah-bitset", "--main", "README.md"]
+  s.require_paths = ["lib", "ext"]
+  s.rubyforge_project = "ewah-bitset"
+  s.rubygems_version = "1.8.15"
+  s.summary = "A wrapper around Lemire's EWAHBoolArray from https://github.com/lemire/EWAHBoolArray"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/ext/boolarray.h

@@ -0,0 +1,179 @@
+#ifndef BOOLARRAY_H
+#define BOOLARRAY_H
+
+#include <cassert>
+#include <iostream>
+#include <vector>
+#include <stdexcept>
+#include <sstream>
+#include <iso646.h> // mostly for Microsoft compilers
+
+typedef unsigned long  ulong;
+typedef unsigned int  uint;
+typedef unsigned short  uword16;
+typedef unsigned int  uword32;
+typedef unsigned long long  uword64;
+
+
+using namespace std;
+
+/**
+ * A dynamic bitset implementation. (without compression).
+ * This is not tremendously useful, but it is provided as a reference.
+ */
+template <class uword=uword32>
+class BoolArray {
+public:
+    BoolArray(const size_t n, const uword initval= 0):buffer(n / wordinbits + (n % wordinbits == 0 ? 0 : 1),initval),sizeinbits(n) {	}
+
+    BoolArray():buffer(),sizeinbits(0) {}
+
+    BoolArray(const BoolArray & ba) : buffer(ba.buffer),sizeinbits(ba.sizeinbits) {}
+    void read(istream & in) {
+        sizeinbits = 0;
+        in.read(reinterpret_cast<char *>(&sizeinbits), sizeof(sizeinbits));
+        buffer.resize(sizeinbits / wordinbits + (sizeinbits % wordinbits == 0 ? 0 : 1));
+        in.read(reinterpret_cast<char *>(&buffer[0]),buffer.size()*sizeof(uword));
+    }
+
+    void readBuffer(istream & in,const size_t size) {
+        buffer.resize(size);
+        in.read(reinterpret_cast<char *>(&buffer[0]),buffer.size()*sizeof(uword));
+        sizeinbits = size*sizeof(uword)*8;
+    }
+
+    void setSizeInBits(const size_t sizeib) {
+        sizeinbits = sizeib;
+    }
+
+
+    void write(ostream & out) {
+        write(out,sizeinbits);
+    }
+
+    void write(ostream & out, const size_t numberofbits) const {
+        const size_t size = numberofbits/wordinbits + (numberofbits%wordinbits == 0 ? 0: 1);
+        out.write(reinterpret_cast<const char *>(&numberofbits), sizeof(numberofbits));
+        out.write(reinterpret_cast<const char *>(&buffer[0]),size*sizeof(uword));
+    }
+
+    void writeBuffer(ostream & out, const size_t numberofbits) const {
+        const size_t size = numberofbits/wordinbits + (numberofbits%wordinbits == 0 ? 0: 1);
+        out.write(reinterpret_cast<const char *>(&buffer[0]),size*sizeof(uword));
+    }
+
+    size_t sizeOnDisk() const {
+        size_t size = sizeinbits/wordinbits + (sizeinbits%wordinbits == 0 ? 0: 1);
+        return sizeof(sizeinbits) + size*sizeof(uword);
+    }
+
+
+    BoolArray& operator=(const BoolArray & x) {
+        this->buffer = x.buffer;
+        this->sizeinbits = x.sizeinbits;
+        return *this;
+    }
+
+    bool operator==(const BoolArray & x) const {
+        if(sizeinbits != x.sizeinbits) return false;
+        assert(buffer.size() == x.buffer.size());
+        for(size_t k = 0; k < buffer.size(); ++k)
+            if(buffer[k] != x.buffer[k]) return false;
+        return true;
+    }
+
+    bool operator!=(const BoolArray & x) const {
+        return ! operator==(x);
+    }
+
+    void setWord(const size_t pos, const uword val) {
+        assert(pos < buffer.size());
+        buffer[pos] = val;
+    }
+
+    void add(const uword val) {
+        if(sizeinbits % wordinbits != 0) throw invalid_argument("you probably didn't want to do this");
+        sizeinbits += wordinbits;
+        buffer.push_back(val);
+    }
+
+    uword getWord(const size_t pos) const {
+        assert(pos < buffer.size());
+        return buffer[pos];
+    }
+
+    /**
+     * set to true (whether it was already set to true or not)
+     *
+     * TODO this is an expensive (random access) API, you really ought to
+     * prepare a new word and then append it.
+     */
+    void set(const size_t pos) {
+        buffer[pos/wordinbits] |= ( static_cast<uword>(1) << (pos % wordinbits) ) ;
+    }
+
+    /**
+     * set to false (whether it was already set to false or not)
+     *
+     * TODO this is an expensive (random access) API, you really ought to
+     * prepare a new word and then append it.
+     */
+    void unset(const size_t pos) {
+        buffer[pos/wordinbits] |= ~( static_cast<uword>(1) << (pos % wordinbits) ) ;
+    }
+
+    /**
+     * true of false? (set or unset)
+     */
+    bool get(const size_t pos) const {
+        assert(pos/wordinbits < buffer.size());
+        return (buffer[pos/wordinbits] & ( static_cast<uword>(1) << (pos % wordinbits) )) != 0;
+    }
+
+    /**
+     * set all bits to 0
+     */
+    void reset() {
+        memset(&buffer[0],0,sizeof(uword)*buffer.size());
+        sizeinbits = 0;
+    }
+
+    size_t sizeInBits() const {
+        return sizeinbits;
+    }
+
+    ~BoolArray() {}
+
+    void logicaland(const BoolArray & ba, BoolArray & out);
+    
+    void logicalor(const BoolArray & ba, BoolArray & out);
+    
+
+
+    inline void printout(ostream &o = cout) {
+        for(size_t k = 0; k < sizeinbits; ++k)
+            o << get(k) << " ";
+        o << endl;
+    }
+
+    void append(const BoolArray & a);
+
+    enum { wordinbits =  sizeof(uword) * 8};
+
+private:
+    vector<uword>  buffer;
+    size_t sizeinbits;
+
+};
+
+template <class uword>
+void BoolArray<uword>::append(const BoolArray & a) {
+    if(sizeinbits % wordinbits == 0) {
+        buffer.insert(buffer.end(),a.buffer.begin(),a.buffer.end());
+    } else {
+        throw invalid_argument("Cannot append if parent does not meet boundary");
+    }
+    sizeinbits += a.sizeinbits;
+}
+
+#endif

data/ext/ewah-bitset.cpp

@@ -0,0 +1,176 @@
+#include <ruby.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "ewah.h"
+
+typedef VALUE (ruby_method)(...);
+
+typedef struct ewah {
+  EWAHBoolArray<uword64> *bits;
+} EWAH;
+
+extern "C" VALUE ewah_new(VALUE klass) {
+  EWAH *b = ALLOC(EWAH);
+  b->bits = new EWAHBoolArray<uword64>();
+  VALUE bitset = Data_Wrap_Struct(klass, 0, free, b);
+  rb_obj_call_init(bitset, 0, 0);
+  return bitset;
+}
+
+extern "C" VALUE ewah_init(VALUE self) {
+  return self;
+}
+
+/* Core API */
+extern "C" VALUE ewah_set(VALUE self, VALUE position) {
+  if (position == Qnil) {
+    rb_raise(rb_eRuntimeError, "Position to set not specified");
+  }
+  
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  bitset->bits->set(FIX2INT(position));
+  
+  return self;
+}
+
+extern "C" VALUE ewah_each(VALUE self) {
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  
+  for(EWAHBoolArray<uword64>::const_iterator i = bitset->bits->begin(); i != bitset->bits->end(); ++i)
+    rb_yield(INT2FIX(*i));
+  
+  return Qnil;
+}
+
+extern "C" VALUE ewah_swap(VALUE self, VALUE other) {
+  EWAH *bitset;
+  EWAH *obitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  Data_Get_Struct(other, EWAH, obitset);
+  
+  bitset->bits->swap(*(obitset->bits));
+  
+  return self;
+}
+
+extern "C" VALUE ewah_reset(VALUE self) {
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  bitset->bits->reset();
+  return self;
+}
+
+/* Set Operations */
+extern "C" VALUE ewah_logical_or(VALUE self, VALUE other) {
+  EWAH *bitset;
+  EWAH *obitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  Data_Get_Struct(other, EWAH, obitset);
+  
+  VALUE newBitset = ewah_new(rb_path2class("EwahBitset"));
+
+  EWAH *newBits;
+  Data_Get_Struct(newBitset, EWAH, newBits);
+  bitset->bits->logicalor(*(obitset->bits), *(newBits->bits));
+  
+  return newBitset;
+}
+
+extern "C" VALUE ewah_logical_and(VALUE self, VALUE other) {
+  EWAH *bitset;
+  EWAH *obitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  Data_Get_Struct(other, EWAH, obitset);
+  
+  VALUE newBitset = ewah_new(rb_path2class("EwahBitset"));
+  
+  EWAH *newBits;
+  Data_Get_Struct(newBitset, EWAH, newBits);
+  bitset->bits->logicaland(*(obitset->bits), *(newBits->bits));
+  
+  return newBitset;
+}
+
+extern "C" VALUE ewah_equals(VALUE self, VALUE other) {
+  EWAH *bitset;
+  EWAH *obitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  Data_Get_Struct(other, EWAH, obitset);
+  
+  if(*(bitset->bits) == *(obitset->bits)) {
+    return Qtrue;
+  } else {
+    return Qfalse;
+  }
+}
+
+/* Information & Serialization */
+extern "C" VALUE ewah_size_in_bits(VALUE self) {
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  return INT2FIX(bitset->bits->sizeInBits());
+}
+
+extern "C" VALUE ewah_size_in_bytes(VALUE self) {
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  return INT2FIX(bitset->bits->sizeInBytes());
+}
+
+extern "C" VALUE ewah_to_binary_s(VALUE self) {
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  
+  stringstream ss;
+  bitset->bits->printout(ss);
+  
+  return rb_str_new(ss.str().c_str(), ss.str().size());
+}
+
+extern "C" VALUE ewah_serialize(VALUE self) {
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+  
+  stringstream ss;
+  bitset->bits->write(ss);
+  
+  return rb_str_new(ss.str().c_str(), ss.str().size());
+}
+
+extern "C" VALUE ewah_deserialize(VALUE self, VALUE bytes) {
+  EWAH *bitset;
+  Data_Get_Struct(self, EWAH, bitset);
+
+  stringstream ss;
+  ss.write(RSTRING_PTR(bytes), RSTRING_LEN(bytes));
+  bitset->bits->read(ss, true);
+  
+  return self;
+}
+
+static VALUE rb_cC;
+extern "C" void Init_ewahbitset() {
+  rb_cC = rb_define_class("EwahBitset", rb_cObject);
+  rb_define_singleton_method(rb_cC, "new", (ruby_method*) &ewah_new, 0);
+  rb_define_method(rb_cC, "initialize", (ruby_method*) &ewah_init, 0);
+  
+  rb_define_method(rb_cC, "set", (ruby_method*) &ewah_set, 1);
+  rb_define_method(rb_cC, "each", (ruby_method*) &ewah_each, 0);
+  rb_define_method(rb_cC, "swap", (ruby_method*) &ewah_swap, 1);
+  rb_define_method(rb_cC, "reset", (ruby_method*) &ewah_reset, 0);
+  
+  rb_define_method(rb_cC, "==", (ruby_method*) &ewah_equals, 1);
+  rb_define_method(rb_cC, "logical_or", (ruby_method*) &ewah_logical_or, 1);
+  rb_define_method(rb_cC, "logical_and", (ruby_method*) &ewah_logical_and, 1);
+  
+  rb_define_method(rb_cC, "to_binary_s", (ruby_method*) &ewah_to_binary_s, 0);
+  rb_define_method(rb_cC, "serialize", (ruby_method*) &ewah_serialize, 0);
+  rb_define_method(rb_cC, "deserialize", (ruby_method*) &ewah_deserialize, 1);
+  rb_define_method(rb_cC, "size_in_bits", (ruby_method*) ewah_size_in_bits, 0);
+  rb_define_method(rb_cC, "size_in_bytes", (ruby_method*) ewah_size_in_bytes, 0);
+}