bitwise 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1 @@
+source "http://rubygems.org"

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Kenn Ejima
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+Bitwise
+=======
+
+Fast, memory efficient bitwise operations on large binary strings.
+
+Internally a bit array is represented as a ruby string with `Encoding::ASCII_8BIT` encoding, which keeps billions of bits in a workable footprint.
+
+* 1,000,000 bits = 125KB
+* 10,000,000 bits = 1.25MB
+* 100,000,000 bits = 12.5MB
+* 1,000,000,000 bits = 125MB
+
+Install
+-------
+
+    gem install bitwise
+
+Usage
+-----
+
+Bitwise assignment and retrieval:
+
+```ruby
+b = Bitwise.new(1)
+
+b.to_bits
+ => "00000000"
+
+b.set_at(1)
+b.set_at(4)
+
+b.to_bits
+ => "01001000"
+
+b.clear_at(1)
+
+b.to_bits
+ => "00001000"
+```
+
+Index assignment and retrieval:
+
+```ruby
+b = Bitwise.new
+b.indexes = [1, 10, 100]
+
+b.to_bits
+ => "01000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000"
+
+b.cardinality
+ => 3
+
+b.size
+ => 13
+
+b.set_at(20)
+
+b.to_bits
+ => "01000000001000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000"
+
+b.value.unpack('C*')
+ => [64, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8]
+
+b.cardinality
+ => 4
+
+b.indexes
+ => [1, 10, 20, 100]
+```

data/Rakefile

@@ -0,0 +1,29 @@
+# require "bundler/gem_tasks"
+# require 'rubygems'
+# require 'bundler'
+
+require "rake/extensiontask"
+Rake::ExtensionTask.new("bitwise") do |extension|
+  extension.lib_dir = "lib/bitwise"
+end
+
+require 'rspec/core/rake_task'
+task :default => :spec
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ["--color"]
+  t.fail_on_error = false
+end
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "bitwise"
+  gem.homepage = "http://github.com/kenn/bitwise"
+  gem.license = "MIT"
+  gem.summary = %Q{Fast, memory efficient bitwise operations on large binary strings}
+  gem.description = %Q{Fast, memory efficient bitwise operations on large binary strings}
+  gem.email = "kenn.ejima@gmail.com"
+  gem.authors = ["Kenn Ejima"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bitwise.gemspec

@@ -0,0 +1,49 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "bitwise"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kenn Ejima"]
+  s.date = "2011-12-13"
+  s.description = "Fast, memory efficient bitwise operations on large binary strings"
+  s.email = "kenn.ejima@gmail.com"
+  s.extensions = ["ext/bitwise/extconf.rb"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "bitwise.gemspec",
+    "ext/bitwise/bitwise.c",
+    "ext/bitwise/extconf.rb",
+    "lib/bitwise.rb",
+    "lib/bitwise/bitwise.bundle",
+    "spec/bitwise_spec.rb"
+  ]
+  s.homepage = "http://github.com/kenn/bitwise"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.10"
+  s.summary = "Fast, memory efficient bitwise operations on large binary strings"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/ext/bitwise/bitwise.c

@@ -0,0 +1,61 @@
+#include "ruby.h"
+
+VALUE BitwiseClass;
+
+int COUNT_TABLE[] = {
+  0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8,
+};
+
+static VALUE bw_string_union(VALUE self, VALUE max, VALUE min)
+{
+  VALUE result = rb_str_new(RSTRING_PTR(max), RSTRING_LEN(max)); 
+  int i;
+  int min_len = RSTRING_LEN(min);
+  for (i = 0; i < RSTRING_LEN(max); i++) {
+    RSTRING_PTR(result)[i] |= ((i < min_len) ? RSTRING_PTR(min)[i] : 0);
+  }
+  return result;
+}
+
+static VALUE bw_string_intersect(VALUE self, VALUE max, VALUE min)
+{
+  VALUE result = rb_str_new(RSTRING_PTR(min), RSTRING_LEN(min)); 
+  int i;
+  for (i = 0; i < RSTRING_LEN(min); i++) {
+    RSTRING_PTR(result)[i] &= RSTRING_PTR(max)[i];
+  }
+  return result;
+}
+
+static VALUE bw_population_count(VALUE self, VALUE str) {
+  int count, i;
+  unsigned char *buffer = RSTRING_PTR(str);
+  count = 0;
+  for (i = 0; i < RSTRING_LEN(str); i++) {
+    count += COUNT_TABLE[buffer[i]];
+  }
+  return INT2NUM(count);
+}
+
+void Init_bitwise()
+{
+  BitwiseClass = rb_define_class("Bitwise", rb_cObject);
+  rb_define_singleton_method(BitwiseClass, "string_union", bw_string_union, 2);
+  rb_define_singleton_method(BitwiseClass, "string_intersect", bw_string_intersect, 2);
+  rb_define_singleton_method(BitwiseClass, "population_count", bw_population_count, 1);
+}

data/ext/bitwise/extconf.rb

@@ -0,0 +1,2 @@
+require 'mkmf'
+create_makefile("bitwise")

data/lib/bitwise.rb

@@ -0,0 +1,105 @@
+# coding: ascii-8bit
+
+require "bitwise/bitwise"
+
+class Bitwise
+  attr_accessor :value
+
+  def initialize(size = 0)
+    @value = "\x00" * size
+  end
+
+  def size
+    @value.bytesize
+  end
+  alias :to_s :size
+
+  def to_bits
+    @value.unpack('B*').first
+  end
+
+  def set_at(index)
+    get_byte(index)
+    @value.setbyte(@div, @byte | bitmask)
+  end
+
+  def clear_at(index)
+    get_byte(index)
+    @value.setbyte(@div, @byte  & ~bitmask)
+  end
+
+  def set_at?(index)
+    get_bit(index) == 1
+  end
+
+  def clear_at?(index)
+    get_bit(index) == 0
+  end
+
+  def get_bit(index)
+    get_byte(index)
+    (@byte & bitmask) > 0 ? 1 : 0
+  end
+
+  def bitmask
+    2**(7 - @mod)
+  end
+
+  def get_byte(index)
+    @div, @mod = index.divmod(8)
+    @byte = @value.getbyte(@div)
+  end
+
+  def intersect(other)
+    min, max = [ self.value, other.value ].sort_by{|i| i.bytesize }
+    result = Bitwise.new
+    result.value = Bitwise.string_intersect(max, min)
+    result
+  end
+  alias :& :intersect
+
+  def union(other)
+    min, max = [ self.value, other.value ].sort_by{|i| i.bytesize }
+    result = Bitwise.new
+    result.value = Bitwise.string_union(max, min)
+    result
+  end
+  alias :| :union
+
+  def value=(string)
+    @value = string.force_encoding(Encoding::ASCII_8BIT)
+    @value.bytesize
+  end
+
+  def indexes=(array)
+    max_index = array.max
+    @value = "\x00" * (max_index.div(8) + 1)
+    array.each do |index|
+      set_at(index)
+    end
+    @value.bytesize
+  end
+
+  def indexes
+    indexes = []
+    position = 0
+    @value.each_byte do |c|
+      BITS_TABLE[c].each do |i|
+        indexes << (position*8 + i)
+      end
+      position += 1
+    end
+    indexes
+  end
+
+  def cardinality
+    Bitwise.population_count(self.value)
+  end
+
+  BITS_TABLE = (0..255).map do |i|
+    (0..7).map do |j|
+      j = 7 - j
+      ((i & 2**j) > 0) ? (7 - j) : nil
+    end.compact
+  end
+end

data/spec/bitwise_spec.rb

@@ -0,0 +1,96 @@
+require 'bitwise'
+require 'set'
+
+describe Bitwise do
+  before do
+    @bitwise = Bitwise.new(1)
+  end
+
+  it "should set and clear" do
+    @bitwise.to_bits.should == '00000000'
+
+    @bitwise.set_at 1
+    @bitwise.set_at 2
+    @bitwise.set_at 3
+    @bitwise.set_at 5
+    @bitwise.to_bits.should == '01110100'
+    @bitwise.cardinality.should == 4
+
+    @bitwise.clear_at(1)
+    @bitwise.clear_at(3)
+    @bitwise.to_bits.should == '00100100'
+    @bitwise.cardinality.should == 2
+  end
+
+  it "should set by indexes" do
+    @bitwise.indexes = [1,10]
+    @bitwise.indexes.should == [1,10]
+    @bitwise.to_bits.should == '0100000000100000'
+    @bitwise.cardinality.should == 2
+  end
+
+  describe "benchmark" do
+    def assign_indexes(total, picks)
+      set = Set.new
+
+      picks.times do
+        set << (rand*total).to_i
+      end
+
+      @indexes = set.to_a
+    end
+
+    def measure
+      start = Time.now.to_f
+      yield
+      Time.now.to_f - start
+    end
+
+    it "indexes assignment" do
+      assign_indexes(1000, 10)
+      measure { @bitwise.indexes = @indexes }.should < 0.0001
+
+      assign_indexes(10_000, 100)
+      measure { @bitwise.indexes = @indexes }.should < 0.001
+
+      assign_indexes(100_000, 1000)
+      measure { @bitwise.indexes = @indexes }.should < 0.01
+
+      assign_indexes(1000_000, 10_000)
+      measure { @bitwise.indexes = @indexes }.should < 0.1
+    end
+
+    it "cardinality sparse" do
+      @bitwise.indexes = [1, 10, 100, 1000, 10_000, 100_000, 1000_000, 10_000_000, 100_000_000]
+      measure { @bitwise.cardinality.should == 9 }.should < 1.0
+    end
+
+    it "cardinality dense" do
+      assign_indexes(10_000_000, 100_000)
+      @bitwise.indexes = @indexes
+      measure { @bitwise.cardinality }.should < 1.0
+    end
+
+    it "union sparse" do
+      b1 = Bitwise.new
+      b2 = Bitwise.new
+      b1.indexes = [1, 10, 100, 1000, 10_000, 100_000, 1000_000, 10_000_000, 100_000_000]
+      b2.indexes = [2, 20, 200, 2000, 20_000, 200_000, 2000_000, 20_000_000, 200_000_000]
+      measure do
+        b3 = b1.union(b2)
+        b3.cardinality.should == 18
+      end.should < 1.0
+    end
+
+    it "intersect sparse" do
+      b1 = Bitwise.new
+      b2 = Bitwise.new
+      b1.indexes = [1, 10, 100, 1000, 10_000, 100_000, 1000_000, 10_000_000, 100_000_000]
+      b2.indexes = [2, 20, 200, 2000, 20_000, 200_000, 2000_000, 20_000_000, 200_000_000]
+      measure do
+        b3 = b1.intersect(b2)
+        b3.cardinality.should == 0
+      end.should < 1.0
+    end
+  end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: bitwise
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Kenn Ejima
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-13 00:00:00.000000000 Z
+dependencies: []
+description: Fast, memory efficient bitwise operations on large binary strings
+email: kenn.ejima@gmail.com
+executables: []
+extensions:
+- ext/bitwise/extconf.rb
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- .document
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- bitwise.gemspec
+- ext/bitwise/bitwise.c
+- ext/bitwise/extconf.rb
+- lib/bitwise.rb
+- lib/bitwise/bitwise.bundle
+- spec/bitwise_spec.rb
+homepage: http://github.com/kenn/bitwise
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Fast, memory efficient bitwise operations on large binary strings
+test_files: []