bitindex 0.0.2

data/README.md

@@ -0,0 +1,39 @@
+# Bitindex
+
+Bitindex is a simple gem for writing binary files where the position of a 1 in the bits of the file indicates a true value and 0 indicates a false.  The default is to read from left to right so that a file that consists of `1000 1000` could be read to return true for the value 0 and the value 4 and false for all other integer values.
+
+```ruby
+require 'bitindex'
+
+file = 'example.bits'
+size_in_bytes = 1024
+w = Bitindex::Writer.new file, size_in_bytes
+w.write [1,2,3,8000]
+
+r = Bitindex::Reader.new file
+r.is_set? 8000   # true
+r.is_set? 8001   # false
+r.is_set? 2**32  # false
+r.all_set(0..10) # [1,2,3]
+
+```
+
+
+# Install
+
+```ruby
+require 'bitindex'
+```
+
+# Specifics
+
+A project I was working was given a file that mapped the 32bit address space into a 530MB file where bits set to 1 indicated integer values that were set.  I decided to expand the library to handle both the reading and writing of these files.  Creating files of that size on my desktop (few year old MacBook Pro) takes about X minutes.  But reading from the file is fairly quick.  
+
+Future enhancements:
+* compressing and decompressing the file
+* <del>optimizing the Writer so that it faster for index files with long sequences of 1s or 0s</del> A `block_size` option has been added to enable long blocks of zeros.  On my laptop, the creation of a 530M index with index values went from 10 minutes to about a minute using `{block_size: 1024*1024}`.
+* allowing the Writer to set and unset individual bits
+
+# Limitations
+
+This library ignores the potential difference of hardware, bit ordering and all that stuff and relys on the ruby implementation to handle those details.  It is suitable for creating and using files under a common platform and operating system.  

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+
+task :default => 'spec'

data/lib/bitindex/common.rb

@@ -0,0 +1,39 @@
+module Bitindex
+  module Common
+
+    SIGNED_16BIT =   2**15 / 8
+    UNSIGNED_16BIT = 2**16 / 8
+
+    SIGNED_32BIT   = 2**31 / 8
+    UNSIGNED_32BIT = 2**32 / 8
+
+    def bit_set? byte, value, ltor = true
+      mask = build_mask value, ltor
+      (byte & mask) != 0
+    end
+
+    def set_bit byte, value, ltor = true
+      mask = build_mask value, ltor
+      (byte | mask)
+    end
+  
+    def unset_bit byte, value, ltor = true
+      mask = build_mask value, ltor
+      mask ^= 0xff
+      (byte & mask)
+    end
+
+    def byte_pos value
+      (value >> 3)
+    end
+
+    def build_mask value, ltor = true
+      shift = value & 0x07
+      if ltor
+        (0x80 >> shift)
+      else
+        (0x01 << shift)
+      end
+    end
+  end
+end

data/lib/bitindex/reader.rb

@@ -0,0 +1,47 @@
+module Bitindex
+  class Reader
+    include Bitindex::Common
+
+    attr_reader :filepath
+    attr_accessor :ltor
+
+    def initialize filepath, opts = {}
+
+      raise "#{ filepath } is not readable" unless filepath and File.readable?(filepath)
+      @filepath = filepath
+
+      @ltor = !(opts[:ltor] == false)
+    end
+
+    def is_set? value
+      result = false
+      File.open(self.filepath, 'rb') do |io|
+        idx = byte_pos value
+        if idx < io.size
+          io.seek idx, IO::SEEK_SET
+          result = bit_set? io.getbyte, value, self.ltor
+        end
+      end
+      result
+    end
+
+    def all_set array
+      arr = []
+      File.open(self.filepath, 'rb') do |io|
+        cur_idx = nil
+        cur_byte = nil
+        array.each do |n|
+          idx = byte_pos n
+          if idx < io.size
+            if cur_idx.nil? or cur_idx < idx
+              io.seek idx, IO::SEEK_SET
+              cur_byte = io.getbyte
+            end
+            arr << n if bit_set? cur_byte, n, self.ltor
+          end
+        end
+      end
+      arr
+    end
+  end
+end

data/lib/bitindex/version.rb

@@ -0,0 +1,3 @@
+module Bitindex
+  VERSION = "0.0.2"
+end

data/lib/bitindex/writer.rb

@@ -0,0 +1,63 @@
+module Bitindex
+  class Writer
+    include Bitindex::Common
+
+    attr_reader :filepath, :size
+    attr_reader :ltor
+    attr_reader :block_size
+
+    def initialize filepath, size, opts = {}
+      raise "#{ size } is invalid size" unless size and size.to_i > 0
+      @size = size
+
+      raise "#{ filepath } is invalid file" unless filepath
+      @filepath = filepath
+
+      @ltor = !(opts[:ltor] == false)
+      @block_size = (opts[:block_size] and opts[:block_size].to_i > 0) ? opts[:block_size].to_i : 1024
+    end
+
+    def write sorted
+      File.open(self.filepath, 'wb') do |io|
+        last_num = nil
+        cur_byte = 0
+        cur_idx = 0
+        sorted.each do |n|
+          raise "values must be sorted" unless last_num.nil? or n >= last_num
+          
+          idx = byte_pos n
+          if idx < self.size
+            if idx > cur_idx
+              io.putc cur_byte
+              pad_to_pos io, cur_idx, idx - 1
+              cur_idx = idx
+              cur_byte = 0
+            end
+
+            cur_byte = set_bit cur_byte, n, self.ltor
+          end          
+        end
+
+        io.putc cur_byte
+        pad_to_pos io, cur_idx, self.size - 1 if cur_idx < self.size - 1
+      end
+    end
+
+protected
+
+    def pad_to_pos io, start, last
+      while start < last
+        diff = last - start
+        if diff > self.block_size
+          a = Array.new(self.block_size, 0)
+          io.write(a.pack('C' * self.block_size))
+          start += self.block_size
+        else
+          a = Array.new(diff, 0)
+          io.write(a.pack('C' * diff))
+          start = last
+        end
+      end       
+    end
+  end
+end

data/lib/bitindex.rb

@@ -0,0 +1,7 @@
+require 'bitindex/version'
+require 'bitindex/common'
+require 'bitindex/reader'
+require 'bitindex/writer'
+
+module Bitindex
+end

data/spec/common_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+
+class BitIndexCommonTest; include Bitindex::Common; end
+
+describe Bitindex::Common do
+
+  before(:each) { @test = BitIndexCommonTest.new }
+
+  describe '.set_bit' do
+    it 'sets bits left to right' do
+      @test.set_bit(0x00, 7).should == 0x01 # 0000 0001
+      @test.set_bit(0x00, 0).should == 0x80 # 1000 0000
+    end
+
+    it 'sets bits right to left' do
+      @test.set_bit(0x00, 7, false).should == 0x80 # 1000 0000
+      @test.set_bit(0x00, 0, false).should == 0x01 # 0000 0001
+    end    
+
+    it 'does not modify other bits' do
+      byte = 0x00
+      (0..7).each {|n| byte = @test.set_bit(byte, n) }
+      byte.should == 0xff
+    end
+
+    it 'does not modify a previously set bit' do
+      @test.set_bit(0x88, 0).should == 0x88
+      @test.set_bit(0x11, 0, false).should == 0x11
+    end
+  end
+
+  describe '.unset_bit' do
+    it 'unsets bits left to right' do
+      @test.unset_bit(0xff, 7).should == 0xfe # 1111 1110
+      @test.unset_bit(0xff, 0).should == 0x7f # 0111 1111
+    end
+
+    it 'unsets bits right to left' do
+      @test.unset_bit(0xff, 7, false).should == 0x7f # 0111 1111
+      @test.unset_bit(0xff, 0, false).should == 0xfe # 1111 1110
+    end
+
+    it 'does not modify other bits' do
+      byte = 0xff
+      (0..7).each {|n| byte = @test.unset_bit(byte, n) }
+      byte.should == 0x00
+    end
+
+    it 'does not modify a previously unset bit' do
+      @test.unset_bit(0x77, 0).should == 0x77 # 0111 0111
+      @test.unset_bit(0xee, 0, false).should == 0xee # 1110 1110
+    end
+  end
+
+  describe '.bit_set?' do
+    it 'should determine if a bit within a byte is set' do
+      (0..15).each {|n| @test.bit_set?(0x00, n).should be_false }
+      (0..15).each {|n| @test.bit_set?(0xff, n).should be_true }
+    end
+
+    it 'finds set bit left to right' do
+      byte = 0xf0 # 11110000
+      (0..3).each {|n| @test.bit_set?(byte, n).should be_true }
+      (4..7).each {|n| @test.bit_set?(byte, n).should be_false }
+    end
+
+    it 'finds set bit right to left' do
+      byte = 0xf0 # 11110000
+      (0..3).each {|n| @test.bit_set?(byte, n, false).should be_false }
+      (4..7).each {|n| @test.bit_set?(byte, n, false).should be_true }
+    end
+  end
+end

data/spec/reader_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+describe Bitindex::Reader do
+
+  describe '.initialize' do
+    it 'requires a valid file path' do
+      expect { Bitindex::Reader.new }.to raise_error
+      expect { Bitindex::Reader.new 'not_there.bit' }.to raise_error
+
+      filepath = 'is_there.bit'
+      FileUtils.touch filepath
+      r = Bitindex::Reader.new filepath
+      r.filepath.should == filepath
+      FileUtils.rm filepath 
+    end
+    
+    it 'can set left to right as an option' do
+
+      filepath = 'is_there.bit'
+      FileUtils.touch filepath
+
+      r = Bitindex::Reader.new filepath
+      r.ltor.should be_true
+
+      r = Bitindex::Reader.new filepath, { ltor: false }
+      r.ltor.should be_false
+
+      FileUtils.rm filepath 
+    end
+  end
+
+  describe '.is_set?' do
+    it 'can read values in a single byte file' do
+      
+      filepath = 'reader.bit'
+      File.open(filepath, 'wb') do |io|
+        io.putc 0x88 # 1000 1000
+      end
+
+      r = Bitindex::Reader.new filepath
+      r.is_set?(0).should be_true
+      r.is_set?(1).should be_false
+      r.is_set?(3).should be_false
+      r.is_set?(4).should be_true
+      r.is_set?(5).should be_false
+      r.is_set?(7).should be_false
+
+      
+      r = Bitindex::Reader.new filepath, { ltor: false }
+      r.is_set?(0).should be_false
+      r.is_set?(1).should be_false
+      r.is_set?(3).should be_true
+      r.is_set?(4).should be_false
+      r.is_set?(5).should be_false
+      r.is_set?(7).should be_true
+
+      FileUtils.rm filepath
+    end
+
+    it 'can read values in a multi-byte file' do
+
+      filepath = 'reader.bit'
+
+      # 1111 1111 0000 0000 1000 1000
+      File.open(filepath, 'wb') do |io|
+        io.putc 0xff
+        io.putc 0x00
+        io.putc 0x88
+      end
+
+      r = Bitindex::Reader.new filepath
+      r.is_set?(0).should be_true
+      r.is_set?(8).should be_false
+      r.is_set?(16).should be_true
+      r.is_set?(24).should be_false
+
+      r = Bitindex::Reader.new filepath, { ltor: false }
+      r.is_set?(0).should be_true
+      r.is_set?(8).should be_false
+      r.is_set?(16).should be_false
+      r.is_set?(24).should be_false
+
+      FileUtils.rm filepath
+    end
+  end
+
+  describe '.all_set' do
+    it 'can read values in a single byte file' do
+      filepath = 'reader.bit'
+      File.open(filepath, 'wb') do |io|
+        io.putc 0x77 # 0111 0111
+      end    
+
+      r = Bitindex::Reader.new filepath
+      r.all_set(0..10).should == [1,2,3,5,6,7]
+
+      r = Bitindex::Reader.new filepath, ltor: false
+      r.all_set(0..10).should == [0,1,2,4,5,6]
+
+      FileUtils.rm filepath
+    end
+
+    it 'can read values in a multi-byte file' do
+      filepath = 'reader.bit'
+      # 1111 1111 0000 0000 1000 1000
+      File.open(filepath, 'wb') do |io|
+        io.putc 0xff
+        io.putc 0x00
+        io.putc 0x88
+      end
+
+      r = Bitindex::Reader.new filepath
+      r.all_set(0..100).should == [0,1,2,3,4,5,6,7,16,20]
+
+      r = Bitindex::Reader.new filepath, ltor: false
+      r.all_set(0..100).should == [0,1,2,3,4,5,6,7,19,23]
+
+      FileUtils.rm filepath
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rspec'
+require 'bitindex'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter     = 'documentation'
+end

data/spec/writer_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+describe Bitindex::Writer do
+
+  describe '.initialize' do
+    it 'requires a filepath and size' do
+      expect { Bitindex::Writer.new }.to raise_error
+      expect { Bitindex::Writer.new 'file.bit' }.to raise_error
+      expect { Bitindex::Writer.new 10 }.to raise_error
+      w = Bitindex::Writer.new 'file.bit', 10
+      w.filepath.should == 'file.bit'
+      w.size.should == 10
+    end
+
+    it 'can set "left to right" as an option' do
+      w = Bitindex::Writer.new 'file.bit', 10
+      w.ltor.should be_true
+
+      w = Bitindex::Writer.new 'file.bit', 10, { ltor: false }
+      w.ltor.should be_false
+    end
+  end
+
+  describe '.write' do
+    it 'writes a single byte file' do
+      filepath = 'file.bit'
+      w = Bitindex::Writer.new filepath, 1
+      w.write [0, 1, 5, 7]
+      
+      File.open(filepath, 'rb') do |io|
+        io.size.should == 1
+        io.getbyte.should == 0xc5  # 1100 0101
+      end
+  
+      FileUtils.rm filepath
+    end
+
+    it 'writes a multi-byte file' do
+      filepath = 'file.bit'
+      w = Bitindex::Writer.new filepath, 3
+      w.write [0,1,2,3,4,5,6,7,16]
+      
+      File.open(filepath, 'rb') do |io|
+        io.size.should == 3
+        io.getbyte.should == 0xff
+        io.getbyte.should == 0x00
+        io.getbyte.should == 0x80
+      end
+
+      FileUtils.rm filepath
+    end
+
+    it 'writes a sparse multi-byte file' do
+      filepath = 'file.bit'
+      filesize = 1024
+      w = Bitindex::Writer.new filepath, filesize
+      w.write [1,2,3,8000]
+
+      File.open(filepath, 'rb') do |io|
+        io.size.should == filesize
+        io.seek(999, IO::SEEK_SET)
+        io.getbyte.should == 0x00
+        io.getbyte.should == 0x80
+        io.getbyte.should == 0x00
+      end
+
+      FileUtils.rm filepath
+    end
+  end
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: bitindex
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- Galen Palmer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-07-08 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 10
+        version: "2.10"
+  type: :development
+  version_requirements: *id001
+description: This library contains a Writer and Reader interface to create files where the position of a bit within the file indicates a true or false value.
+email: 
+- palmergs@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- lib/bitindex/common.rb
+- lib/bitindex/reader.rb
+- lib/bitindex/version.rb
+- lib/bitindex/writer.rb
+- lib/bitindex.rb
+- spec/common_spec.rb
+- spec/reader_spec.rb
+- spec/spec_helper.rb
+- spec/writer_spec.rb
+- README.md
+has_rdoc: true
+homepage: http://malachitedesigns.com
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: bitindex
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Build a file where, given an integer value, the bit at that position indicates true or false.
+test_files: 
+- spec/common_spec.rb
+- spec/reader_spec.rb
+- spec/spec_helper.rb
+- spec/writer_spec.rb