bitstream 0.0.1

data/README.en

@@ -0,0 +1,71 @@
+= BitStream
+
+BitStream is a mixin to write data structures of bit streams such as picture, music, movie files, and e.t.c.. You can refer contents of bit streams even when you are defining the data structures. With the function, you can write a data structure easily that the header contains the data length of the body field.
+
+== Installation
+
+gem install bitstream
+
+== Sample
+
+gzip.rb: A metadata definition of a gzip file.
+( http://www.gzip.org/zlib/rfc-gzip.html )
+
+  require 'bitstream'
+
+  class Gzip
+  
+    include BitStream
+    byte_order :little_endian
+
+    FHCRC    = 1 << 1
+    FEXTRA   = 1 << 2
+    FNAME    = 1 << 3
+    FCOMMENT = 1 << 4
+
+    fields do
+      unsigned :id1,   8
+      unsigned :id2,   8
+      unsigned :cm,    8
+      unsigned :flg,   8
+      unsigned :mtime, 32
+      unsigned :xfl,   8
+      unsigned :os,    8
+      if (flg & FEXTRA) != 0
+        unsigned :xlen, 16
+        string   :extra_field, xlen
+      end
+      if (flg & FNAME) != 0
+        # cstring means a NULL-terminated string.
+        cstring :original_file_name
+      end
+      if (flg & FCOMMENT) != 0
+        # cstring means a NULL-terminated string.
+        cstring :file_comment
+      end
+      if (flg & FHCRC) != 0
+        unsigned :crc16, 16
+      end
+    end
+
+  end
+
+gzip-viewer.rb: A viewer of the original file name of a gzip file.
+
+  require_relative 'gzip'
+
+  gzip = nil
+  File.open(ARGV[0], "rb") do |file|
+    gzip = Gzip.create(file.read)
+  end
+
+  if gzip.respond_to? :original_file_name
+    puts "original_file_name:#{gzip.original_file_name}"
+  else
+    puts "The gzip does not contain its original file name."
+  end
+
+== License
+
+This library is distributed under the dual license of the Ruby license (in the narrow sense) and the 2-clause BSD license. Please see http://www.ruby-lang.org and BSDL.
+Copyright (c) 2011, Natsuki Kawai.

data/lib/bitstream.rb

@@ -0,0 +1,454 @@
+# Author:: Natsuki Kawai (natsuki.kawai@gmail.com)
+# Copyright:: Copyright 2011 Natsuki Kawai
+# License:: 2-clause BSDL or Ruby's
+
+require 'random-accessible'
+
+require 'types/integer'
+require 'types/string'
+require 'types/cstring'
+require 'types/character'
+
+module BitStream
+
+  class BitStreamError < Exception
+  end
+
+  Properties = Struct.new(
+    :curr_offset, :fields, :mode, :raw_data,
+    :initial_offset, :user_props, :eval_queue
+  )
+
+  class ArrayProxy
+
+    include RandomAccessible
+
+    def initialize(instance)
+      @fields = []
+      @values = []
+      @updated = []
+      @instance = instance
+      @size = 0
+    end
+
+    def add_field(field)
+      @fields << field
+      @size += 1
+    end
+
+    def get_field(pos)
+      @fields[pos]
+    end
+
+    def read_access(pos)
+      unless @updated[pos]
+        field = @fields[pos]
+        unless field.has_read?
+          BitStream.read_one_field(field, @instance)
+        end
+        @values[pos] = field.value
+        @fields[pos] = nil
+        @updated[pos] = true
+      end
+      return @values[pos]
+    end
+
+    def write_access(pos, val)
+      @fields[pos] = nil
+      @values[pos] = val
+      @updated[pos] = true
+    end
+
+    def shrink(n)
+      @fields.pop(n)
+      @size -= n
+      dif = @values.size - @size
+      if dif > 0
+        @values.pop(dif)
+        @updated.pop(dif)
+      end
+    end
+
+    attr_reader :size
+
+  end
+
+  # TODO: Use data structure that enqueues and dequeues in O(1).
+  class Queue < Array
+
+    alias :enq :push
+    alias :deq :shift
+
+  end
+
+  class NestWrapper
+
+    def initialize(type, inherited_props, user_props)
+      @type = type
+      # TODO: Implement priority between the properties.
+      @props = user_props.merge(inherited_props)
+    end
+
+    def length
+      # TODO: Implement me.
+      nil
+    end
+
+    def read(s, offset)
+      instance = @type.create_with_offset(s, offset, @props)
+      [instance, instance.length]
+    end
+
+    #def write(s, offset, data)
+      # TODO: Implement me.
+    #end
+  end
+
+  module Utils
+
+    def self.class2symbol(type)
+      name = type.name.split("::").last
+      name = self.camel2snake(name).intern
+    end
+
+    def self.camel2snake(camel)
+      snake = camel.dup
+      snake[0] = snake[0].downcase
+      snake.gsub(/[A-Z]/) do |s|
+        "_" + s.downcase
+      end
+    end
+
+  end
+
+  def self.read_one_field(value, instance)
+    props = instance.bitstream_properties
+    queue = props.eval_queue
+
+    while value.offset.nil?
+      field = queue.deq
+      field.offset = props.curr_offset
+      length = field.length
+      length = field.decide_length if length.nil?
+      props.curr_offset += length
+    end
+    value.read
+  end
+  
+  def self.index_all_fields(instance)
+    props = instance.bitstream_properties
+    queue = props.eval_queue
+    
+    queue.each do |field|
+      field.offset = props.curr_offset
+      length = field.length
+      length = field.decide_length if length.nil?
+      props.curr_offset += length
+    end
+    queue.clear
+  end
+  
+  class Value
+    
+    def initialize(type, raw_data)
+      @type = type
+      @raw_data = raw_data
+      @length = @type.length if @type.respond_to? :length
+      @has_read = false
+    end
+
+    def has_read?
+      @has_read
+    end
+
+    def read
+      unless @has_read
+        if @offset.nil?
+          raise "Has not been set offset."
+        else
+          @value, @length = @type.read(@raw_data, @offset)
+          @has_read = true
+        end
+      end
+      return @value
+    end
+
+    def decide_length
+      # @length must not be nil if @has_read.
+      if @length.nil?
+        if @offset.nil?
+          raise "Has not been set offset."
+        else
+          @value, @length = @type.read(@raw_data, @offset)
+          @has_read = true
+        end
+      end
+      return @length
+    end
+
+    attr_reader :length, :value
+    attr_accessor :offset
+
+  end
+
+  module ClassMethods
+
+    def initialize_for_class_methods(types)
+      @field_defs = []
+      @fields = {}
+      @types = types.dup
+      @index = 0
+      @class_props = {}
+      @bitstream_mutex = Mutex.new
+    end
+
+    def fields(&field_def)
+      @field_defs << field_def
+    end
+
+    def initialize_instance(raw_data, instance)
+      props = instance.bitstream_properties
+      props.mode = :field_def
+      props.user_props.merge!(@class_props)
+      @bitstream_mutex.synchronize do
+        @instance = instance
+
+        @field_defs.each do |field_def|
+          field_def.call
+        end
+      end
+    end
+
+
+    def self.types
+      @types
+    end
+    
+    def self.register_types(types)
+      types.each do |t|
+        register_type(t, nil)
+      end
+    end
+    
+    def self.register_type(type, name = nil)
+      if name.nil?
+        name = Utils.class2symbol type
+      end
+      
+      @types = {} if @types.nil?
+      @types[name] = type
+
+      add_type(type, name, self)
+    end
+
+    def self.alias_type(alias_name, aliasee)
+      @types[alias_name] = @types[aliasee]
+      alias_method(alias_name, aliasee)
+    end
+
+    def props
+      @instance.bitstream_properties.user_props
+    end
+
+    def byte_order(order)
+      @class_props[:byte_order] = order.intern
+    end
+
+    def self.add_type(type, name = nil, bs = self)
+      bs.instance_eval do
+        define_method(name) do |*args|
+          name = args.shift.intern
+          #if respond_to? name
+          #  raise "#{name} has already defined."
+          #end
+          
+          props = @instance.bitstream_properties
+          fields = props.fields
+          queue = props.eval_queue
+          user_props = props.user_props
+          
+          case props.mode
+          when :field_def
+            if type.respond_to? :read
+              type_instance = type
+            else
+              type_instance = type.instance(user_props, *args)
+            end
+            field = Value.new(type_instance, props.raw_data)
+            queue.enq(field)
+            @instance.bitstream_properties.fields[name] = field
+
+            name_in_method = name
+
+            define_method name do
+              field = bitstream_properties.fields[name_in_method]
+              if field.value.nil?
+                BitStream.read_one_field(field, self)
+              end
+              field.value
+            end
+
+            instance = @instance
+            singleton_class.instance_eval do
+              define_method name_in_method do
+                instance.send(name_in_method)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def array(name, size, type_name, *type_args)
+      name = name.intern
+      type_name = type_name.intern
+      type = @types[type_name]
+      props = @instance.bitstream_properties
+      queue = props.eval_queue
+      user_props = props.user_props
+
+      if type.nil?
+        raise BitStreamError, "There is no type named \"#{type_name}\""
+      end
+
+      if type.respond_to? :read
+        unless type_args.empty?
+          raise BitStreamError, "#{type} does not accept any arguments."
+        end
+        type_instance = type
+      else
+        type_instance = type.instance(user_props, *type_args)
+      end
+
+      case props.mode
+      when :field_def
+        field = ArrayProxy.new(@instance)
+        size.times do
+          field_element = Value.new(type_instance, props.raw_data)
+          field.add_field(field_element)
+          queue.enq(field_element)
+        end
+
+        define_method name do
+          field
+        end
+
+        name_in_method = name
+        instance = @instance
+        singleton_class.instance_eval do
+          define_method name do
+            instance.send(name_in_method)
+          end
+        end
+      end
+    end
+
+    def dyn_array(name, type_name, *type_args)
+      name = name.intern
+      type_name = type_name.intern
+      type = @types[type_name]
+      props = @instance.bitstream_properties
+      fields = props.fields
+      queue = props.eval_queue
+      user_props = props.user_props
+
+      if type.nil?
+        raise BitStreamError, "There is no type named \"#{type_name}\""
+      end
+
+      if type.respond_to? :read
+        unless type_args.empty?
+          raise BitStreamError, "#{type} does not accept any arguments."
+        end
+        type_instance = type
+      else
+        type_instance = type.instance(user_props, *type_args)
+      end
+
+      case props.mode
+      when :field_def
+        if fields[name].nil?
+          fields[name] = ArrayProxy.new(@instance)
+        end
+        field = Value.new(type_instance, props.raw_data)
+        fields[name].add_field(field)
+        queue.enq(field)
+
+        name_in_method = name
+        
+        define_method name do
+          return fields[name_in_method]
+        end
+
+        instance = @instance
+        singleton_class.instance_eval do
+          define_method name do
+            instance.send(name_in_method)
+          end
+        end
+      end
+    end
+
+    def add_type(type, name = nil)
+      if name.nil?
+        name = Utils.class2symbol(type)
+      end
+      @types[name] = type
+      ClassMethods.add_type(type, name, self.singleton_class)
+    end
+
+    register_types [Unsigned, Signed, Cstring, String, Char]
+    alias_type :unsigned_int, :unsigned
+    alias_type :int, :signed
+
+    def create(s, props = {})
+      create_with_offset(s, 0, props)
+    end
+
+    def create_with_offset(s, offset, props = {})
+      klass = Class.new(self)
+      instance = klass.new(s, offset)
+      instance.bitstream_properties.user_props = props
+      initialize_instance(s, instance)
+      return instance
+    end
+
+    #def method_missing(name, *args)
+      # TODO: Support methods like "int16" "uint1"
+    #  super name, args
+    #end
+
+    def instance(inherited_props, user_props = {})
+      NestWrapper.new(self, inherited_props, user_props)
+    end
+    
+  end
+
+  def self.included(obj)
+    obj.extend ClassMethods
+    obj.initialize_for_class_methods(ClassMethods.types)
+  end
+
+  def initialize(s, offset = 0)
+    props = Properties.new
+    props.curr_offset = offset
+    props.fields = {}
+    props.raw_data = s
+    props.initial_offset = offset
+    props.eval_queue = Queue.new
+    @bitstream_properties = props
+  end
+
+  def length
+    BitStream.index_all_fields(self)
+    props = @bitstream_properties
+    props.curr_offset - props.initial_offset
+  end
+
+  #def properties=(props)
+    # Method to override.
+  #end
+
+  attr_accessor :bitstream_properties
+
+end

data/lib/types/character.rb

@@ -0,0 +1,35 @@
+# Author:: Natsuki Kawai (natsuki.kawai@gmail.com)
+# Copyright:: Copyright 2011 Natsuki Kawai
+# License:: 2-clause BSDL or Ruby's
+
+
+module BitStream
+
+  class Char
+
+    @instance = new
+
+    def self.instance(props)
+      @instance
+    end
+
+    def length
+      8
+    end
+
+    def read(s, offset)
+      byteoffset = offset / 8
+      bitoffset  = offset % 8
+
+      value = s[byteoffset]
+
+      return [value, 8]
+    end
+
+    def write(s, offset, value)
+    end
+
+  end
+
+end
+

data/lib/types/cstring.rb

@@ -0,0 +1,40 @@
+# Author:: Natsuki Kawai (natsuki.kawai@gmail.com)
+# Copyright:: Copyright 2011 Natsuki Kawai
+# License:: 2-clause BSDL or Ruby's
+
+
+module BitStream
+
+  class Cstring
+
+    @instance = new
+
+    def self.instance(props)
+      @instance
+    end
+
+    def length
+      nil
+    end
+
+    def read(s, offset)
+      byteindex = offset / 8
+      bitindex  = offset % 8
+      val = ""
+      begin
+        byte = s[byteindex].unpack('C')[0]
+        val << byte
+        byteindex += 1
+      end while byte != 0
+
+      bytelen = val.size
+      val.slice!(val.size - 1)
+      return [val, 8 * bytelen]
+    end
+
+    def write(s, offset, data)
+    end
+
+  end
+
+end

data/lib/types/integer.rb

@@ -0,0 +1,131 @@
+# Author:: Natsuki Kawai (natsuki.kawai@gmail.com)
+# Copyright:: Copyright 2011 Natsuki Kawai
+# License:: 2-clause BSDL or Ruby's
+
+
+module BitStream
+
+  class Unsigned
+
+    @be_instances = Hash.new do |hash, key|
+      hash[key] = new(key, true)
+    end
+
+    @le_instances = Hash.new do |hash, key|
+      hash[key] = new(key, false)
+    end
+
+    BE_SYMBOLS = [:big_endian, :be, :msb_first, :motorola, nil]
+    LE_SYMBOLS = [:little_endian, :le, :lsb_first, :intel]
+
+    def self.instance(props, bit_width)
+      byte_order = props[:byte_order]
+      if BE_SYMBOLS.include?(byte_order)
+        @be_instances[bit_width]
+      elsif LE_SYMBOLS.include?(byte_order)
+        @le_instances[bit_width]
+      else
+        STDERR.puts("Unknown byte order #{byte_order.inspect}.",
+                    "Assuming that the byte order is big endian.")
+        @be_instances[bit_width]
+      end
+    end
+
+    attr_reader :bit_width
+
+    def initialize(bit_width, big_endian)
+      @bit_width = bit_width
+      @big_endian = big_endian
+    end
+
+    def length
+      @bit_width
+    end
+
+    def read(s, offset)
+      value = 0
+      byteoffset = offset / 8
+      bitoffset  = offset % 8
+      bytelength = (@bit_width + bitoffset + 7) / 8
+
+      bytes = s[byteoffset, bytelength].unpack('C*')
+      bytes.reverse! unless @big_endian
+
+      bytes.each do |b|
+        value <<= 8
+        value |= b
+      end
+
+      value &= ~(-1 << (bytelength * 8 - bitoffset))
+      value >>= (8 - (@bit_width + bitoffset) % 8) % 8
+
+      return [value, @bit_width]
+    end
+
+    def write(s, offset, value)
+      byteoffset = offset / 8
+      bitoffset  = offset % 8
+
+      if bitoffset != 0
+        raise "#{self.class.name}#write has not supported non-byte-aligned fields yet."
+      end
+      unless @big_endian
+        raise "#{self.class.name}#write has not supported little endian yet."
+      end
+
+      i = 0
+      tail = ""
+      while value != 0
+        index = byteoffset + @bit_width / 8 + i - 1
+        if s.bytesize <= index
+          tail.insert(0, [value & 0xff].pack('C'))
+        else
+          s[index] = [value & 0xff].pack('C')
+        end
+        value >>= 8
+        i -= 1
+      end
+      s << tail
+
+      return @bit_width
+    end
+
+  end
+
+  class Signed
+
+    @instances = Hash.new do |hash, key|
+      hash[key] = new(key)
+    end
+
+    def self.instance(props, bit_width)
+      unsigned = Unsigned.instance(props, bit_width)
+      return @instances[unsigned]
+    end
+
+    def initialize(unsigned)
+      @unsigned = unsigned
+    end
+
+    def length
+      @unsigned.length
+    end
+
+    def read(s, offset)
+      val, len = @unsigned.read(s, offset)
+      mask = -1 << (len - 1)
+      if (val & mask) != 0
+        val |= mask
+      end
+      return [val, len]
+    end
+
+    def write(s, offset, value)
+      mask = ~(-1 << len)
+      value &= mask
+      return @unsigned.write(s, offset, value)
+    end
+
+  end
+
+end