binaryparse 0.1.0

data/lib/blocker.rb

@@ -0,0 +1,736 @@
+# The purpose of BinaryBlocker is to parse semi-complicated binary files. It 
+# can handle all the various data types supported by Array#pack and 
+# String#unpack, but the real benefit is with more complicated layouts, that
+# might include files with headers and a variable number of body records 
+# followed by a footer. It also supports, packed numeric 
+# (BinaryBlocker::PackedNumberEncoder) and date fields 
+# (BinaryBlocker::PackedDateEncoder)
+
+require 'uconv'
+require 'date'
+require 'time'
+
+module BinaryBlocker
+  class << self
+    def klasses
+      @klasses ||= {}
+    end
+
+    # To simplify naming of classes and laying out fields in your structures
+    # they can be registered:
+    #
+    # BinaryBlocker.register_klass(:string, FixedStringEncoder)
+    def register_klass(sym, klass)
+      @klasses ||= {}
+      @klasses[sym] = klass
+    end
+
+    # Handy helper method that returns the size of a given pack/unpack format
+    # string
+    def sizeof_format(format)
+      length = 0
+      format.scan(/(\S_?)\s*(\d*)/).each do |directive,count|
+        count = count.to_i
+        count = 1 if count == 0
+        
+        length += case directive
+        when 'A', 'a', 'C', 'c', 'Z', 'x' : count
+        when 'B', 'b' : (count / 8.0).ceil
+        when 'D', 'd', 'E', 'G' : count * 8
+        when 'e', 'F', 'f', 'g' : count * 4
+        when 'H', 'h' : (count / 2.0).ceil
+        when 'I', 'i', 'L', 'l', 'N', 'V' : count * 4
+        when 'n', 'S', 's', 'v' : count * 2
+        when 'Q', 'q' : count * 8
+        when 'X' : count * -1
+        else raise ArgumentError.new("#{directive} is not supported in sizeof_format")
+        end
+      end
+      
+      length
+    end
+
+    def pack_symbols
+      { 
+        :int8   => 'c', 
+        :uint8  => 'C',
+        :int16  => 's',
+        :uint16 => 'S',
+        :int32  => 'i',   
+        :uint32 => 'I',
+        :int64  => 'q',    
+        :uint64 => 'Q'
+      }
+    end
+
+    # As we have to process some fields to determine what type they are
+    # (that is we sometimes start and half to backup), this routine takes
+    # any io (that can be repositioned) and yields to a block -- if the 
+    # block returns not _true_ it will reset the original position of the 
+    # io stream. If you need to use BinaryBlocker with a non-repositioning
+    # stream (like a TCP/IP stream), see the handy BufferedIO class.
+    def with_guarded_io_pos(io)
+      pos = io.pos
+      status = yield io
+    ensure
+      io.pos = pos unless status
+    end
+    
+  end
+  
+  # This is the base class for all the various encoders. It supports a variety
+  # of options (as Symbols in a Hash):
+  #
+  # pre_block::     passed the value before being blocked
+  # post_block::    passed the blocked value before returned
+  # pre_deblock::   passed the io before attempting to deblock it (hard to imagine
+  #                 why you would need this but I was compelled by orthaganality)
+  # post_deblock::  passed the deblocked value before being stored internally
+  # get_filter::    more info
+  # set_filter::    all done
+  #
+  # It also supports either a string or io parameter which will be used to 
+  # initialize the class
+  class Encoder
+    
+    # Parameters: (io | buf, options_hash)
+    #
+    # Options (lambda):
+    #
+    # pre_block::     passed the value before being blocked
+    # post_block::    passed the blocked value before returned
+    # pre_deblock::   passed the io before attempting to deblock it (hard to imagine
+    #                 why you would need this but I was compelled by orthaganality)
+    # post_deblock::  passed the deblocked value before being stored internally
+    # get_filter::    more info
+    # set_filter::    all done
+    #   
+    def initialize(*opts)
+      initialize_options(*opts)
+      initialize_data(*opts)
+    end
+    
+    # 
+    def block(io=nil)
+      val = if @pre_block
+        @pre_block.call(self.value)
+      else
+        self.value
+      end
+      result = internal_block(val)
+      if @post_block
+        result = @post_block.call(result)
+      end
+      io.write(result) if io
+      result
+    end
+
+    # This routine takes an io and will parse the stream 
+    # on success it returns the object, on failure it 
+    # returns a nil
+    def deblock(io)
+      with_guarded_io_pos(io) do
+        if @pre_deblock
+          # does this serve any real purpose? other
+          # than making me feel good and orthoginal
+          io = @pre_deblock.call(io)
+        end
+        self.value = internal_deblock(io)
+        if @post_deblock
+          self.value = @post_deblock.call(self.value)
+        end
+        self.value
+      end
+    end
+    
+    protected
+
+    # Override valid? to allow check constraints on a particular 
+    # record type, on failure 
+    def valid?
+      true
+    end
+      
+    def value
+      v = @value
+      if @get_filter
+        @get_filter.call(v) 
+      else
+        v
+      end
+    end
+    
+    def value=(val)
+      val = @set_filter.call(val) if @set_filter
+      @value = val
+    end
+    
+    def initialize_options(*opts)
+      @opts ||= {}
+      opts = opts.find { |o| o.respond_to? :to_hash }
+      if opts
+        @opts = @opts.merge(opts)
+        @pre_block = @opts[:pre_block]
+        @pre_deblock = @opts[:pre_deblock]
+        
+        @get_filter = @opts[:get_filter]
+        @set_filter = @opts[:set_filter]
+        
+        @post_block = @opts[:post_block]
+        @post_deblock = @opts[:post_deblock]
+      end
+    end
+    
+    def initialize_data(*opts)
+      data = opts.find { |o| !o.respond_to? :to_hash }
+      if data.respond_to? :to_str
+        raise "Unable to parse block" unless deblock(StringIO.new(data))
+      elsif data.respond_to? :read
+        raise "Unable to parse block" unless deblock(data)        
+      end
+    end
+    
+    def with_guarded_io_pos(io)
+      result = nil
+      BinaryBlocker.with_guarded_io_pos(io) do
+        result = yield io
+        valid?
+      end && result
+    end
+
+    def internal_block(value) 
+      value
+    end
+    
+    def internal_deblock(io)
+      ''
+    end    
+  end
+
+  # All Encoders that store multiple items subclass from 
+  # here. 
+  class GroupEncoder < Encoder
+    class << self
+      def inherited(obj)
+        obj.instance_eval do
+          @klasses = BinaryBlocker.klasses.clone
+          @attributes = []
+          @lookup = {}
+        end
+        super
+      end
+
+      # One and only one (this is the easiest :-)
+      def has_one(sym, klass, *opts) 
+        klass = @klasses[klass] if @klasses[klass]
+        @lookup[sym] = @attributes.size
+        @attributes << lambda { klass.new(*opts) }
+      end
+      
+      def include_klasses(klasses, *opts)
+        klasses = klasses.map do |k| 
+          case
+          when @klasses[k]          : lambda { @klasses[k].new(*opts) }
+          when k.respond_to?(:call) : k
+          when k.respond_to?(:new)  : lambda { k.new(*opts) }
+          else raise "Unable to process class: #{k}"
+          end
+        end
+      end
+      
+      def has_one_of(sym, klasses, *opts)
+        klasses = include_klasses(klasses, *opts) 
+        @lookup[sym] = @attributes.size 
+        @attributes << lambda { OneOfEncoder.new(klasses, *opts) }
+      end
+
+      def has_counted_array(sym, count_type, klasses, *opts)
+        klasses = include_klasses(klasses, *opts)
+        @lookup[sym] = @attributes.size 
+        @attributes << lambda { CountedArrayEncoder.new(count_type, klasses, *opts) }
+      end
+      
+      def has_fixed_array(sym, count, klasses, *opts)
+        klasses = include_klasses(klasses, *opts)
+        @lookup[sym] = @attributes.size 
+        @attributes << lambda { FixedArrayEncoder.new(count, klasses, *opts) }
+      end
+
+      def has_bit_field(sym, type, bit_info, *opts)
+        @lookup[sym] = @attributes.size
+        @attributes << lambda { BitFieldEncoder.new(type, bit_info, *opts) }
+      end
+          
+       def has_list_of(sym, klasses, *opts)
+        klasses = include_klasses(klasses, *opts)
+        @lookup[sym] = @attributes.size 
+        @attributes << lambda { ListOfEncoder.new(klasses, *opts) }
+      end
+      
+      def register_klass(sym, klass)
+        @klasses ||= {}
+        @klasses[sym] = klass
+      end
+      
+      def clear_registered_klasses
+        @klasses = {}
+      end
+      
+      def attributes
+        @attributes
+      end
+      
+      def lookup
+        @lookup
+      end
+    end
+    
+    def initialize(*opts)
+      @lookup = self.class.lookup.clone
+      @value = self.class.attributes.map { |a| a.call }
+      super      
+    end 
+    
+    def block
+      @value.inject("") { |a,b| a + b.block }
+    end
+    
+    def deblock(io)
+      BinaryBlocker.with_guarded_io_pos(io) do
+        @value.all? { |o| o.deblock(io) }
+      end
+    end    
+    
+    def method_missing(sym, *args)
+      super unless @lookup
+      if pos = @lookup[sym]
+        return @value[pos].value
+      else
+        sym = sym.to_s
+        if sym[-1] == ?=
+          if pos = @lookup[sym[0..-2].to_sym]
+            return @value[pos].value = args.first
+          end
+        end
+      end
+      super
+    end
+
+    def value
+      self
+    end
+    
+    def value=(val)
+      raise "Now what?"
+    end    
+    
+    def valid?
+      @value.all? { |a| a.valid? }
+    end
+  end  
+    
+  class SimpleEncoder < Encoder
+    
+    def self.register(sym, fmt, *opts)
+      klass = Class.new(SimpleEncoder)
+      klass.send(:define_method,:initialize) do |*opts|
+        initialize_options(*opts)
+      
+        @format = fmt
+        @length = BinaryBlocker.sizeof_format(@format)
+        
+        @key = @opts[:key]
+        @valid = @opts[:valid]
+        
+        initialize_data(*opts)
+      end
+      BinaryBlocker.register_klass(sym, klass)
+    end
+    
+    def internal_block(val)
+      [val].pack(@format)
+    end
+    
+    def internal_deblock(io)
+      buffer = io.read(@length)
+      result = buffer.unpack(@format)
+      result.first
+    end
+    
+    def valid?
+      if @valid
+        @valid.call(self.value)
+      else
+        self.value != nil && (@key == nil || @key === self.value)
+      end
+    end
+    
+    def inspect
+      "SE: #{self.class} - #{@format} - #{@length} - #{self.value.inspect}"
+    end
+  end
+  
+  SimpleEncoder.register( :int8, 'c')
+  SimpleEncoder.register(:uint8, 'C')
+  SimpleEncoder.register( :int16, 's')
+  SimpleEncoder.register(:uint16, 'S')
+  SimpleEncoder.register( :int32, 'i')    
+  SimpleEncoder.register(:uint32, 'I')
+  SimpleEncoder.register( :int64, 'q')    
+  SimpleEncoder.register(:uint64, 'Q')
+  
+  class FixedStringEncoder < SimpleEncoder
+    def initialize(*opts)
+      initialize_options(*opts)
+      
+      @length = @opts[:length].to_i
+      raise ArgumentError.new("Missing or invalid string length") unless @length > 0
+      @format = "Z#{@length}"
+      
+      @key = @opts[:key]
+      @valid = @opts[:valid]
+      
+      initialize_data(*opts)
+    end
+  end
+  BinaryBlocker.register_klass(:string, FixedStringEncoder)
+  
+  class FixedUTF16StringEncoder < SimpleEncoder
+    def initialize(*opts)
+      initialize_options(*opts)
+      
+      @length = @opts[:length].to_i
+      @length *= 2
+      raise ArgumentError.new("Missing or invalid string length") unless @length > 0
+      @format = "Z#{@length}"
+      
+      @key = @opts[:key]
+      @valid = @opts[:valid]
+      
+      initialize_data(*opts)
+    end
+    
+    def internal_block(val)
+      [Uconv.u8tou16(val || "")].pack(@format)
+    end
+    
+    def internal_deblock(io)
+      buffer = io.read(@length)
+      Uconv.u16tou8(buffer).sub(/\000+$/,'')
+    end    
+  end
+  BinaryBlocker.register_klass(:utf16_string, FixedUTF16StringEncoder)
+  BinaryBlocker.register_klass(:utf16, FixedUTF16StringEncoder)
+
+  class PackedNumberEncoder < SimpleEncoder
+    def initialize(*opts)
+      initialize_options(*opts)
+      
+      @length = @opts[:length].to_i
+      raise ArgumentError.new("Missing or invalid string length") unless @length > 0
+      @length += 1 if @length[0] == 1
+      @bytes = @length / 2
+      @format = "H#{@length}"
+      
+      @key = @opts[:key]
+      @valid = @opts[:valid]
+      
+      initialize_data(*opts)
+    end
+    
+    def internal_block(val)
+      ["%0#{@length}d" % val].pack(@format)
+    end
+    
+    def internal_deblock(io)
+      buffer = io.read(@bytes)
+      result = buffer.unpack(@format)
+      result.first.to_i
+    end    
+  end
+  BinaryBlocker.register_klass(:packed, PackedNumberEncoder)
+
+  class PackedDateEncoder < PackedNumberEncoder
+    def initialize_options(*opts)
+      super
+      @opts[:length] = 8
+    end
+    
+    def internal_block(val)
+      super val.year * 10000 + val.month * 100 + val.mday
+    end
+    
+    def internal_deblock(io)
+      buffer = io.read(@bytes)
+      result = buffer.unpack(@format)
+      year, month, day = result.first.unpack("A4A2A2").map { |v| v.to_i }
+      Date.civil(year, month, day)
+    end    
+        
+  end
+  BinaryBlocker.register_klass(:date, PackedDateEncoder)
+  
+  class PackedDateTimeEncoder < PackedNumberEncoder
+    def initialize_options(*opts)
+      super
+      @opts[:length] = 14
+    end
+    
+    def internal_block(val)
+      super sprintf("%04d%02d%02d%02d%02d%02d", val.year, val.month, val.mday, val.hour, val.min, val.sec).to_i
+    end
+    
+    def internal_deblock(io)
+      buffer = io.read(@bytes)
+      result = buffer.unpack(@format)
+      year, month, day, hour, min, sec = result.first.unpack("A4A2A2A2A2A2").map { |v| v.to_i }
+      Time.local(year, month, day, hour, min, sec)
+    end    
+        
+  end
+  BinaryBlocker.register_klass(:time, PackedDateTimeEncoder)
+  BinaryBlocker.register_klass(:datetime, PackedDateTimeEncoder)
+  
+  class OneOfEncoder < Encoder
+    
+    def inspect
+      "OneOf #{@classes.join(',')} -> #{@obj.class} -> #{@obj.inspect}" 
+    end
+    
+    def initialize(classes, *opts)
+      @classes = classes.map { |o| o.call(*opts) }
+      @obj = nil
+      super(*opts)
+    end
+    
+    def internal_block(val)
+      val.block if val
+    end
+    
+    def internal_deblock(io)
+      @obj = nil
+      with_guarded_io_pos(io) do
+        @classes.each do |obj|
+          if obj.deblock(io)
+            @obj = obj
+            break 
+          end 
+          false
+        end
+      end       
+      @obj
+    end
+    
+    def method_missing(sym, *args)
+      if @obj
+        @obj.send sym, *args
+      else
+        super
+      end
+    end
+
+    def valid?
+      @obj && @obj.valid?
+    end
+  end
+   
+  class FixedArrayEncoder < GroupEncoder
+    def initialize(count, classes, *opts)
+      initialize_options(*opts)
+      @count = count
+      @classes = classes
+      @value = Array.new(count) { OneOfEncoder.new(classes, *opts) }
+      initialize_data(*opts)
+    end
+    
+    def internal_block(val)
+      val.inject("") { |r, o| r + o.block }
+    end
+    
+    def deblock(io)
+      result = []
+      with_guarded_io_pos(io) do
+        @count.times do
+          result << OneOfEncoder.new(@classes, io, @opts)
+        end
+      end 
+      @value = result
+    end
+        
+    def [](offset)
+      raise RangeError.new("Access (#{offset}) out of range (#{@count})") unless (0...@count) === offset
+      @value[offset]
+    end
+    
+    def []=(offset,val)
+      raise RangeError.new("Access (#{offset}) out of range (#{@count})") unless (0...@count) === offset
+      @value[offset] = val
+    end
+  end
+
+  class ListOfEncoder < GroupEncoder
+    def initialize(classes, *opts)
+      initialize_options(*opts)
+      @count = 0
+      @classes = classes
+      @value = []
+      initialize_data(*opts)
+    end
+    
+    def internal_block(val)
+      val.inject("") { |r, o| r + o.block }
+    end
+    
+    def deblock(io)
+      result = []
+      with_guarded_io_pos(io) do
+        oe = OneOfEncoder.new(@classes, @opts)
+        while oe.deblock(io)
+          result << oe
+          oe = OneOfEncoder.new(@classes, @opts)
+        end
+      end 
+      @value = result
+    end
+        
+    def [](offset)
+      raise RangeError.new("Access (#{offset}) out of range (#{@value.size})") unless (0...@value.size) === offset
+      @value[offset]
+    end
+    
+    def []=(offset,val)
+      raise RangeError.new("Access (#{offset}) out of range (#{@value.size})") unless (0...@value.size) === offset
+      @value[offset] = val
+    end
+    
+    def <<(val)
+      @value << val
+    end    
+    
+    def size
+      @value.size
+    end
+    alias length size
+  end
+ 
+  class CountedArrayEncoder < GroupEncoder
+    def initialize(count_type, classes, *opts)
+      # this is dynamic now, but we init to zero for the range checking
+      @count_enc = BinaryBlocker.pack_symbols[count_type] || count_type
+      @count = 0 
+      initialize_options(*opts)
+      @classes = classes
+      @value = []
+      initialize_data(*opts)
+    end
+    
+    def internal_block(val)
+      buf = [val.size].pack(@count_enc)
+      val.inject(buf) { |r, o| r + o.block }
+    end
+    
+    def deblock(io)
+      length = BinaryBlocker.sizeof_format(@count_enc)
+      result = []
+      with_guarded_io_pos(io) do
+        @count = io.read(length).unpack(@count_enc)
+        @count.times do
+          result << OneOfEncoder.new(@classes, io, @opts)
+        end
+      end 
+      @value = result
+    end
+        
+    def [](offset)
+      raise RangeError.new("Access (#{offset}) out of range (#{@count})") unless (0...@count) === offset
+      @value[offset]
+    end
+    
+    def []=(offset,val)
+      raise RangeError.new("Access (#{offset}) out of range (#{@count})") unless (0...@count) === offset
+      @value[offset] = val
+    end
+    
+    def <<(val)
+      @count += 1
+      @value << val
+    end
+    
+    def size
+      @count
+    end
+    alias length size
+  end
+
+  class BitFieldEncoder < Encoder
+    def initialize(type, bit_info, *opts)
+      @type = BinaryBlocker.pack_symbols[type] || type
+      @length = BinaryBlocker.sizeof_format(@type)
+      @bit_info = {}
+      pos = 0
+      bit_info.each do |bi|
+        case bi
+        when Symbol
+          @bit_info[bi.to_sym] = [pos,1]
+          pos += 1
+          
+        when Fixnum
+          pos += bi
+          
+        when Array
+          @bit_info[bi.first.to_sym] = [pos, bi.last.to_i]
+          pos += bi.last.to_i
+        end
+      end
+      @value = 0
+      initialize_options(*opts)
+      initialize_data(*opts)
+    end
+   
+    def internal_block(val)
+      [val.raw_value].pack(@type)
+    end
+    
+    def internal_deblock(io)
+      buffer = io.read(@length)
+      result = buffer.unpack(@type)
+      result.first
+    end
+
+    def value
+      self
+    end
+    
+    def raw_value
+      @value
+    end
+    
+    def method_missing(sym, *args)
+      if (bi = @bit_info[sym])
+        v = @value >> bi.first
+        mask = (1 << bi.last) - 1
+        v = v & mask
+      else
+        sym = sym.to_s
+        if sym[-1] == ?=
+          if bi = @bit_info[sym[0..-2].to_sym]
+            @value &= ~(((1 << bi.last) - 1) << bi.first)
+            @value |= args.first.to_i << bi.first 
+            return @value
+          end
+        end
+        raise NoMethodError.new("undefined method `#{sym}''")
+        # I was using super, but it was throwing a different error
+        # which seemed wrong, not sure why
+      end
+    end
+        
+  end
+        
+  class Blocker < GroupEncoder
+    def inspect
+      "Blocker: #{@value.inspect} (index: #{@lookup})"
+    end        
+  end
+  
+end