bindata 0.5.0

data/TODO

@@ -0,0 +1,14 @@
+* Add a way to do something like: read a bunch of integers and stop reading
+  after reading an integer with a value of 0.
+
+* Add scoping so the value of a param doesn't need to call parent
+
+* Optimise int.rb for speed.
+
+* Maybe add an endian method to struct so you can say int16 instead of int16le.
+  - Should this by lazily evaluated, or evaluated only once when
+    instantiating fields?
+
+* Think how offset_of should work.
+
+* Raise error when a struct defines a name of an existing method

data/examples/gzip.rb

@@ -0,0 +1,174 @@
+require 'bindata'
+require 'forwardable'
+
+# An example of a reader / writer for the GZIP file format as per rfc1952.
+# Note that compression is not implemented to keep the example small.
+class Gzip
+  extend Forwardable
+
+  # Known compression methods
+  DEFLATE = 8
+
+  class Extra < BinData::Struct
+    uint16le :len,  :length => lambda { data.length }
+    string   :data, :initial_length => :len
+  end
+
+  class Header < BinData::Struct
+    uint16le :id,         :value => 0x8b1f, :check_value => 0x8b1f
+    uint8    :compression_method, :initial_value => DEFLATE
+    uint8    :flags,      :value => :calculate_flags_val,
+                          # Upper 3 bits must be zero
+                          :check_value => lambda { (value & 0xe0) == 0 }
+    uint32le :mtime
+    uint8    :extra_flags
+    uint8    :os,         :initial_value => 255   # unknown OS
+
+    # These fields are optional depending on the bits in flags
+    extra    :extra,      :readwrite => :extra?
+    stringz  :file_name,  :readwrite => :file_name?
+    stringz  :comment,    :readwrite => :comment?
+    uint16le :crc16,      :readwrite => :crc16?
+
+
+    ## Convenience methods for accessing and manipulating flags
+
+    attr_writer :text
+
+    # Access bits of flags
+    def text?;      flag_val(0) end
+    def crc16?;     flag_val(1) end
+    def extra?;     flag_val(2) end
+    def file_name?; flag_val(3) end
+    def comment?;   flag_val(4) end
+
+    def flag_val(bit) (flags & (1 << bit)) != 0 end
+
+    # Calculate the value of flags based on current state.
+    def calculate_flags_val
+      ((@text               ? 1 : 0) << 0) |
+
+      # Never include header crc.  This is because the current versions of the
+      # command-line version of gzip (up through version 1.3.x) do not
+      # support header crc's, and will report that it is a "multi-part gzip
+      # file" and give up.
+      ((!clear?(:crc16)     ? 0 : 0) << 1) |
+
+      ((!clear?(:extra)     ? 1 : 0) << 2) |
+      ((!clear?(:file_name) ? 1 : 0) << 3) |
+      ((!clear?(:comment)   ? 1 : 0) << 4)
+    end
+  end
+
+  class Footer < BinData::Struct
+    uint32le :crc32
+    uint32le :uncompressed_size
+  end
+
+  def initialize
+    @header = Header.new
+    @footer = Footer.new
+  end
+
+  attr_accessor :compressed
+  def_delegators :@header, :file_name=, :file_name, :file_name?
+  def_delegators :@header, :comment=, :comment, :comment?
+  def_delegators :@header, :compression_method
+  def_delegators :@footer, :crc32, :uncompressed_size
+
+  def mtime
+    Time.at(@header.mtime)
+  end
+
+  def mtime=(tm)
+    @header.mtime = tm.to_i
+  end
+
+  def total_size
+    @header.num_bytes + @compressed.size + @footer.num_bytes
+  end
+
+  def compressed_data
+    @compressed
+  end
+
+  def set_compressed_data(compressed, crc32, uncompressed_size)
+    @compressed               = compressed
+    @footer.crc32             = crc32
+    @footer.uncompressed_size = uncompressed_size
+  end
+
+  def read(file_name)
+    File.open(file_name, "r") do |io|
+      @header.read(io)
+
+      # Determine the size of the compressed data.  This is needed because
+      # we don't actually uncompress the data.  Ideally the uncompression
+      # method would read the correct number of bytes from the IO and the
+      # IO would be positioned ready to read the footer.
+
+      pos = io.pos
+      io.seek(-@footer.num_bytes, IO::SEEK_END)
+      compressed_size = io.pos - pos
+      io.seek(pos)
+
+      @compressed = io.read(compressed_size)
+      @footer.read(io)
+    end
+  end
+
+  def write(file_name)
+    File.open(file_name, "w") do |io|
+      @header.write(io)
+      io.write(@compressed)
+      @footer.write(io)
+    end
+  end
+end
+
+if __FILE__ == $0
+  # Write a gzip file.
+  print "Creating a gzip file ... "
+  g = Gzip.new
+  # Uncompressed data is "the cat sat on the mat"
+  g.set_compressed_data("+\311HUHN,Q(\006\342\374<\205\022 77\261\004\000",
+                        3464689835, 22)
+  g.file_name = "poetry"
+  g.mtime = Time.now
+  g.comment = "A stunning piece of prose"
+  g.write("poetry.gz")
+  puts "done."
+  puts
+
+  # Read the created gzip file.
+  print "Reading newly created gzip file ... "
+  g = Gzip.new
+  g.read("poetry.gz")
+  puts "done."
+  puts
+
+  puts "Printing gzip file details in the format of gzip -l -v"
+
+  # compression ratio
+  ratio = 100.0 * (g.uncompressed_size - g.compressed.size) /
+            g.uncompressed_size
+
+  comp_meth = (g.compression_method == Gzip::DEFLATE) ? "defla" : ""
+
+  # Output using the same format as gzip -l -v
+  puts "method  crc     date  time           compressed        " +
+       "uncompressed  ratio uncompressed_name"
+  puts "%5s %08x %6s %5s %19s %19s %5.1f%% %s"  % [comp_meth,
+                                                   g.crc32,
+                                                   g.mtime.strftime('%b %d'),
+                                                   g.mtime.strftime('%H:%M'),
+                                                   g.total_size,
+                                                   g.uncompressed_size,
+                                                   ratio,
+                                                   g.file_name]
+  puts "Comment: #{g.comment}" if g.comment?
+  puts
+
+  puts "Executing gzip -l -v"
+  puts `gzip -l -v poetry.gz`
+end

data/lib/bindata.rb

@@ -0,0 +1,13 @@
+# BinData -- Binary data manipulator.
+# Copyright (c) 2007 Dion Mendel.
+
+require 'bindata/array'
+require 'bindata/choice'
+require 'bindata/int'
+require 'bindata/string'
+require 'bindata/stringz'
+require 'bindata/struct'
+
+module BinData
+  VERSION = "0.5.0"
+end

data/lib/bindata/array.rb

@@ -0,0 +1,160 @@
+require 'bindata/base'
+
+module BinData
+  # An Array is a list of data objects of the same type.
+  #
+  #   require 'bindata'
+  #   require 'stringio'
+  #
+  #   a = BinData::Array.new(:type => :int8, :initial_length => 5)
+  #   io = StringIO.new("\x03\x04\x05\x06\x07")
+  #   a.read(io)
+  #   a.snapshot #=> [3, 4, 5, 6, 7]
+  #
+  # == Parameters
+  #
+  # Parameters may be provided at initialisation to control the behaviour of
+  # an object.  These params are:
+  #
+  # <tt>:type</tt>::           The symbol representing the data type of the
+  #                            array elements.  If the type is to have params
+  #                            passed to it, then it should be provided as
+  #                            <tt>[type_symbol, hash_params]</tt>.
+  # <tt>:initial_length</tt>:: The initial length of the array.
+  class Array < Base
+    include Enumerable
+
+    # Register this class
+    register(self.name, self)
+
+    # These are the parameters used by this class.
+    mandatory_parameters :type, :initial_length
+
+    # Creates a new Array
+    def initialize(params = {}, env = nil)
+      super(params, env)
+
+      type, el_params = param(:type)
+      klass = self.class.lookup(type)
+      raise TypeError, "unknown type '#{type}' for #{self}" if klass.nil?
+
+      @element_list    = nil
+      @element_klass   = klass
+      @element_params  = el_params || {}
+
+      # TODO: how to increase the size of the array?
+    end
+
+    # Clears the element at position +index+.  If +index+ is not given, then
+    # the internal state of the array is reset to that of a newly created
+    # object.
+    def clear(index = nil)
+      if @element_list.nil?
+        # do nothing as the array is already clear
+      elsif index.nil?
+        @element_list = nil
+      else
+        elements[index].clear
+      end
+    end
+
+    # Returns if the element at position +index+ is clear?.  If +index+
+    # is not given, then returns whether all fields are clear.
+    def clear?(index = nil)
+      if @element_list.nil?
+        true
+      elsif index.nil?
+        elements.each { |f| return false if not f.clear? }
+        true
+      else
+        elements[index].clear?
+      end
+    end
+
+    # Reads the values for all fields in this object from +io+.
+    def _do_read(io)
+      elements.each { |f| f.do_read(io) }
+    end
+
+    # To be called after calling #do_read.
+    def done_read
+      elements.each { |f| f.done_read }
+    end
+
+    # Writes the values for all fields in this object to +io+.
+    def _write(io)
+      elements.each { |f| f.write(io) }
+    end
+
+    # Returns the number of bytes it will take to write the element at
+    # +index+.  If +index+, then returns the number of bytes required
+    # to write all fields.
+    def _num_bytes(index)
+      if index.nil?
+        elements.inject(0) { |sum, f| sum + f.num_bytes }
+      else
+        elements[index].num_bytes
+      end
+    end
+
+    # Returns a snapshot of the data in this array.
+    def snapshot
+      elements.collect { |e| e.snapshot }
+    end
+
+    # An array has no fields.
+    def field_names
+      []
+    end
+
+    # Returns the element at +index+.  If the element is a single_value
+    # then the value of the element is returned instead.
+    def [](index)
+      obj = elements[index]
+      obj.single_value? ? obj.value : obj
+    end
+
+    # Sets the element at +index+.  If the element is a single_value
+    # then the value of the element is set instead.
+    def []=(index, value)
+      obj = elements[index]
+      unless obj.single_value?
+        raise NoMethodError, "undefined method `[]=' for #{self}", caller
+      end
+      obj.value = value
+    end
+
+    # Iterate over each element in the array.  If the elements are
+    # single_values then the values of the elements are iterated instead.
+    def each
+      elements.each do |el|
+        yield(el.single_value? ? el.value : el)
+      end
+    end
+
+    # The number of elements in this array.
+    def length
+      elements.length
+    end
+    alias_method :size, :length
+
+    #---------------
+    private
+
+    # Returns the list of all elements in the array.  The elements
+    # will be instantiated on the first call to this method.
+    def elements
+      if @element_list.nil?
+        @element_list = []
+
+        # create the desired number of instances
+        eval_param(:initial_length).times do |i|
+          env = create_env
+          env.index = i
+          @element_list << @element_klass.new(@element_params, env)
+        end
+      end
+      @element_list
+    end
+  end
+end

data/lib/bindata/base.rb

@@ -0,0 +1,260 @@
+require 'bindata/lazy'
+require 'bindata/registry'
+
+module BinData
+  # Error raised when unexpected results occur when reading data from IO.
+  class ValidityError < StandardError ; end
+
+  # This is the abstract base class for all data objects.
+  #
+  # == Parameters
+  #
+  # Parameters may be provided at initialisation to control the behaviour of
+  # an object.  These params are:
+  #
+  # [<tt>:readwrite</tt>]     If false, calls to #read or #write will
+  #                           not perform any I/O.  Default is true.
+  # [<tt>:check_offset</tt>]  Raise an error if the current IO offest doesn't
+  #                           meet this criteria.  A boolean return indicates
+  #                           success or failure.  Any other return is compared
+  #                           to the current offset.  This parameter is
+  #                           only checked before reading.
+  class Base
+    class << self
+      # Returns the mandatory parameters used by this class.  Any given args
+      # are appended to the parameters list.  The parameters for a class will
+      # include the parameters of its ancestors.
+      def mandatory_parameters(*args)
+        unless defined? @mandatory_parameters
+          @mandatory_parameters = []
+          ancestors[1..-1].each do |parent|
+            if parent.respond_to?(:mandatory_parameters)
+              @mandatory_parameters.concat(parent.mandatory_parameters)
+            end
+          end
+        end
+        unless (args.empty?)
+          args.each { |arg| @mandatory_parameters << arg.to_sym }
+          @mandatory_parameters.uniq!
+        end
+        @mandatory_parameters
+      end
+      alias_method :mandatory_parameter, :mandatory_parameters
+
+      # Returns the optional parameters used by this class.  Any given args
+      # are appended to the parameters list.  The parameters for a class will
+      # include the parameters of its ancestors.
+      def optional_parameters(*args)
+        unless defined? @optional_parameters
+          @optional_parameters = []
+          ancestors[1..-1].each do |parent|
+            if parent.respond_to?(:optional_parameters)
+              @optional_parameters.concat(parent.optional_parameters)
+            end
+          end
+        end
+        unless (args.empty?)
+          args.each { |arg| @optional_parameters << arg.to_sym }
+          @optional_parameters.uniq!
+        end
+        @optional_parameters
+      end
+      alias_method :optional_parameter, :optional_parameters
+
+      # Returns both the mandatory and optional parameters used by this class.
+      def parameters
+        (mandatory_parameters + optional_parameters).uniq
+      end
+
+      # Instantiates this class and reads from +io+.  For single value objects
+      # just the value is returned, otherwise the newly created data object is
+      # returned.
+      def read(io)
+        data = self.new
+        data.read(io)
+        data.single_value? ? data.value : data
+      end
+
+      # Registers the mapping of +name+ to +klass+.
+      def register(name, klass)
+        Registry.instance.register(name, klass)
+      end
+      private :register
+
+      # Returns the class matching a previously registered +name+.
+      def lookup(name)
+        Registry.instance.lookup(name)
+      end
+    end
+
+    # Define the parameters we use in this class.
+    optional_parameters :check_offset, :readwrite
+
+    # Creates a new data object.
+    #
+    # +params+ is a hash containing symbol keys.  Some params may
+    # reference callable objects (methods or procs).  +env+ is the
+    # environment that these callable objects are evaluated in.
+    def initialize(params = {}, env = nil)
+      # default :readwrite param to true if unspecified
+      unless params.has_key?(:readwrite)
+        params = params.dup
+        params[:readwrite] = true
+      end
+
+      # ensure mandatory parameters exist
+      self.class.mandatory_parameters.each do |prm|
+        unless params.has_key?(prm)
+          raise ArgumentError, "parameter ':#{prm}' must be specified " +
+                               "in #{self}"
+        end
+      end
+
+      known_params = self.class.parameters
+
+      # partition parameters into known and extra parameters
+      @params = {}
+      extra   = {}
+      params.each do |k,v|
+        k = k.to_sym
+        raise ArgumentError, "parameter :#{k} is nil in #{self}" if v.nil?
+        if known_params.include?(k)
+          @params[k] = v.freeze
+        else
+          extra[k] = v.freeze
+        end
+      end
+
+      # set up the environment
+      @env             = env || LazyEvalEnv.new
+      @env.params      = extra
+      @env.data_object = self
+    end
+
+    # Reads data into this bin object by calling #do_read then #done_read.
+    def read(io)
+      # remember the current position in the IO object
+      io.instance_eval "def mark; #{io.pos}; end"
+
+      do_read(io)
+      done_read
+    end
+
+    # Reads the value for this data from +io+.
+    def do_read(io)
+      clear
+      check_offset(io)
+      _do_read(io) if eval_param(:readwrite) != false
+    end
+
+    # Writes the value for this data to +io+.
+    def write(io)
+      _write(io) if eval_param(:readwrite) != false
+    end
+
+    # Returns the number of bytes it will take to write this data.
+    def num_bytes(what = nil)
+      (eval_param(:readwrite) != false) ? _num_bytes(what) : 0
+    end
+
+    # Returns whether this data object contains a single value.  Single
+    # value data objects respond to <tt>#value</tt> and <tt>#value=</tt>.
+    def single_value?
+      respond_to? :value
+    end
+
+    #---------------
+    private
+
+    # Creates a new LazyEvalEnv for use by a child data object.
+    def create_env
+      LazyEvalEnv.new(@env)
+    end
+
+    # Returns the value of the evaluated parameter.  +key+ references a
+    # parameter from the +params+ hash used when creating the data object.
+    # Returns nil if +key+ does not refer to any parameter.
+    def eval_param(key)
+      @env.lazy_eval(@params[key])
+    end
+
+    # Returns the parameter from the +params+ hash referenced by +key+.
+    # Use this method if you are sure the parameter is not to be evaluated.
+    # You most likely want #eval_param.
+    def param(key)
+      @params[key]
+    end
+
+    # Returns whether +key+ exists in the +params+ hash used when creating
+    # this data object.
+    def has_param?(key)
+      @params.has_key?(key.to_sym)
+    end
+
+    # Raise an error if +param1+ and +param2+ are both given as params.
+    def ensure_mutual_exclusion(param1, param2)
+      if has_param?(param1) and has_param?(param2)
+        raise ArgumentError, "params #{param1} and #{param2} " +
+                             "are mutually exclusive"
+      end
+    end
+
+    # Checks that the current offset of +io+ is as expected.  This should
+    # be called from #do_read before performing the reading.
+    def check_offset(io)
+      if has_param?(:check_offset)
+        @env.offset = io.pos - io.mark
+        expected = eval_param(:check_offset)
+
+        if not expected
+          raise ValidityError, "offset not as expected"
+        elsif @env.offset != expected and expected != true
+          raise ValidityError, "offset is '#{@env.offset}' but " +
+                               "expected '#{expected}'"
+        end
+      end
+    end
+
+=begin
+    # To be implemented by subclasses
+
+    # Resets the internal state to that of a newly created object.
+    def clear
+      raise NotImplementedError
+    end
+
+    # Reads the data for this data object from +io+.
+    def _do_read(io)
+      raise NotImplementedError
+    end
+
+    # To be called after calling #do_read.
+    def done_read
+      raise NotImplementedError
+    end
+
+    # Writes the value for this data to +io+.
+    def _write(io)
+      raise NotImplementedError
+    end
+
+    # Returns the number of bytes it will take to write this data.
+    def _num_bytes
+      raise NotImplementedError
+    end
+
+    # Returns a snapshot of this data object.
+    def snapshot
+      raise NotImplementedError
+    end
+
+    # Returns a list of the names of all fields accessible through this
+    # object.
+    def field_names
+      raise NotImplementedError
+    end
+
+    # To be implemented by subclasses
+=end
+  end
+end