bindata 0.9.2 → 0.9.3

data/ChangeLog

@@ -1,5 +1,14 @@
 = BinData Changelog
 
+== Version 0.9.3 (2008-12-03)
+
+* Arrays can now :read_until => :eof
+* TCPSocket and UDPSocket can now be used as input streams (patch courtesy
+  of Peter Suschlik).
+* Added 128 bit integers.
+* Significant memory usage reduction.
+* Added custom mandatory and default parameters for user defined MultiValues.
+
 == Version 0.9.2 (2008-07-18)
 
 * Added lazy instantiation to allow recursive definitions.

data/TODO

@@ -1,3 +1,20 @@
 * Think how offset_of should work.
 
-* Add (Multi|Single)Value.custom_parameters
+    + perhaps #offset_of and #abs_offset_of methods?
+    + needed for struct and array
+
+* Add ability to determine name of a data object.
+    e.g.  obj.a[4].c
+
+  This will be used when throwing exceptions to aid debugging.
+
+* Using the above names, add tracing capability when reading.
+
+* Clean up Array to make it as close to ruby Array as possible.
+
+* Need more examples.
+
+* Refactor test cases.
+    add more comprehensive integration tests to serve as examples
+
+* Need better documentation.

data/lib/bindata.rb

@@ -18,5 +18,5 @@ require 'bindata/struct'
 # A declarative way to read and write structured binary data.
 # 
 module BinData
-  VERSION = "0.9.2"
+  VERSION = "0.9.3"
 end

data/lib/bindata/array.rb

@@ -27,6 +27,10 @@ module BinData
   #   obj.read(data)
   #   obj.snapshot #=> [3, 4, 5, 6, 7]
   #
+  #   obj = BinData::Array.new(:type => :int8, :read_until => :eof)
+  #   obj.read(data)
+  #   obj.snapshot #=> [3, 4, 5, 6, 7, 8, 9]
+  #
   # == Parameters
   #
   # Parameters may be provided at initialisation to control the behaviour of
@@ -42,7 +46,9 @@ module BinData
   #                            read an array until a sentinel value is found.
   #                            The variables +index+, +element+ and +array+
   #                            are made available to any lambda assigned to
-  #                            this parameter.
+  #                            this parameter.  If the value of this parameter
+  #                            is the symbol :eof, then the array will read
+  #                            as much data from the stream as possible.
   #
   # Each data object in an array has the variable +index+ made available
   # to any lambda evaluated as a parameter of that data object.
@@ -53,16 +59,13 @@ module BinData
     register(self.name, self)
 
     # These are the parameters used by this class.
-    mandatory_parameter :type
-    optional_parameters :initial_length, :read_until
-    mutually_exclusive_parameters :initial_length, :read_until
+    bindata_mandatory_parameter :type
+    bindata_optional_parameters :initial_length, :read_until
+    bindata_mutually_exclusive_parameters :initial_length, :read_until
 
     class << self
-      # Returns a sanitized +params+ that is of the form expected
-      # by #initialize.
-      def sanitize_parameters(sanitizer, params)
-        params = params.dup
-
+      # Ensures that +params+ is of the form expected by #initialize.
+      def sanitize_parameters!(sanitizer, params)
         unless params.has_key?(:initial_length) or params.has_key?(:read_until)
           # ensure one of :initial_length and :read_until exists
           params[:initial_length] = 0
@@ -74,7 +77,9 @@ module BinData
 
         if params.has_key?(:type)
           type, el_params = params[:type]
-          params[:type] = sanitizer.sanitize(type, el_params)
+          klass = sanitizer.lookup_klass(type)
+          sanitized_params = sanitizer.sanitize_params(klass, el_params)
+          params[:type] = [klass, sanitized_params]
         end
 
         super(sanitizer, params)
@@ -82,10 +87,10 @@ module BinData
     end
 
     # Creates a new Array
-    def initialize(params = {}, env = nil)
-      super(params, env)
+    def initialize(params = {}, parent = nil)
+      super(params, parent)
 
-      klass, el_params = param(:type)
+      klass, el_params = no_eval_param(:type)
 
       @element_list    = nil
       @element_klass   = klass
@@ -139,6 +144,11 @@ module BinData
       self.last
     end
 
+    def index(obj)
+      # TODO handle single values
+      elements.index(obj)
+    end
+
     # Pushes the given object(s) on to the end of this array. 
     # This expression returns the array itself, so several appends may 
     # be chained together.
@@ -166,7 +176,7 @@ module BinData
       end
 
       data = elements[*args]
-      if data.respond_to?(:each)
+      if args.length > 1 or ::Range === args[0]
         data.collect { |el| (el && el.single_value?) ? el.value : el }
       else
         (data && data.single_value?) ? data.value : data
@@ -202,13 +212,11 @@ module BinData
     # If the array is empty, the first form returns nil, and the second
     # form returns an empty array.
     def first(n = nil)
-      if n.nil?
-        if elements.empty?
-          # explicitly return nil as arrays grow automatically
-          nil
-        else
-          self[0]
-        end
+      if n.nil? and elements.empty?
+        # explicitly return nil as arrays grow automatically
+        nil
+      elsif n.nil?
+        self[0]
       else
         self[0, n]
       end
@@ -249,15 +257,28 @@ module BinData
     def _do_read(io)
       if has_param?(:initial_length)
         elements.each { |f| f.do_read(io) }
-      else # :read_until
-        @element_list = nil
-        loop do
-          element = append_new_element
-          element.do_read(io)
-          variables = { :index => self.length - 1, :element => self.last,
-                        :array => self }
-          finished = eval_param(:read_until, variables)
-          break if finished
+      elsif has_param?(:read_until)
+        if no_eval_param(:read_until) == :eof
+          @element_list = nil
+          loop do
+            element = append_new_element
+            begin
+              element.do_read(io)
+            rescue
+              @element_list.pop
+              break
+            end
+          end
+        else
+          @element_list = nil
+          loop do
+            element = append_new_element
+            element.do_read(io)
+            variables = { :index => self.length - 1, :element => self.last,
+                          :array => self }
+            finished = eval_param(:read_until, variables)
+            break if finished
+          end
         end
       end
     end
@@ -304,9 +325,7 @@ module BinData
       # ensure @element_list is initialised
       elements()
 
-      env = create_env
-      env.add_variable(:index, @element_list.length)
-      element = @element_klass.new(@element_params, env)
+      element = @element_klass.new(@element_params, self)
       @element_list << element
       element
     end

data/lib/bindata/base.rb

@@ -1,5 +1,6 @@
 require 'bindata/io'
 require 'bindata/lazy'
+require 'bindata/params'
 require 'bindata/registry'
 require 'bindata/sanitize'
 require 'stringio'
@@ -32,98 +33,43 @@ module BinData
   #                           <tt>:check_offset</tt>, except that it will
   #                           adjust the IO offset instead of raising an error.
   class Base
+
     class << self
-      # Returns the mandatory parameters used by this class.  Any given args
-      # are appended to the parameters list.  The parameters for a class will
-      # include the parameters of its ancestors.
-      def mandatory_parameters(*args)
-        unless defined? @mandatory_parameters
-          @mandatory_parameters = []
-          ancestors[1..-1].each do |parent|
-            if parent.respond_to?(:mandatory_parameters)
-              pmp = parent.mandatory_parameters
-              @mandatory_parameters.concat(pmp)
-            end
-          end
-        end
-        if not args.empty?
-          args.each { |arg| @mandatory_parameters << arg.to_sym }
-          @mandatory_parameters.uniq!
-        end
-        @mandatory_parameters
-      end
-      alias_method :mandatory_parameter, :mandatory_parameters
-
-      # Returns the optional parameters used by this class.  Any given args
-      # are appended to the parameters list.  The parameters for a class will
-      # include the parameters of its ancestors.
-      def optional_parameters(*args)
-        unless defined? @optional_parameters
-          @optional_parameters = []
-          ancestors[1..-1].each do |parent|
-            if parent.respond_to?(:optional_parameters)
-              pop = parent.optional_parameters
-              @optional_parameters.concat(pop)
-            end
-          end
-        end
-        if not args.empty?
-          args.each { |arg| @optional_parameters << arg.to_sym }
-          @optional_parameters.uniq!
-        end
-        @optional_parameters
+      extend Parameters
+
+      # Define methods for:
+      #   bindata_mandatory_parameters
+      #   bindata_optional_parameters
+      #   bindata_default_parameters
+      #   bindata_mutually_exclusive_parameters
+
+      define_x_parameters(:bindata_mandatory, []) do |array, args|
+        args.each { |arg| array << arg.to_sym }
+        array.uniq!
       end
-      alias_method :optional_parameter, :optional_parameters
-
-      # Returns the default parameters used by this class.  Any given args
-      # are appended to the parameters list.  The parameters for a class will
-      # include the parameters of its ancestors.
-      def default_parameters(params = {})
-        unless defined? @default_parameters
-          @default_parameters = {}
-          ancestors[1..-1].each do |parent|
-            if parent.respond_to?(:default_parameters)
-              pdp = parent.default_parameters
-              @default_parameters = @default_parameters.merge(pdp)
-            end
-          end
-        end
-        if not params.empty?
-          @default_parameters = @default_parameters.merge(params)
-        end
-        @default_parameters
+
+      define_x_parameters(:bindata_optional, []) do |array, args|
+        args.each { |arg| array << arg.to_sym }
+        array.uniq!
       end
-      alias_method :default_parameter, :default_parameters
-
-      # Returns the pairs of mutually exclusive parameters used by this class.
-      # Any given args are appended to the parameters list.  The parameters for
-      # a class will include the parameters of its ancestors.
-      def mutually_exclusive_parameters(*args)
-        unless defined? @mutually_exclusive_parameters
-          @mutually_exclusive_parameters = []
-          ancestors[1..-1].each do |parent|
-            if parent.respond_to?(:mutually_exclusive_parameters)
-              pmep = parent.mutually_exclusive_parameters
-              @mutually_exclusive_parameters.concat(pmep)
-            end
-          end
-        end
-        if not args.empty?
-          @mutually_exclusive_parameters << [args[0].to_sym, args[1].to_sym]
-        end
-        @mutually_exclusive_parameters
+
+      define_x_parameters(:bindata_default, {}) do |hash, args|
+        params = args.length > 0 ? args[0] : {}
+        hash.merge!(params)
       end
 
-      # Returns a list of parameters that are accepted by this object
-      def accepted_parameters
-        (mandatory_parameters + optional_parameters + default_parameters.keys).uniq
+      define_x_parameters(:bindata_mutually_exclusive, []) do |array, args|
+        array << [args[0].to_sym, args[1].to_sym]
       end
 
-      # Returns a sanitized +params+ that is of the form expected
-      # by #initialize.
-      def sanitize_parameters(sanitizer, params, *args)
-        params = params.dup
+      # Returns a list of internal parameters that are accepted by this object
+      def internal_parameters
+        (bindata_mandatory_parameters + bindata_optional_parameters +
+         bindata_default_parameters.keys).uniq
+      end
 
+      # Ensures that +params+ is of the form expected by #initialize.
+      def sanitize_parameters!(sanitizer, params)
         # replace :readwrite with :onlyif
         if params.has_key?(:readwrite)
           warn ":readwrite is deprecated. Replacing with :onlyif"
@@ -131,12 +77,12 @@ module BinData
         end
 
         # add default parameters
-        default_parameters.each do |k,v|
+        bindata_default_parameters.each do |k,v|
           params[k] = v unless params.has_key?(k)
         end
 
         # ensure mandatory parameters exist
-        mandatory_parameters.each do |prm|
+        bindata_mandatory_parameters.each do |prm|
           if not params.has_key?(prm)
             raise ArgumentError, "parameter ':#{prm}' must be specified " +
                                  "in #{self}"
@@ -144,14 +90,17 @@ module BinData
         end
 
         # ensure mutual exclusion
-        mutually_exclusive_parameters.each do |param1, param2|
+        bindata_mutually_exclusive_parameters.each do |param1, param2|
           if params.has_key?(param1) and params.has_key?(param2)
             raise ArgumentError, "params #{param1} and #{param2} " +
                                  "are mutually exclusive"
           end
         end
+      end
 
-        params
+      # Can this data object self reference itself?
+      def recursive?
+        false
       end
 
       # Instantiates this class and reads from +io+.  For single value objects
@@ -171,26 +120,27 @@ module BinData
     end
 
     # Define the parameters we use in this class.
-    optional_parameters :check_offset, :adjust_offset
-    default_parameters :onlyif => true
-    mutually_exclusive_parameters :check_offset, :adjust_offset
+    bindata_optional_parameters :check_offset, :adjust_offset
+    bindata_default_parameters :onlyif => true
+    bindata_mutually_exclusive_parameters :check_offset, :adjust_offset
 
     # Creates a new data object.
     #
     # +params+ is a hash containing symbol keys.  Some params may
-    # reference callable objects (methods or procs).  +env+ is the
-    # environment that these callable objects are evaluated in.
-    def initialize(params = {}, env = nil)
-      unless SanitizedParameters === params
-        params = Sanitizer.sanitize(self, params)
-      end
+    # reference callable objects (methods or procs).  +parent+ is the
+    # parent data object (e.g. struct, array, choice) this object resides
+    # under.
+    def initialize(params = {}, parent = nil)
+      @params = Sanitizer.sanitize(self.class, params)
+      @parent = parent
+    end
 
-      @params = params.accepted_parameters
+    # The parent data object.
+    attr_accessor :parent
 
-      # set up the environment
-      @env             = env || LazyEvalEnv.new
-      @env.params      = params.extra_parameters
-      @env.data_object = self
+    # Returns all the custom parameters supplied to this data object.
+    def parameters
+      @params.extra_parameters
     end
 
     # Reads data into this data object by calling #do_read then #done_read.
@@ -272,33 +222,33 @@ module BinData
       snapshot.inspect
     end
 
+    # Returns the object this object represents.
+    def obj
+      self
+    end
+
     #---------------
     private
 
-    # Creates a new LazyEvalEnv for use by a child data object.
-    def create_env
-      LazyEvalEnv.new(@env)
-    end
-
     # Returns the value of the evaluated parameter.  +key+ references a
     # parameter from the +params+ hash used when creating the data object.
     # +values+ contains data that may be accessed when evaluating +key+.
     # Returns nil if +key+ does not refer to any parameter.
     def eval_param(key, values = nil)
-      @env.lazy_eval(@params[key], values)
+      LazyEvaluator.eval(no_eval_param(key), self, values)
     end
 
     # Returns the parameter from the +params+ hash referenced by +key+.
     # Use this method if you are sure the parameter is not to be evaluated.
     # You most likely want #eval_param.
-    def param(key)
-      @params[key]
+    def no_eval_param(key)
+      @params.internal_parameters[key]
     end
 
     # Returns whether +key+ exists in the +params+ hash used when creating
     # this data object.
     def has_param?(key)
-      @params.has_key?(key.to_sym)
+      @params.internal_parameters.has_key?(key)
     end
 
     # Checks that the current offset of +io+ is as expected.  This should