bindata 2.4.15 → 2.5.1

data/lib/bindata/registry.rb

@@ -1,6 +1,6 @@
 module BinData
-
-  class UnRegisteredTypeError < StandardError ; end
+  # Raised when #lookup fails.
+  class UnRegisteredTypeError < StandardError; end
 
   # This registry contains a register of name -> class mappings.
   #
@@ -18,7 +18,6 @@ module BinData
   #
   # Names are stored in under_score_style, not camelCase.
   class Registry
-
     def initialize
       @registry = {}
     end
@@ -37,55 +36,56 @@ module BinData
     end
 
     def lookup(name, hints = {})
-      the_class = @registry[normalize_name(name, hints)]
-      if the_class
-        the_class
-      elsif @registry[normalize_name(name, hints.merge(endian: :big))]
-        raise(UnRegisteredTypeError, "#{name}, do you need to specify endian?")
-      else
-        raise(UnRegisteredTypeError, name)
+      search_names(name, hints).each do |search|
+        register_dynamic_class(search)
+        if @registry.has_key?(search)
+          return @registry[search]
+        end
       end
+
+      # give the user a hint if the endian keyword is missing
+      search_names(name, hints.merge(endian: :big)).each do |search|
+        register_dynamic_class(search)
+        if @registry.has_key?(search)
+          raise(UnRegisteredTypeError, "#{name}, do you need to specify endian?")
+        end
+      end
+
+      raise(UnRegisteredTypeError, name)
     end
 
     # Convert CamelCase +name+ to underscore style.
     def underscore_name(name)
-      name.
-        to_s.
-        sub(/.*::/, "").
-        gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
-        gsub(/([a-z\d])([A-Z])/, '\1_\2').
-        tr("-", "_").
-        downcase
+      name
+        .to_s
+        .sub(/.*::/, "")
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .tr('-', '_')
+        .downcase
     end
 
     #---------------
     private
 
-    def normalize_name(name, hints)
-      name = underscore_name(name)
-
-      if !registered?(name)
-        search_prefix = [""].concat(Array(hints[:search_prefix]))
-        search_prefix.each do |prefix|
-          nwp = name_with_prefix(name, prefix)
-          if registered?(nwp)
-            name = nwp
-            break
-          end
-
-          nwe = name_with_endian(nwp, hints[:endian])
-          if registered?(nwe)
-            name = nwe
-            break
-          end
-        end
+    def search_names(name, hints)
+      base = underscore_name(name)
+      searches = []
+
+      search_prefix = [""] + Array(hints[:search_prefix])
+      search_prefix.each do |prefix|
+        nwp = name_with_prefix(base, prefix)
+        nwe = name_with_endian(nwp, hints[:endian])
+
+        searches << nwp
+        searches << nwe if nwe
       end
 
-      name
+      searches
     end
 
     def name_with_prefix(name, prefix)
-      prefix = prefix.to_s.chomp("_")
+      prefix = prefix.to_s.chomp('_')
       if prefix == ""
         name
       else
@@ -94,26 +94,21 @@ module BinData
     end
 
     def name_with_endian(name, endian)
-      return name if endian.nil?
+      return nil if endian.nil?
 
-      suffix = (endian == :little) ? "le" : "be"
-      if /^u?int\d+$/ =~ name
+      suffix = (endian == :little) ? 'le' : 'be'
+      if /^u?int\d+$/.match?(name)
         name + suffix
       else
-        name + "_" + suffix
+        name + '_' + suffix
       end
     end
 
-    def registered?(name)
-      register_dynamic_class(name) unless @registry.key?(name)
-
-      @registry.key?(name)
-    end
-
     def register_dynamic_class(name)
-      if /^u?int\d+(le|be)$/ =~ name || /^s?bit\d+(le)?$/ =~ name
+      if /^u?int\d+(le|be)$/.match?(name) || /^s?bit\d+(le)?$/.match?(name)
         class_name = name.gsub(/(?:^|_)(.)/) { $1.upcase }
         begin
+          # call const_get for side effect of creating class
           BinData.const_get(class_name)
         rescue NameError
         end
@@ -122,9 +117,9 @@ module BinData
 
     def warn_if_name_is_already_registered(name, class_to_register)
       prev_class = @registry[name]
-      if $VERBOSE && prev_class && prev_class != class_to_register
-        warn "warning: replacing registered class #{prev_class} " \
-             "with #{class_to_register}"
+      if prev_class && prev_class != class_to_register
+        Kernel.warn "warning: replacing registered class #{prev_class} " \
+                    "with #{class_to_register}"
       end
     end
   end

data/lib/bindata/rest.rb

@@ -1,4 +1,4 @@
-require "bindata/base_primitive"
+require 'bindata/base_primitive'
 
 module BinData
   # Rest will consume the input stream from the current position to the end of

data/lib/bindata/sanitize.rb

@@ -49,14 +49,10 @@ module BinData
       @prototype = SanitizedPrototype.new(field_type, field_params, hints)
     end
 
-    attr_reader :prototype
+    attr_reader :prototype, :name
 
     def name_as_sym
-      @name.nil? ? nil : @name.to_sym
-    end
-
-    def name
-      @name
+      @name&.to_sym
     end
 
     def has_parameter?(param)
@@ -74,11 +70,7 @@ module BinData
 
     def initialize(hints, base_fields = nil)
       @hints = hints
-      if base_fields
-        @fields = base_fields.raw_fields
-      else
-        @fields = []
-      end
+      @fields =  base_fields ? base_fields.raw_fields : []
     end
 
     def add_field(type, name, params)
@@ -179,7 +171,6 @@ module BinData
   # is to recursively sanitize the parameters of an entire BinData object chain
   # at a single time.
   class SanitizedParameters < Hash
-
     # Memoized constants
     BIG_ENDIAN    = SanitizedBigEndian.new
     LITTLE_ENDIAN = SanitizedLittleEndian.new
@@ -210,7 +201,7 @@ module BinData
       sanitize!
     end
 
-    alias_method :has_parameter?, :key?
+    alias has_parameter? key?
 
     def has_at_least_one_of?(*keys)
       keys.each do |key|
@@ -257,7 +248,9 @@ module BinData
     end
 
     def sanitize_object_prototype(key)
-      sanitize(key) { |obj_type, obj_params| create_sanitized_object_prototype(obj_type, obj_params) }
+      sanitize(key) do |obj_type, obj_params|
+        create_sanitized_object_prototype(obj_type, obj_params)
+      end
     end
 
     def sanitize_fields(key, &block)
@@ -306,7 +299,7 @@ module BinData
     end
 
     def needs_sanitizing?(key)
-      has_key?(key) && ! self[key].is_a?(SanitizedParameter)
+      has_parameter?(key) && !self[key].is_a?(SanitizedParameter)
     end
 
     def ensure_no_nil_values
@@ -320,7 +313,7 @@ module BinData
 
     def merge_default_parameters!
       @the_class.default_parameters.each do |key, value|
-        self[key] = value unless has_key?(key)
+        self[key] = value unless has_parameter?(key)
       end
     end
 

data/lib/bindata/section.rb

@@ -0,0 +1,97 @@
+require 'bindata/base'
+require 'bindata/dsl'
+
+module BinData
+  # A Section is a layer on top of a stream that transforms the underlying
+  # data.  This allows BinData to process a stream that has multiple
+  # encodings.  e.g.  Some data data is compressed or encrypted.
+  #
+  #   require 'bindata'
+  #
+  #   class XorTransform < BinData::IO::Transform
+  #      def initialize(xor)
+  #        super()
+  #        @xor = xor
+  #      end
+  #
+  #      def read(n)
+  #        chain_read(n).bytes.map { |byte| (byte ^ @xor).chr }.join
+  #      end
+  #
+  #      def write(data)
+  #        chain_write(data.bytes.map { |byte| (byte ^ @xor).chr }.join)
+  #      end
+  #   end
+  #
+  #   obj = BinData::Section.new(transform: -> { XorTransform.new(0xff) },
+  #                              type: [:string, read_length: 5])
+  #
+  #   obj.read("\x97\x9A\x93\x93\x90") #=> "hello"
+  #
+  #
+  # == Parameters
+  #
+  # Parameters may be provided at initialisation to control the behaviour of
+  # an object.  These params are:
+  #
+  # <tt>:transform</tt>:: A callable that returns a new BinData::IO::Transform.
+  # <tt>:type</tt>::      The single type inside the buffer.  Use a struct if
+  #                       multiple fields are required.
+  class Section < BinData::Base
+    extend DSLMixin
+
+    dsl_parser    :section
+    arg_processor :section
+
+    mandatory_parameters :transform, :type
+
+    def initialize_instance
+      @type = get_parameter(:type).instantiate(nil, self)
+    end
+
+    def clear?
+      @type.clear?
+    end
+
+    def assign(val)
+      @type.assign(val)
+    end
+
+    def snapshot
+      @type.snapshot
+    end
+
+    def respond_to_missing?(symbol, include_all = false) # :nodoc:
+      @type.respond_to?(symbol, include_all) || super
+    end
+
+    def method_missing(symbol, *args, &block) # :nodoc:
+      @type.__send__(symbol, *args, &block)
+    end
+
+    def do_read(io) # :nodoc:
+      io.transform(eval_parameter(:transform)) do |transformed_io, _raw_io|
+        @type.do_read(transformed_io)
+      end
+    end
+
+    def do_write(io) # :nodoc:
+      io.transform(eval_parameter(:transform)) do |transformed_io, _raw_io|
+        @type.do_write(transformed_io)
+      end
+    end
+
+    def do_num_bytes # :nodoc:
+      to_binary_s.size
+    end
+  end
+
+  class SectionArgProcessor < BaseArgProcessor
+    include MultiFieldArgSeparator
+
+    def sanitize_parameters!(obj_class, params)
+      params.merge!(obj_class.dsl_params)
+      params.sanitize_object_prototype(:type)
+    end
+  end
+end

data/lib/bindata/skip.rb

@@ -1,4 +1,5 @@
-require "bindata/base_primitive"
+require 'bindata/base_primitive'
+require 'bindata/dsl'
 
 module BinData
   # Skip will skip over bytes from the input stream.  If the stream is not
@@ -18,12 +19,14 @@ module BinData
   #
   #
   #   class B < BinData::Record
-  #     skip until_valid: [:string, {read_length: 2, assert: "ef"} ]
-  #     string :b, read_length: 5
+  #     skip do
+  #       string read_length: 2, assert: 'ef'
+  #     end
+  #     string :s, read_length: 5
   #   end
   #
   #   obj = B.read("abcdefghij")
-  #   obj.b #=> "efghi"
+  #   obj.s #=> "efghi"
   #
   #
   # == Parameters
@@ -33,15 +36,18 @@ module BinData
   #
   # <tt>:length</tt>::        The number of bytes to skip.
   # <tt>:to_abs_offset</tt>:: Skips to the given absolute offset.
-  # <tt>:until_valid</tt>::   Skips untils a given byte pattern is matched.
+  # <tt>:until_valid</tt>::   Skips until a given byte pattern is matched.
   #                           This parameter contains a type that will raise
   #                           a BinData::ValidityError unless an acceptable byte
   #                           sequence is found.  The type is represented by a
-  #                           Symbol, or if the type is to have params #
-  #                           passed to it, then it should be provided as #
+  #                           Symbol, or if the type is to have params
+  #                           passed to it, then it should be provided as
   #                           <tt>[type_symbol, hash_params]</tt>.
   #
   class Skip < BinData::BasePrimitive
+    extend DSLMixin
+
+    dsl_parser    :skip
     arg_processor :skip
 
     optional_parameters :length, :to_abs_offset, :until_valid
@@ -57,10 +63,11 @@ module BinData
     #---------------
     private
 
-    def value_to_binary_string(val)
+    def value_to_binary_string(_)
       len = skip_length
-      if len < 0
-        raise ValidityError, "#{debug_name} attempted to seek backwards by #{len.abs} bytes"
+      if len.negative?
+        raise ArgumentError,
+              "#{debug_name} attempted to seek backwards by #{len.abs} bytes"
       end
 
       "\000" * skip_length
@@ -68,66 +75,148 @@ module BinData
 
     def read_and_return_value(io)
       len = skip_length
-      if len < 0
-        raise ValidityError, "#{debug_name} attempted to seek backwards by #{len.abs} bytes"
+      if len.negative?
+        raise ArgumentError,
+              "#{debug_name} attempted to seek backwards by #{len.abs} bytes"
       end
 
-      io.seekbytes(len)
+      io.skipbytes(len)
       ""
     end
 
     def sensible_default
       ""
     end
-  end
 
-  class SkipArgProcessor < BaseArgProcessor
-    def sanitize_parameters!(obj_class, params)
-      unless params.has_at_least_one_of?(:length, :to_abs_offset, :until_valid)
-        raise ArgumentError,
-          "#{obj_class} requires either :length, :to_abs_offset or :until_valid"
+    # Logic for the :length parameter
+    module SkipLengthPlugin
+      def skip_length
+        eval_parameter(:length)
       end
-      params.must_be_integer(:to_abs_offset, :length)
-      params.sanitize_object_prototype(:until_valid)
     end
-  end
 
-  # Logic for the :length parameter
-  module SkipLengthPlugin
-    def skip_length
-      eval_parameter(:length)
+    # Logic for the :to_abs_offset parameter
+    module SkipToAbsOffsetPlugin
+      def skip_length
+        eval_parameter(:to_abs_offset) - abs_offset
+      end
     end
-  end
 
-  # Logic for the :to_abs_offset parameter
-  module SkipToAbsOffsetPlugin
-    def skip_length
-      eval_parameter(:to_abs_offset) - abs_offset
-    end
-  end
+    # Logic for the :until_valid parameter
+    module SkipUntilValidPlugin
+      def skip_length
+        @skip_length ||= 0
+      end
 
-  # Logic for the :until_valid parameter
-  module SkipUntilValidPlugin
-    def skip_length
-      # no skipping when writing
-      0
-    end
+      def read_and_return_value(io)
+        prototype = get_parameter(:until_valid)
+        validator = prototype.instantiate(nil, self)
+        fs = fast_search_for_obj(validator)
 
-    def read_and_return_value(io)
-      prototype = get_parameter(:until_valid)
-      validator = prototype.instantiate(nil, self)
-
-      valid = false
-      until valid
-        begin
-          io.with_readahead do
-            validator.read(io)
-            valid = true
+        io.transform(ReadaheadIO.new) do |transformed_io, raw_io|
+          pos = 0
+          loop do
+            seek_to_pos(pos, raw_io)
+            validator.clear
+            validator.do_read(transformed_io)
+            break
+          rescue ValidityError
+            pos += 1
+
+            if fs
+              seek_to_pos(pos, raw_io)
+              pos += next_search_index(raw_io, fs)
+            end
           end
-        rescue ValidityError
-          io.readbytes(1)
+
+          seek_to_pos(pos, raw_io)
+          @skip_length = pos
         end
       end
+
+      def seek_to_pos(pos, io)
+        io.rollback
+        io.skip(pos)
+      end
+
+      # A fast search has a pattern string at a specific offset.
+      FastSearch = ::Struct.new('FastSearch', :pattern, :offset)
+
+      def fast_search_for(obj)
+        if obj.respond_to?(:asserted_binary_s)
+          FastSearch.new(obj.asserted_binary_s, obj.rel_offset)
+        else
+          nil
+        end
+      end
+
+      # If a search object has an +asserted_value+ field then we
+      # perform a faster search for a valid object.
+      def fast_search_for_obj(obj)
+        if BinData::Struct === obj
+          obj.each_pair(true) do |_, field|
+            fs = fast_search_for(field)
+            return fs if fs
+          end
+        elsif BinData::BasePrimitive === obj
+          return fast_search_for(obj)
+        end
+
+        nil
+      end
+
+      SEARCH_SIZE = 100_000
+
+      def next_search_index(io, fs)
+        buffer = binary_string("")
+
+        # start searching at fast_search offset
+        pos = fs.offset
+        io.skip(fs.offset)
+
+        loop do
+          data = io.read(SEARCH_SIZE)
+          raise EOFError, "no match" if data.nil?
+
+          buffer << data
+          index = buffer.index(fs.pattern)
+          if index
+            return pos + index - fs.offset
+          end
+
+          # advance buffer
+          searched = buffer.slice!(0..-fs.pattern.size)
+          pos += searched.size
+        end
+      end
+
+      class ReadaheadIO < BinData::IO::Transform
+        def before_transform
+          if !seekable?
+            raise IOError, "readahead is not supported on unseekable streams"
+          end
+
+          @mark = offset
+        end
+
+        def rollback
+          seek_abs(@mark)
+        end
+      end
+    end
+  end
+
+  class SkipArgProcessor < BaseArgProcessor
+    def sanitize_parameters!(obj_class, params)
+      params.merge!(obj_class.dsl_params)
+
+      unless params.has_at_least_one_of?(:length, :to_abs_offset, :until_valid)
+        raise ArgumentError,
+              "#{obj_class} requires :length, :to_abs_offset or :until_valid"
+      end
+
+      params.must_be_integer(:to_abs_offset, :length)
+      params.sanitize_object_prototype(:until_valid)
     end
   end
 end

data/lib/bindata/string.rb

@@ -1,4 +1,4 @@
-require "bindata/base_primitive"
+require 'bindata/base_primitive'
 
 module BinData
   # A String is a sequence of bytes.  This is the same as strings in Ruby 1.8.
@@ -121,6 +121,14 @@ module BinData
     def sensible_default
       ""
     end
+
+    # Warns when reading if :value && no :read_length
+    module WarnNoReadLengthPlugin
+      def read_and_return_value(io)
+        Kernel.warn "#{debug_name} does not have a :read_length parameter - returning empty string"
+        ""
+      end
+    end
   end
 
   class StringArgProcessor < BaseArgProcessor
@@ -142,12 +150,4 @@ module BinData
       pad_byte
     end
   end
-
-  # Warns when reading if :value && no :read_length
-  module WarnNoReadLengthPlugin
-    def read_and_return_value(io)
-      warn "#{debug_name} does not have a :read_length parameter - returning empty string"
-      ""
-    end
-  end
 end

data/lib/bindata/stringz.rb

@@ -1,4 +1,4 @@
-require "bindata/base_primitive"
+require 'bindata/base_primitive'
 
 module BinData
   # A BinData::Stringz object is a container for a zero ("\0") terminated
@@ -25,7 +25,6 @@ module BinData
   # <tt>:max_length</tt>:: The maximum length of the string including the zero
   #                        byte.
   class Stringz < BinData::BasePrimitive
-
     optional_parameters :max_length
 
     def assign(val)
@@ -47,14 +46,14 @@ module BinData
 
     def read_and_return_value(io)
       max_length = eval_parameter(:max_length)
-      str = ""
+      str = binary_string("")
       i = 0
       ch = nil
 
       # read until zero byte or we have read in the max number of bytes
       while ch != "\0" && i != max_length
         ch = io.readbytes(1)
-        str += ch
+        str << ch
         i += 1
       end
 
@@ -66,9 +65,15 @@ module BinData
     end
 
     def trim_and_zero_terminate(str)
+      max_length = eval_parameter(:max_length)
+      if max_length && max_length < 1
+        msg = "max_length must be >= 1 in #{debug_name} (got #{max_length})"
+        raise ArgumentError, msg
+      end
+
       result = binary_string(str)
       truncate_after_first_zero_byte!(result)
-      trim_to!(result, eval_parameter(:max_length))
+      trim_to!(result, max_length)
       append_zero_byte_if_needed!(result)
       result
     end
@@ -79,16 +84,13 @@ module BinData
 
     def trim_to!(str, max_length = nil)
       if max_length
-        max_length = 1 if max_length < 1
         str.slice!(max_length..-1)
-        if str.length == max_length && str[-1, 1] != "\0"
-          str[-1, 1] = "\0"
-        end
+        str[-1, 1] = "\0" if str.length == max_length
       end
     end
 
     def append_zero_byte_if_needed!(str)
-      if str.length == 0 || str[-1, 1] != "\0"
+      if str.empty? || str[-1, 1] != "\0"
         str << "\0"
       end
     end