bindata 2.4.15 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80617b407b7a7c2213a57dd816a4c34b144dc7bf84cfbedb021b4121f733d375
-  data.tar.gz: 4a71111bd4ee444bd71bde4d2e1cf4242501618d4884b93f72eabdc7f0cd6ed6
+  metadata.gz: 6fb38e8b0ef072a6acb74e9a623307791ba9619423e445635bf949c6ede9a90a
+  data.tar.gz: 026e68ad13001b0ae2922bbe909882d5bec2c44c3380ab894cf73e3fbce8de61
 SHA512:
-  metadata.gz: 4de56ea71fa7e19e79d668eca56fd767a6dc781ce2184ef6d0373835ca73650d423cecd65c9066a1cd3e2055d66f87223688b78737da5630401cda06ec6ab5ee
-  data.tar.gz: 33865f2dca175323ed73af86b52ec4c507b3b1ef1af6b4eeef103fbfe422607cb40476e558dda85e13ffad603ab6f8150a5798bdd1051a164dc54604ec7c5cf1
+  metadata.gz: 3a46d236c6206417f271782956e3aaa376f793ef9cc58f88de92577b10537a5cd2ce8cc8f2399e1a58d0bb1f1ab749d57f0ea6aa8532c78fd80dc0e998b3abb4
+  data.tar.gz: a1d2988f1bcfda8c3f1177338d34b5af0e7f62dd4eea55905ef3c49e5cb989a7b07e30d1b7d42606a4ca005bcfa3ef034829520b52df86ca9bafe25cb7f2bf1c

data/ChangeLog.rdoc

@@ -1,5 +1,16 @@
 = BinData Changelog
 
+== Version 2.5.0 (2024-02-16)
+
+* Removed experimental :check_offset and :adjust_offset parameters.
+* Ruby 2.5 is now required.
+* Allow for nested tracing.
+* Skip :until_valid is now fast for :asserted_value.
+* Added Section - a way to transform the data stream.
+* Added transforms for brotli, lz4, xor, zlib, zstd.
+* Updated to current minitest
+* Fixed typos.  Thanks to Patrick Linnane.
+
 == Version 2.4.15 (2023-02-07)
 
 * Added ruby 2.4.0 requirement to gemspec.  Thanks to theldoria.

data/README.md

@@ -1,8 +1,9 @@
 # What is BinData?
 
-[![Version     ](https://img.shields.io/gem/v/bindata.svg)                       ](https://rubygems.org/gems/bindata)
-[![Github CI   ](https://github.com/dmendel/bindata/actions/workflows/ci.yml/badge.svg)     ](https://github.com/dmendel/bindata/actions/workflows/ci.yml)
-[![Coverage    ](https://img.shields.io/coveralls/dmendel/bindata.svg)         ](https://coveralls.io/r/dmendel/bindata)
+[![Github CI](https://github.com/dmendel/bindata/actions/workflows/ci.yml/badge.svg)](https://github.com/dmendel/bindata/actions/workflows/ci.yml)
+[![Version](https://img.shields.io/gem/v/bindata.svg)](https://rubygems.org/gems/bindata)
+[![Downloads](https://img.shields.io/gem/dt/bindata.svg)](https://rubygems.org/gems/bindata)
+[![Coverage](https://img.shields.io/coveralls/dmendel/bindata.svg)](https://coveralls.io/r/dmendel/bindata)
 
 Do you ever find yourself writing code like this?
 
@@ -14,7 +15,7 @@ width, height = io.read(8).unpack("VV")
 puts "Rectangle #{name} is #{width} x #{height}"
 ```
 
-It’s ugly, violates DRY and feels like you’re writing Perl, not Ruby.
+It’s ugly, violates DRY and doesn't feel like Ruby.
 
 There is a better way. Here’s how you’d write the above using BinData.
 
@@ -47,13 +48,9 @@ for dependent and variable length fields is built in.
 
     $ gem install bindata
 
-or if running ruby 1.8
-
-    $ gem install bindata -v '~> 1.8.0'
-
 # Documentation
 
-[Read the wiki](http://github.com/dmendel/bindata/wiki).
+[BinData manual](http://github.com/dmendel/bindata/wiki).
 
 # Contact
 

data/bindata.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |s|
     end
   end
   s.license = 'BSD-2-Clause'
-  s.required_ruby_version = ">= 2.4.0"
+  s.required_ruby_version = ">= 2.5.0"
 
   s.add_development_dependency('rake')
-  s.add_development_dependency('minitest', "> 5.0.0", "< 5.12.0")
-  s.add_development_dependency('coveralls')
+  s.add_development_dependency('minitest', "> 5.0.0")
+  s.add_development_dependency('simplecov')
   s.description = <<-END.gsub(/^ +/, "")
     BinData is a declarative way to read and write binary file formats.
 

data/examples/list.rb

@@ -35,7 +35,7 @@ require 'bindata'
 #       end
 #     end
 #
-# Notice how we get stuck on attemping to write a declaration for
+# Notice how we get stuck on attempting to write a declaration for
 # the contents of the list.  We can't determine if the list item is
 # an atom or list because we haven't read it yet.  It appears that
 # we can't proceed.

data/lib/bindata/alignment.rb

@@ -19,11 +19,11 @@ module BinData
     def do_num_bytes; 0; end
 
     def do_read(io)
-      io.reset_read_bits
+      io.readbytes(0)
     end
 
     def do_write(io)
-      io.flushbits
+      io.writebytes("")
     end
   end
 
@@ -45,18 +45,26 @@ module BinData
       def initialize(io)
         @io = io
       end
+
+      def binary_string(str)
+        str.to_s.dup.force_encoding(Encoding::BINARY)
+      end
+
       def readbytes(n)
-        n.times.inject("") do |bytes, _|
-          bytes += @io.readbits(8, :big).chr
+        n.times.inject(binary_string("")) do |bytes, _|
+          bytes + @io.readbits(8, :big).chr
         end
       end
+      def writebytes(str)
+        str.each_byte { |v| @io.writebits(v, 8, :big) }
+      end
     end
 
     def bit_aligned?
       true
     end
 
-    def read_and_return_value(io)
+    def do_read(io)
       super(BitAlignedIO.new(io))
     end
 
@@ -65,7 +73,7 @@ module BinData
     end
 
     def do_write(io)
-      value_to_binary_string(_value).each_byte { |v| io.writebits(v, 8, :big) }
+      super(BitAlignedIO.new(io))
     end
   end
 
@@ -74,6 +82,6 @@ module BinData
   end
 
   def Primitive.bit_aligned
-    fail "'bit_aligned' is not needed for BinData::Primitives"
+    fail "'bit_aligned' is not supported for BinData::Primitives"
   end
 end

data/lib/bindata/array.rb

@@ -72,18 +72,18 @@ module BinData
     end
 
     def initialize_instance
-      @element_list = nil
+      @elements = nil
     end
 
     def clear?
-      @element_list.nil? || elements.all?(&:clear?)
+      @elements.nil? || elements.all?(&:clear?)
     end
 
     def assign(array)
       return if self.equal?(array)  # prevent self assignment
       raise ArgumentError, "can't set a nil value for #{debug_name}" if array.nil?
 
-      @element_list = []
+      @elements = []
       concat(array)
     end
 
@@ -220,23 +220,23 @@ module BinData
       elements.each { |el| yield el }
     end
 
-    def debug_name_of(child) #:nodoc:
+    def debug_name_of(child) # :nodoc:
       index = find_index_of(child)
       "#{debug_name}[#{index}]"
     end
 
-    def offset_of(child) #:nodoc:
+    def offset_of(child) # :nodoc:
       index = find_index_of(child)
       sum = sum_num_bytes_below_index(index)
 
       child.bit_aligned? ? sum.floor : sum.ceil
     end
 
-    def do_write(io) #:nodoc:
+    def do_write(io) # :nodoc:
       elements.each { |el| el.do_write(io) }
     end
 
-    def do_num_bytes #:nodoc:
+    def do_num_bytes # :nodoc:
       sum_num_bytes_for_all_elements
     end
 
@@ -251,7 +251,7 @@ module BinData
     end
 
     def elements
-      @element_list ||= []
+      @elements ||= []
     end
 
     def append_new_element
@@ -279,66 +279,66 @@ module BinData
         end
       end
     end
-  end
 
-  class ArrayArgProcessor < BaseArgProcessor
-    def sanitize_parameters!(obj_class, params) #:nodoc:
-      # ensure one of :initial_length and :read_until exists
-      unless params.has_at_least_one_of?(:initial_length, :read_until)
-        params[:initial_length] = 0
+    # Logic for the :read_until parameter
+    module ReadUntilPlugin
+      def do_read(io)
+        loop do
+          element = append_new_element
+          element.do_read(io)
+          variables = { index: self.length - 1, element: self.last, array: self }
+          break if eval_parameter(:read_until, variables)
+        end
       end
-
-      params.warn_replacement_parameter(:length, :initial_length)
-      params.warn_replacement_parameter(:read_length, :initial_length)
-      params.must_be_integer(:initial_length)
-
-      params.merge!(obj_class.dsl_params)
-      params.sanitize_object_prototype(:type)
     end
-  end
 
-  # Logic for the :read_until parameter
-  module ReadUntilPlugin
-    def do_read(io)
-      loop do
-        element = append_new_element
-        element.do_read(io)
-        variables = { index: self.length - 1, element: self.last, array: self }
-        break if eval_parameter(:read_until, variables)
+    # Logic for the read_until: :eof parameter
+    module ReadUntilEOFPlugin
+      def do_read(io)
+        loop do
+          element = append_new_element
+          begin
+            element.do_read(io)
+          rescue EOFError, IOError
+            elements.pop
+            break
+          end
+        end
       end
     end
-  end
 
-  # Logic for the read_until: :eof parameter
-  module ReadUntilEOFPlugin
-    def do_read(io)
-      loop do
-        element = append_new_element
-        begin
-          element.do_read(io)
-        rescue EOFError, IOError
-          elements.pop
-          break
+    # Logic for the :initial_length parameter
+    module InitialLengthPlugin
+      def do_read(io)
+        elements.each { |el| el.do_read(io) }
+      end
+
+      def elements
+        if @elements.nil?
+          @elements = []
+          eval_parameter(:initial_length).times do
+            @elements << new_element
+          end
         end
+
+        @elements
       end
     end
   end
 
-  # Logic for the :initial_length parameter
-  module InitialLengthPlugin
-    def do_read(io)
-      elements.each { |el| el.do_read(io) }
-    end
-
-    def elements
-      if @element_list.nil?
-        @element_list = []
-        eval_parameter(:initial_length).times do
-          @element_list << new_element
-        end
+  class ArrayArgProcessor < BaseArgProcessor
+    def sanitize_parameters!(obj_class, params) # :nodoc:
+      # ensure one of :initial_length and :read_until exists
+      unless params.has_at_least_one_of?(:initial_length, :read_until)
+        params[:initial_length] = 0
       end
 
-      @element_list
+      params.warn_replacement_parameter(:length, :initial_length)
+      params.warn_replacement_parameter(:read_length, :initial_length)
+      params.must_be_integer(:initial_length)
+
+      params.merge!(obj_class.dsl_params)
+      params.sanitize_object_prototype(:type)
     end
   end
 end

data/lib/bindata/base.rb

@@ -17,7 +17,7 @@ module BinData
       # Instantiates this class and reads from +io+, returning the newly
       # created data object.  +args+ will be used when instantiating.
       def read(io, *args, &block)
-        obj = self.new(*args)
+        obj = new(*args)
         obj.read(io, &block)
         obj
       end
@@ -48,7 +48,7 @@ module BinData
       end
 
       # Registers all subclasses of this class for use
-      def register_subclasses #:nodoc:
+      def register_subclasses # :nodoc:
         singleton_class.send(:undef_method, :inherited)
         define_singleton_method(:inherited) do |subclass|
           RegisteredClasses.register(subclass.name, subclass)
@@ -90,6 +90,8 @@ module BinData
 
     # Creates a new data object based on this instance.
     #
+    # This implements the prototype design pattern.
+    #
     # All parameters will be be duplicated.  Use this method
     # when creating multiple objects with the same parameters.
     def new(value = nil, parent = nil)
@@ -117,8 +119,8 @@ module BinData
     end
 
     # Returns a lazy evaluator for this object.
-    def lazy_evaluator #:nodoc:
-      @lazy ||= LazyEvaluator.new(self)
+    def lazy_evaluator # :nodoc:
+      @lazy_evaluator ||= LazyEvaluator.new(self)
     end
 
     # Returns the parameter referenced by +key+.
@@ -177,7 +179,7 @@ module BinData
 
     # Returns the hexadecimal string representation of this data object.
     def to_hex(&block)
-      to_binary_s(&block).unpack('H*')[0]
+      to_binary_s(&block).unpack1('H*')
     end
 
     # Return a human readable representation of this data object.
@@ -191,7 +193,7 @@ module BinData
     end
 
     # Work with Ruby's pretty-printer library.
-    def pretty_print(pp) #:nodoc:
+    def pretty_print(pp) # :nodoc:
       pp.pp(snapshot)
     end
 
@@ -202,40 +204,28 @@ module BinData
 
     # Returns a user friendly name of this object for debugging purposes.
     def debug_name
-      if @parent
-        @parent.debug_name_of(self)
-      else
-        "obj"
-      end
+      @parent ? @parent.debug_name_of(self) : 'obj'
     end
 
     # Returns the offset (in bytes) of this object with respect to its most
     # distant ancestor.
     def abs_offset
-      if @parent
-        @parent.abs_offset + @parent.offset_of(self)
-      else
-        0
-      end
+      @parent ? @parent.abs_offset + @parent.offset_of(self) : 0
     end
 
     # Returns the offset (in bytes) of this object with respect to its parent.
     def rel_offset
-      if @parent
-        @parent.offset_of(self)
-      else
-        0
-      end
+      @parent ? @parent.offset_of(self) : 0
     end
 
-    def ==(other) #:nodoc:
+    def ==(other) # :nodoc:
       # double dispatch
       other == snapshot
     end
 
     # A version of +respond_to?+ used by the lazy evaluator.  It doesn't
     # reinvoke the evaluator so as to avoid infinite evaluation loops.
-    def safe_respond_to?(symbol, include_private = false) #:nodoc:
+    def safe_respond_to?(symbol, include_private = false) # :nodoc:
       base_respond_to?(symbol, include_private)
     end
 
@@ -329,7 +319,6 @@ module BinData
     # Performs sanity checks on the given parameters.
     # This method converts the parameters to the form expected
     # by the data object.
-    def sanitize_parameters!(obj_class, obj_params)
-    end
+    def sanitize_parameters!(obj_class, obj_params); end
   end
 end

data/lib/bindata/base_primitive.rb

@@ -65,7 +65,7 @@ module BinData
       @value = nil
     end
 
-    def clear? #:nodoc:
+    def clear? # :nodoc:
       @value.nil?
     end
 
@@ -73,13 +73,7 @@ module BinData
       raise ArgumentError, "can't set a nil value for #{debug_name}" if val.nil?
 
       raw_val = val.respond_to?(:snapshot) ? val.snapshot : val
-      @value =
-        begin
-          raw_val.dup
-        rescue TypeError
-          # can't dup Fixnums
-          raw_val
-        end
+      @value = raw_val.dup
     end
 
     def snapshot
@@ -94,18 +88,19 @@ module BinData
       assign(val)
     end
 
-    def respond_to?(symbol, include_private = false) #:nodoc:
+    def respond_to_missing?(symbol, include_all = false) # :nodoc:
       child = snapshot
-      child.respond_to?(symbol, include_private) || super
+      child.respond_to?(symbol, include_all) || super
     end
 
-    def method_missing(symbol, *args, &block) #:nodoc:
+    def method_missing(symbol, *args, &block) # :nodoc:
       child = snapshot
       if child.respond_to?(symbol)
-        self.class.class_eval \
-          "def #{symbol}(*args, &block);" \
-          "  snapshot.#{symbol}(*args, &block);" \
-          "end"
+        self.class.class_eval <<-END, __FILE__, __LINE__ + 1
+          def #{symbol}(*args, &block)         # def clamp(*args, &block)
+            snapshot.#{symbol}(*args, &block)  #   snapshot.clamp(*args, &block)
+          end                                  # end
+        END
         child.__send__(symbol, *args, &block)
       else
         super
@@ -125,15 +120,15 @@ module BinData
       snapshot.hash
     end
 
-    def do_read(io) #:nodoc:
+    def do_read(io) # :nodoc:
       @value = read_and_return_value(io)
     end
 
-    def do_write(io) #:nodoc:
+    def do_write(io) # :nodoc:
       io.writebytes(value_to_binary_string(_value))
     end
 
-    def do_num_bytes #:nodoc:
+    def do_num_bytes # :nodoc:
       value_to_binary_string(_value).length
     end
 
@@ -172,7 +167,7 @@ module BinData
         assert!
       end
 
-      def do_read(io) #:nodoc:
+      def do_read(io) # :nodoc:
         super(io)
         assert!
       end
@@ -205,7 +200,16 @@ module BinData
         reading? ? @value : eval_parameter(:asserted_value)
       end
 
-      def do_read(io) #:nodoc:
+      # The asserted value as a binary string.
+      #
+      # Rationale: while reading, +#to_binary_s+ will use the
+      # value read in, rather than the +:asserted_value+.
+      # This feature is used by Skip.
+      def asserted_binary_s
+        value_to_binary_string(eval_parameter(:asserted_value))
+      end
+
+      def do_read(io) # :nodoc:
         super(io)
         assert!
       end

data/lib/bindata/bits.rb

@@ -5,7 +5,7 @@ module BinData
   # Defines a number of classes that contain a bit based integer.
   # The integer is defined by endian and number of bits.
 
-  module BitField #:nodoc: all
+  module BitField # :nodoc: all
     @@mutex = Mutex.new
 
     class << self
@@ -156,10 +156,10 @@ module BinData
 
   # Create classes for dynamic bitfields
   {
-    "Bit"    => :big,
-    "BitLe"  => :little,
-    "Sbit"   => [:big, :signed],
-    "SbitLe" => [:little, :signed],
+    'Bit'    => :big,
+    'BitLe'  => :little,
+    'Sbit'   => [:big, :signed],
+    'SbitLe' => [:little, :signed]
   }.each_pair { |name, args| BitField.define_class(name, :nbits, *args) }
 
   # Create classes on demand

data/lib/bindata/buffer.rb

@@ -41,7 +41,7 @@ module BinData
   #       end
   #     end
   #   end
-  #     
+  #
   #
   # == Parameters
   #
@@ -80,29 +80,107 @@ module BinData
       @type.snapshot
     end
 
-    def respond_to?(symbol, include_private = false) #:nodoc:
-      @type.respond_to?(symbol, include_private) || super
+    def respond_to_missing?(symbol, include_all = false) # :nodoc:
+      @type.respond_to?(symbol, include_all) || super
     end
 
-    def method_missing(symbol, *args, &block) #:nodoc:
+    def method_missing(symbol, *args, &block) # :nodoc:
       @type.__send__(symbol, *args, &block)
     end
 
-    def do_read(io) #:nodoc:
-      io.with_buffer(eval_parameter(:length)) do
-        @type.do_read(io)
+    def do_read(io) # :nodoc:
+      buf_len = eval_parameter(:length)
+      io.transform(BufferIO.new(buf_len)) do |transformed_io, _|
+        @type.do_read(transformed_io)
       end
     end
 
-    def do_write(io) #:nodoc:
-      io.with_buffer(eval_parameter(:length)) do
-        @type.do_write(io)
+    def do_write(io) # :nodoc:
+      buf_len = eval_parameter(:length)
+      io.transform(BufferIO.new(buf_len)) do |transformed_io, _|
+        @type.do_write(transformed_io)
       end
     end
 
-    def do_num_bytes #:nodoc:
+    def do_num_bytes # :nodoc:
       eval_parameter(:length)
     end
+
+    # Transforms the IO stream to restrict access inside
+    # a buffer of specified length.
+    class BufferIO < IO::Transform
+      def initialize(length)
+        super()
+        @bytes_remaining = length
+      end
+
+      def before_transform
+        @buf_start = offset
+        @buf_end = @buf_start + @bytes_remaining
+      end
+
+      def num_bytes_remaining
+        [@bytes_remaining, super].min
+      rescue IOError
+        @bytes_remaining
+      end
+
+      def skip(n)
+        nbytes = buffer_limited_n(n)
+        @bytes_remaining -= nbytes
+
+        chain_skip(nbytes)
+      end
+
+      def seek_abs(n)
+        if n < @buf_start || n >= @buf_end
+          raise IOError, "can not seek to abs_offset outside of buffer"
+        end
+
+        @bytes_remaining -= (n - offset)
+        chain_seek_abs(n)
+      end
+
+      def read(n)
+        nbytes = buffer_limited_n(n)
+        @bytes_remaining -= nbytes
+
+        chain_read(nbytes)
+      end
+
+      def write(data)
+        nbytes = buffer_limited_n(data.size)
+        @bytes_remaining -= nbytes
+        if nbytes < data.size
+          data = data[0, nbytes]
+        end
+
+        chain_write(data)
+      end
+
+      def after_read_transform
+        read(nil)
+      end
+
+      def after_write_transform
+        write("\x00" * @bytes_remaining)
+      end
+
+      def buffer_limited_n(n)
+        if n.nil?
+          @bytes_remaining
+        elsif n.positive?
+          limit = @bytes_remaining
+          n > limit ? limit : n
+# uncomment if we decide to allow backwards skipping
+#        elsif n.negative?
+#          limit = @bytes_remaining + @buf_start - @buf_end
+#          n < limit ? limit : n
+        else
+          0
+        end
+      end
+    end
   end
 
   class BufferArgProcessor < BaseArgProcessor