bindata 2.4.10 → 2.5.0

data/lib/bindata/struct.rb

@@ -1,7 +1,7 @@
 require 'bindata/base'
+require 'bindata/delayed_io'
 
 module BinData
-
   class Base
     optional_parameter :onlyif, :byte_align  # Used by Struct
   end
@@ -65,16 +65,18 @@ module BinData
     RESERVED =
       Hash[*
         (Hash.instance_methods +
-         %w{alias and begin break case class def defined do else elsif
+         %w[alias and begin break case class def defined do else elsif
             end ensure false for if in module next nil not or redo
             rescue retry return self super then true undef unless until
-            when while yield} +
-         %w{array element index value} +
-         %w{type initial_length read_until} +
-         %w{fields endian search_prefix hide only_if byte_align} +
-         %w{choices selection copy_on_change} +
-         %w{read_abs_offset struct_params}).collect(&:to_sym).
-         uniq.collect { |key| [key, true] }.flatten
+            when while yield] +
+         %w[array element index value] +
+         %w[type initial_length read_until] +
+         %w[fields endian search_prefix hide onlyif byte_align] +
+         %w[choices selection copy_on_change] +
+         %w[read_abs_offset struct_params])
+        .collect(&:to_sym)
+        .uniq.collect { |key| [key, true] }
+        .flatten
       ]
 
     def initialize_shared_instance
@@ -89,11 +91,11 @@ module BinData
       @field_objs = []
     end
 
-    def clear #:nodoc:
-      @field_objs.each { |f| f.clear unless f.nil? }
+    def clear # :nodoc:
+      @field_objs.each { |f| f.nil? || f.clear }
     end
 
-    def clear? #:nodoc:
+    def clear? # :nodoc:
       @field_objs.all? { |f| f.nil? || f.clear? }
     end
 
@@ -123,28 +125,28 @@ module BinData
       end
     end
 
-    def debug_name_of(child) #:nodoc:
+    def debug_name_of(child) # :nodoc:
       field_name = @field_names[find_index_of(child)]
       "#{debug_name}.#{field_name}"
     end
 
-    def offset_of(child) #:nodoc:
+    def offset_of(child) # :nodoc:
       instantiate_all_objs
       sum = sum_num_bytes_below_index(find_index_of(child))
       child.bit_aligned? ? sum.floor : sum.ceil
     end
 
-    def do_read(io) #:nodoc:
+    def do_read(io) # :nodoc:
       instantiate_all_objs
-      @field_objs.each { |f| f.do_read(io) if include_obj?(f) }
+      @field_objs.each { |f| f.do_read(io) if include_obj_for_io?(f) }
     end
 
-    def do_write(io) #:nodoc
+    def do_write(io) # :nodoc:
       instantiate_all_objs
-      @field_objs.each { |f| f.do_write(io) if include_obj?(f) }
+      @field_objs.each { |f| f.do_write(io) if include_obj_for_io?(f) }
     end
 
-    def do_num_bytes #:nodoc:
+    def do_num_bytes # :nodoc:
       instantiate_all_objs
       sum_num_bytes_for_all_fields
     end
@@ -154,19 +156,28 @@ module BinData
     end
 
     def []=(key, value)
-      obj = find_obj_for_name(key)
-      if obj
-        obj.assign(value)
-      end
+      find_obj_for_name(key)&.assign(value)
     end
 
     def key?(key)
       @field_names.index(base_field_name(key))
     end
 
-    def each_pair
-      @field_names.compact.each do |name|
-        yield [name, find_obj_for_name(name)]
+    # Calls the given block for each field_name-field_obj pair.
+    #
+    # Does not include anonymous or hidden fields unless
+    # +include_all+ is true.
+    def each_pair(include_all = false)
+      instantiate_all_objs
+
+      pairs = @field_names.zip(@field_objs).select do |name, _obj|
+        name || include_all
+      end
+
+      if block_given?
+        pairs.each { |el| yield(el) }
+      else
+        pairs.each
       end
     end
 
@@ -204,8 +215,6 @@ module BinData
       if index
         instantiate_obj_at(index)
         @field_objs[index]
-      else
-        nil
       end
     end
 
@@ -242,7 +251,7 @@ module BinData
         {}
       else
         hash = Snapshot.new
-        val.each_pair { |k,v| hash[k] = v }
+        val.each_pair { |k, v| hash[k] = v }
         hash
       end
     end
@@ -263,17 +272,23 @@ module BinData
       end
     end
 
+    def include_obj_for_io?(obj)
+      # Used by #do_read and #do_write, to ensure the stream is passed to
+      # DelayedIO objects for delayed processing.
+      include_obj?(obj) || DelayedIO === obj
+    end
+
     def include_obj?(obj)
       !obj.has_parameter?(:onlyif) || obj.eval_parameter(:onlyif)
     end
 
     # A hash that can be accessed via attributes.
-    class Snapshot < ::Hash #:nodoc:
+    class Snapshot < ::Hash # :nodoc:
       def []=(key, value)
         super unless value.nil?
       end
 
-      def respond_to?(symbol, include_private = false)
+      def respond_to_missing?(symbol, include_all = false)
         key?(symbol) || super
       end
 
@@ -281,60 +296,71 @@ module BinData
         key?(symbol) ? self[symbol] : super
       end
     end
-  end
 
-  # Align fields to a multiple of :byte_align
-  module ByteAlignPlugin
-    def do_read(io)
-      initial_offset = io.offset
-      instantiate_all_objs
-      @field_objs.each do |f|
-        if include_obj?(f)
+    # Align fields to a multiple of :byte_align
+    module ByteAlignPlugin
+      def do_read(io)
+        offset = 0
+        instantiate_all_objs
+        @field_objs.each do |f|
+          next unless include_obj?(f)
+
           if align_obj?(f)
-            io.seekbytes(bytes_to_align(f, io.offset - initial_offset))
+            nbytes = bytes_to_align(f, offset.ceil)
+            offset = offset.ceil + nbytes
+            io.readbytes(nbytes)
           end
+
           f.do_read(io)
+          nbytes = f.do_num_bytes
+          offset = (nbytes.is_a?(Integer) ? offset.ceil : offset) + nbytes
         end
       end
-    end
 
-    def do_write(io)
-      initial_offset = io.offset
-      instantiate_all_objs
-      @field_objs.each do |f|
-        if include_obj?(f)
+      def do_write(io)
+        offset = 0
+        instantiate_all_objs
+        @field_objs.each do |f|
+          next unless include_obj?(f)
+
           if align_obj?(f)
-            io.writebytes("\x00" * bytes_to_align(f, io.offset - initial_offset))
+            nbytes = bytes_to_align(f, offset.ceil)
+            offset = offset.ceil + nbytes
+            io.writebytes("\x00" * nbytes)
           end
+
           f.do_write(io)
+          nbytes = f.do_num_bytes
+          offset = (nbytes.is_a?(Integer) ? offset.ceil : offset) + nbytes
         end
       end
-    end
 
-    def sum_num_bytes_below_index(index)
-      sum = 0
-      (0...@field_objs.length).each do |i|
-        obj = @field_objs[i]
-        if include_obj?(obj)
-          sum = sum.ceil + bytes_to_align(obj, sum.ceil) if align_obj?(obj)
+      def sum_num_bytes_below_index(index)
+        sum = 0
+        @field_objs.each_with_index do |obj, i|
+          next unless include_obj?(obj)
+
+          if align_obj?(obj)
+            sum = sum.ceil + bytes_to_align(obj, sum.ceil)
+          end
 
           break if i >= index
 
           nbytes = obj.do_num_bytes
           sum = (nbytes.is_a?(Integer) ? sum.ceil : sum) + nbytes
         end
-      end
 
-      sum
-    end
+        sum
+      end
 
-    def bytes_to_align(obj, rel_offset)
-      align = obj.eval_parameter(:byte_align)
-      (align - (rel_offset % align)) % align
-    end
+      def bytes_to_align(obj, rel_offset)
+        align = obj.eval_parameter(:byte_align)
+        (align - (rel_offset % align)) % align
+      end
 
-    def align_obj?(obj)
-      obj.has_parameter?(:byte_align)
+      def align_obj?(obj)
+        obj.has_parameter?(:byte_align)
+      end
     end
   end
 
@@ -355,13 +381,11 @@ module BinData
 
     def sanitize_search_prefix(params)
       params.sanitize(:search_prefix) do |sprefix|
-        search_prefix = []
-        Array(sprefix).each do |prefix|
-          prefix = prefix.to_s.chomp("_")
-          search_prefix << prefix if prefix != ""
+        search_prefix = Array(sprefix).collect do |prefix|
+          prefix.to_s.chomp("_")
         end
 
-        search_prefix
+        search_prefix - [""]
       end
     end
 

data/lib/bindata/trace.rb

@@ -1,8 +1,26 @@
 module BinData
+
+  # Turn on trace information when reading a BinData object.
+  # If +block+ is given then the tracing only occurs for that block.
+  # This is useful for debugging a BinData declaration.
+  def trace_reading(io = STDERR)
+    @tracer = Tracer.new(io)
+    [BasePrimitive, Choice].each(&:turn_on_tracing)
+
+    if block_given?
+      begin
+        yield
+      ensure
+        [BasePrimitive, Choice].each(&:turn_off_tracing)
+        @tracer = nil
+      end
+    end
+  end
+
   # reference to the current tracer
   @tracer ||= nil
 
-  class Tracer #:nodoc:
+  class Tracer # :nodoc:
     def initialize(io)
       @trace_io = io
     end
@@ -20,47 +38,34 @@ module BinData
     end
   end
 
-  # Turn on trace information when reading a BinData object.
-  # If +block+ is given then the tracing only occurs for that block.
-  # This is useful for debugging a BinData declaration.
-  def trace_reading(io = STDERR)
-    @tracer = Tracer.new(io)
-    [BasePrimitive, Choice].each(&:turn_on_tracing)
-
-    if block_given?
-      begin
-        yield
-      ensure
-        [BasePrimitive, Choice].each(&:turn_off_tracing)
-        @tracer = nil
-      end
-    end
-  end
-
-  def trace_message #:nodoc:
-    yield @tracer if @tracer
+  def trace_message # :nodoc:
+    yield @tracer
   end
 
   module_function :trace_reading, :trace_message
 
-  class BasePrimitive < BinData::Base
-    class << self
-      def turn_on_tracing
+  module TraceHook
+    def turn_on_tracing
+      if !method_defined? :do_read_without_hook
         alias_method :do_read_without_hook, :do_read
         alias_method :do_read, :do_read_with_hook
       end
+    end
 
-      def turn_off_tracing
+    def turn_off_tracing
+      if method_defined? :do_read_without_hook
         alias_method :do_read, :do_read_without_hook
+        remove_method :do_read_without_hook
       end
     end
+  end
+
+  class BasePrimitive < BinData::Base
+    extend TraceHook
 
     def do_read_with_hook(io)
       do_read_without_hook(io)
-      trace_value
-    end
 
-    def trace_value
       BinData.trace_message do |tracer|
         value_string = _value.inspect
         tracer.trace_obj(debug_name, value_string)
@@ -69,27 +74,15 @@ module BinData
   end
 
   class Choice < BinData::Base
-    class << self
-      def turn_on_tracing
-        alias_method :do_read_without_hook, :do_read
-        alias_method :do_read, :do_read_with_hook
-      end
-
-      def turn_off_tracing
-        alias_method :do_read, :do_read_without_hook
-      end
-    end
+    extend TraceHook
 
     def do_read_with_hook(io)
-      trace_selection
-      do_read_without_hook(io)
-    end
-
-    def trace_selection
       BinData.trace_message do |tracer|
         selection_string = eval_parameter(:selection).inspect
         tracer.trace_obj("#{debug_name}-selection-", selection_string)
       end
+
+      do_read_without_hook(io)
     end
   end
 end

data/lib/bindata/transform/brotli.rb

@@ -0,0 +1,35 @@
+require 'brotli'
+
+module BinData
+  module Transform
+    # Transforms a brotli compressed data stream.
+    #
+    #     gem install brotli
+    class Brotli < BinData::IO::Transform
+      transform_changes_stream_length!
+
+      def initialize(read_length)
+        super()
+        @length = read_length
+      end
+
+      def read(n)
+        @read ||= ::Brotli::inflate(chain_read(@length))
+        @read.slice!(0...n)
+      end
+
+      def write(data)
+        @write ||= create_empty_binary_string
+        @write << data
+      end
+
+      def after_read_transform
+        raise IOError, "didn't read all data" unless @read.empty?
+      end
+
+      def after_write_transform
+        chain_write(::Brotli::deflate(@write))
+      end
+    end
+  end
+end

data/lib/bindata/transform/lz4.rb

@@ -0,0 +1,35 @@
+require 'extlz4'
+
+module BinData
+  module Transform
+    # Transforms a LZ4 compressed data stream.
+    #
+    #     gem install extlz4
+    class LZ4 < BinData::IO::Transform
+      transform_changes_stream_length!
+
+      def initialize(read_length)
+        super()
+        @length = read_length
+      end
+
+      def read(n)
+        @read ||= ::LZ4::decode(chain_read(@length))
+        @read.slice!(0...n)
+      end
+
+      def write(data)
+        @write ||= create_empty_binary_string
+        @write << data
+      end
+
+      def after_read_transform
+        raise IOError, "didn't read all data" unless @read.empty?
+      end
+
+      def after_write_transform
+        chain_write(::LZ4::encode(@write))
+      end
+    end
+  end
+end

data/lib/bindata/transform/lzma.rb

@@ -0,0 +1,35 @@
+require 'xz'
+
+module BinData
+  module Transform
+    # Transforms a lzma compressed data stream.
+    #
+    #     gem install ruby-xz
+    class Lzma < BinData::IO::Transform
+      transform_changes_stream_length!
+
+      def initialize(read_length)
+        super()
+        @length = read_length
+      end
+
+      def read(n)
+        @read ||= ::XZ::decompress(chain_read(@length))
+        @read.slice!(0...n)
+      end
+
+      def write(data)
+        @write ||= create_empty_binary_string
+        @write << data
+      end
+
+      def after_read_transform
+        raise IOError, "didn't read all data" unless @read.empty?
+      end
+
+      def after_write_transform
+        chain_write(::XZ::compress(@write))
+      end
+    end
+  end
+end

data/lib/bindata/transform/xor.rb

@@ -0,0 +1,19 @@
+module BinData
+  module Transform
+    # Transforms the data stream by xoring each byte.
+    class Xor < BinData::IO::Transform
+      def initialize(xor)
+        super()
+        @xor = xor
+      end
+
+      def read(n)
+        chain_read(n).bytes.map { |byte| (byte ^ @xor).chr }.join
+      end
+
+      def write(data)
+        chain_write(data.bytes.map { |byte| (byte ^ @xor).chr }.join)
+      end
+    end
+  end
+end

data/lib/bindata/transform/xz.rb

@@ -0,0 +1,35 @@
+require 'xz'
+
+module BinData
+  module Transform
+    # Transforms a xz compressed data stream.
+    #
+    #     gem install ruby-xz
+    class XZ < BinData::IO::Transform
+      transform_changes_stream_length!
+
+      def initialize(read_length)
+        super()
+        @length = read_length
+      end
+
+      def read(n)
+        @read ||= ::XZ::decompress(chain_read(@length))
+        @read.slice!(0...n)
+      end
+
+      def write(data)
+        @write ||= create_empty_binary_string
+        @write << data
+      end
+
+      def after_read_transform
+        raise IOError, "didn't read all data" unless @read.empty?
+      end
+
+      def after_write_transform
+        chain_write(::XZ::compress(@write))
+      end
+    end
+  end
+end

data/lib/bindata/transform/zlib.rb

@@ -0,0 +1,33 @@
+require 'zlib'
+
+module BinData
+  module Transform
+    # Transforms a zlib compressed data stream.
+    class Zlib < BinData::IO::Transform
+      transform_changes_stream_length!
+
+      def initialize(read_length)
+        super()
+        @length = read_length
+      end
+
+      def read(n)
+        @read ||= ::Zlib::Inflate.inflate(chain_read(@length))
+        @read.slice!(0...n)
+      end
+
+      def write(data)
+        @write ||= create_empty_binary_string
+        @write << data
+      end
+
+      def after_read_transform
+        raise IOError, "didn't read all data" unless @read.empty?
+      end
+
+      def after_write_transform
+        chain_write(::Zlib::Deflate.deflate(@write))
+      end
+    end
+  end
+end

data/lib/bindata/transform/zstd.rb

@@ -0,0 +1,35 @@
+require 'zstd-ruby'
+
+module BinData
+  module Transform
+    # Transforms a zstd compressed data stream.
+    #
+    #     gem install zstd-ruby
+    class Zstd < BinData::IO::Transform
+      transform_changes_stream_length!
+
+      def initialize(read_length)
+        super()
+        @length = read_length
+      end
+
+      def read(n)
+        @read ||= ::Zstd::decompress(chain_read(@length))
+        @read.slice!(0...n)
+      end
+
+      def write(data)
+        @write ||= create_empty_binary_string
+        @write << data
+      end
+
+      def after_read_transform
+        raise IOError, "didn't read all data" unless @read.empty?
+      end
+
+      def after_write_transform
+        chain_write(::Zstd::compress(@write))
+      end
+    end
+  end
+end

data/lib/bindata/uint8_array.rb

@@ -1,4 +1,4 @@
-require "bindata/base_primitive"
+require 'bindata/base_primitive'
 
 module BinData
   # Uint8Array is a specialised type of array that only contains
@@ -49,7 +49,7 @@ module BinData
   end
 
   class Uint8ArrayArgProcessor < BaseArgProcessor
-    def sanitize_parameters!(obj_class, params) #:nodoc:
+    def sanitize_parameters!(obj_class, params) # :nodoc:
       # ensure one of :initial_length and :read_until exists
       unless params.has_at_least_one_of?(:initial_length, :read_until)
         params[:initial_length] = 0

data/lib/bindata/version.rb

@@ -1,3 +1,3 @@
 module BinData
-  VERSION = "2.4.10"
+  VERSION = '2.5.0'
 end

data/lib/bindata/virtual.rb

@@ -1,4 +1,4 @@
-require "bindata/base"
+require 'bindata/base'
 
 module BinData
   # A virtual field is one that is neither read, written nor occupies space in
@@ -15,7 +15,7 @@ module BinData
   #
   #   obj = A.read("abcdeabcde")
   #   obj.a #=> "abcde"
-  #   obj.c.offset #=> 10
+  #   obj.c.rel_offset #=> 10
   #
   #   obj = A.read("abcdeABCDE") #=> BinData::ValidityError: assertion failed for obj.c
   #
@@ -29,12 +29,9 @@ module BinData
   # [<tt>:value</tt>]     The virtual object will always have this value.
   #
   class Virtual < BinData::BasePrimitive
+    def do_read(io); end
 
-    def do_read(io)
-    end
-
-    def do_write(io)
-    end
+    def do_write(io); end
 
     def do_num_bytes
       0.0