bindata 2.3.1 → 2.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c2c402445f12f30bacb95e174ec062916c0f899
-  data.tar.gz: 05fd4d2d0d195a817fe0056d7db707073167826a
+  metadata.gz: a82037bfb06f2af645803e71fcaaf50ec475845a
+  data.tar.gz: 2b111c9173a7fb320fd5f98575e864da786dae0b
 SHA512:
-  metadata.gz: e35b838106b7bf8d4317f5f12f82a5726a4b42cc4c9435c414b52e2ba6269ffe76732b2943879d7e5fd316f8857237e59c516403af667e231c5ff17767900877
-  data.tar.gz: 048dec43da3620cbd4d950378af2f8ea9fdfeb3b38f8eb842faffa0b78a0d94344c656b90a649efabc638f07dfb2619f780b1088964171d08ae593073aec5c7c
+  metadata.gz: 44f161193a8242656fe81684dc419f1936abcb64232928abcd2ab32288c767ccdd2b4dfc7a7168a93af073b02c87c8a3e2fb6a638bc275dc121e67644c77f09a
+  data.tar.gz: 7258cad731fab07ddf5414733d950aefe7b0f78bdf08f886e8d4dcd18710f1c9eb716c22bdcb9128e18695c47334ff1b72dd734bea9997b17d3cb5da3cc7b4ab

data/.travis.yml

@@ -1,12 +1,13 @@
 language: ruby
 rvm:
-  - 2.0.0
-  - 2.1.8
-  - 2.2.4
-  - 2.3.0
+  - 2.0
+  - 2.1.10
+  - 2.2
+  - 2.3
   - jruby
   - ruby-head
 
 matrix:
   allow_failures:
+    - rvm: jruby
     - rvm: ruby-head

data/ChangeLog.rdoc

@@ -1,5 +1,10 @@
 = BinData Changelog
 
+== Version 2.3.2 (2016-09-02)
+
+* IO#num_bytes_remaining now works inside Buffers.
+* Added ability to skip to arbitrary byte patterns.  Requested by Stefan Kolb.
+
 == Version 2.3.1 (2016-06-17)
 
 * Improve list of reserved words.  Thanks to Claudius Coenen.

data/lib/bindata/dsl.rb

@@ -122,7 +122,8 @@ module BinData
       end
 
       def dsl_params
-        send(parser_abilities[@parser_type].at(0))
+        abilities = parser_abilities[@parser_type]
+        send(abilities.at(0), abilities.at(1))
       end
 
       def method_missing(*args, &block)
@@ -135,17 +136,18 @@ module BinData
 
       def parser_abilities
         @abilities ||= {
-          :struct     => [:to_struct_params, [:multiple_fields, :optional_fieldnames, :hidden_fields]],
-          :array      => [:to_array_params,  [:multiple_fields, :optional_fieldnames]],
-          :buffer     => [:to_array_params,  [:multiple_fields, :optional_fieldnames, :hidden_fields]],
-          :choice     => [:to_choice_params, [:multiple_fields, :all_or_none_fieldnames, :fieldnames_are_values]],
-          :delayed_io => [:to_array_params,  [:multiple_fields, :optional_fieldnames, :hidden_fields]],
-          :primitive  => [:to_struct_params, [:multiple_fields, :optional_fieldnames]]
+          :struct     => [:to_struct_params, :struct,      [:multiple_fields, :optional_fieldnames, :hidden_fields]],
+          :array      => [:to_object_params, :type,        [:multiple_fields, :optional_fieldnames]],
+          :buffer     => [:to_object_params, :type,        [:multiple_fields, :optional_fieldnames, :hidden_fields]],
+          :choice     => [:to_choice_params, :choices,     [:multiple_fields, :all_or_none_fieldnames, :fieldnames_are_values]],
+          :delayed_io => [:to_object_params, :type,        [:multiple_fields, :optional_fieldnames, :hidden_fields]],
+          :primitive  => [:to_struct_params, :struct,      [:multiple_fields, :optional_fieldnames]],
+          :skip       => [:to_object_params, :until_valid, [:multiple_fields, :optional_fieldnames]],
         }
       end
 
       def option?(opt)
-        parser_abilities[@parser_type].at(1).include?(opt)
+        parser_abilities[@parser_type].at(2).include?(opt)
       end
 
       def ensure_hints
@@ -219,30 +221,30 @@ module BinData
         raise exception, message + " in #{@the_class}", backtrace
       end
 
-      def to_array_params
+      def to_object_params(key)
         case fields.length
         when 0
           {}
         when 1
-          {:type => fields[0].prototype}
+          {key => fields[0].prototype}
         else
-          {:type => [:struct, to_struct_params]}
+          {key=> [:struct, to_struct_params]}
         end
       end
 
-      def to_choice_params
+      def to_choice_params(key)
         if fields.length == 0
           {}
         elsif fields.all_field_names_blank?
-          {:choices => fields.collect { |f| f.prototype }}
+          {key => fields.collect { |f| f.prototype }}
         else
           choices = {}
           fields.each { |f| choices[f.name] = f.prototype }
-          {:choices => choices}
+          {key => choices}
         end
       end
 
-      def to_struct_params
+      def to_struct_params(*unused)
         result = {:fields => fields}
         if not endian.nil?
           result[:endian] = endian
@@ -386,7 +388,8 @@ module BinData
           :buffer     => BinData::Buffer,
           :choice     => BinData::Choice,
           :delayed_io => BinData::DelayedIO,
-          :struct     => BinData::Struct
+          :skip       => BinData::Skip,
+          :struct     => BinData::Struct,
         }
 
         if bindata_classes.include?(@type)

data/lib/bindata/io.rb

@@ -18,7 +18,7 @@ module BinData
         end
 
         @raw_io = io
-        @buffer_end_pos = nil
+        @buffer_end_points = nil
 
         extend seekable? ? SeekableStream : UnSeekableStream
         stream_init
@@ -38,12 +38,12 @@ module BinData
       end
 
       def buffer_limited_n(n)
-        if @buffer_end_pos
+        if @buffer_end_points
           if n.nil? or n > 0
-            max = @buffer_end_pos[1] - offset
+            max = @buffer_end_points[1] - offset
             n = max if n.nil? or n > max
           else
-            min = @buffer_end_pos[0] - offset
+            min = @buffer_end_points[0] - offset
             n = min if n < min
           end
         end
@@ -52,16 +52,16 @@ module BinData
       end
 
       def with_buffer_common(n, &block)
-        prev = @buffer_end_pos
+        prev = @buffer_end_points
         if prev
           avail = prev[1] - offset
           n = avail if n > avail
         end
-        @buffer_end_pos = [offset, offset + n]
+        @buffer_end_points = [offset, offset + n]
         begin
-          block.call(*@buffer_end_pos)
+          block.call(*@buffer_end_points)
         ensure
-          @buffer_end_pos = prev
+          @buffer_end_points = prev
         end
       end
 
@@ -69,14 +69,33 @@ module BinData
       module SeekableStream
         # The number of bytes remaining in the input stream.
         def num_bytes_remaining
-          mark = @raw_io.pos
+          start_mark = @raw_io.pos
           @raw_io.seek(0, ::IO::SEEK_END)
-          bytes_remaining = @raw_io.pos - mark
-          @raw_io.seek(mark, ::IO::SEEK_SET)
+          end_mark = @raw_io.pos
+
+          if @buffer_end_points
+            if @buffer_end_points[1] < end_mark
+              end_mark = @buffer_end_points[1]
+            end
+          end
+
+          bytes_remaining = end_mark - start_mark
+          @raw_io.seek(start_mark, ::IO::SEEK_SET)
 
           bytes_remaining
         end
 
+        # All io calls in +block+ are rolled back after this
+        # method completes.
+        def with_readahead(&block)
+          mark = @raw_io.pos
+          begin
+            block.call
+          ensure
+            @raw_io.seek(mark, ::IO::SEEK_SET)
+          end
+        end
+
         #-----------
         private
 
@@ -112,6 +131,26 @@ module BinData
           raise IOError, "stream is unseekable"
         end
 
+        # All io calls in +block+ are rolled back after this
+        # method completes.
+        def with_readahead(&block)
+          mark = @offset
+          @read_data = ""
+          @in_readahead = true
+
+          class << self
+            alias_method :read_raw_without_readahead, :read_raw
+            alias_method :read_raw, :read_raw_with_readahead
+          end
+
+          begin
+            block.call
+          ensure
+            @offset = mark
+            @in_readahead = false
+          end
+        end
+
         #-----------
         private
 
@@ -125,6 +164,33 @@ module BinData
           data
         end
 
+        def read_raw_with_readahead(n)
+          data = ""
+
+          if @read_data.length > 0 and not @in_readahead
+            bytes_to_consume = [n, @read_data.length].min
+            data << @read_data.slice!(0, bytes_to_consume)
+            n -= bytes_to_consume
+
+            if @read_data.length == 0
+              class << self
+                alias_method :read_raw, :read_raw_without_readahead
+              end
+            end
+          end
+
+          raw_data = @raw_io.read(n)
+          data << raw_data if raw_data
+
+          if @in_readahead
+            @read_data << data
+          end
+
+          @offset += data.size
+
+          data
+        end
+
         def write_raw(data)
           @offset += data.size
           @raw_io.write(data)

data/lib/bindata/skip.rb

@@ -16,6 +16,16 @@ module BinData
   #   obj = A.read("abcdefghij")
   #   obj.a #=> "fghij"
   #
+  #
+  #   class B < BinData::Record
+  #     skip :until_valid => [:string, {:read_length => 2, :assert => "ef"} ]
+  #     string :b, :read_length => 5
+  #   end
+  #
+  #   obj = B.read("abcdefghij")
+  #   obj.b #=> "efghi"
+  #
+  #
   # == Parameters
   #
   # Skip objects accept all the params that BinData::BasePrimitive
@@ -23,17 +33,25 @@ module BinData
   #
   # <tt>:length</tt>::        The number of bytes to skip.
   # <tt>:to_abs_offset</tt>:: Skips to the given absolute offset.
+  # <tt>:until_valid</tt>::   Skips untils a given byte pattern is matched.
+  #                           This parameter contains a type that will raise
+  #                           a BinData::ValidityError unless an acceptable byte
+  #                           sequence is found.  The type is represented by a
+  #                           Symbol, or if the type is to have params #
+  #                           passed to it, then it should be provided as #
+  #                           <tt>[type_symbol, hash_params]</tt>.
   #
   class Skip < BinData::BasePrimitive
 
     arg_processor :skip
 
-    optional_parameters :length, :to_abs_offset
-    mutually_exclusive_parameters :length, :to_abs_offset
+    optional_parameters :length, :to_abs_offset, :until_valid
+    mutually_exclusive_parameters :length, :to_abs_offset, :until_valid
 
     def initialize_shared_instance
       extend SkipLengthPlugin      if has_parameter?(:length)
       extend SkipToAbsOffsetPlugin if has_parameter?(:to_abs_offset)
+      extend SkipUntilValidPlugin if has_parameter?(:until_valid)
       super
     end
 
@@ -66,10 +84,17 @@ module BinData
 
   class SkipArgProcessor < BaseArgProcessor
     def sanitize_parameters!(obj_class, params)
-      unless (params.has_parameter?(:length) or params.has_parameter?(:to_abs_offset))
-        raise ArgumentError, "#{obj_class} requires either :length or :to_abs_offset"
+      unless params.has_parameter?(:length) or
+             params.has_parameter?(:to_abs_offset) or
+             params.has_parameter?(:until_valid)
+        raise ArgumentError, "#{obj_class} requires either :length, :to_abs_offset or :until_valid"
       end
       params.must_be_integer(:to_abs_offset, :length)
+
+      if params.needs_sanitizing?(:until_valid)
+        el_type, el_params = params[:until_valid]
+        params[:until_valid] = params.create_sanitized_object_prototype(el_type, el_params)
+      end
     end
   end
 
@@ -86,4 +111,29 @@ module BinData
       eval_parameter(:to_abs_offset) - abs_offset
     end
   end
+
+  # Logic for the :until_valid parameter
+  module SkipUntilValidPlugin
+    def skip_length
+      # no skipping when writing
+      0
+    end
+
+    def read_and_return_value(io)
+      prototype = get_parameter(:until_valid)
+      validator = prototype.instantiate(nil, self)
+
+      valid = false
+      until valid
+        begin
+          io.with_readahead do
+            validator.read(io)
+            valid = true
+          end
+        rescue ValidityError
+          io.readbytes(1)
+        end
+      end
+    end
+  end
 end

data/lib/bindata/version.rb

@@ -1,3 +1,3 @@
 module BinData
-  VERSION = "2.3.1"
+  VERSION = "2.3.2"
 end

data/test/io_test.rb

@@ -142,6 +142,85 @@ describe BinData::IO::Read, "#with_buffer" do
       io.offset.must_equal(2)
     end
   end
+
+  it "greater than stream size consumes all bytes" do
+    io.with_buffer(30) do
+      io.readbytes(4).must_equal "abcd"
+    end
+    io.offset.must_equal(20)
+  end
+
+  it "restricts #num_bytes_remaining" do
+    io.with_buffer(10) do
+      io.readbytes(2)
+      io.num_bytes_remaining.must_equal 8
+    end
+  end
+
+  it "greater than stream size doesn't restrict #num_bytes_remaining" do
+    io.with_buffer(30) do
+      io.readbytes(2)
+      io.num_bytes_remaining.must_equal 18
+    end
+  end
+end
+
+module IOReadWithReadahead
+  def test_rolls_back_short_reads
+    io.readbytes(2).must_equal "ab"
+    io.with_readahead do
+      io.readbytes(4).must_equal "cdef"
+    end
+    io.offset.must_equal 2
+  end
+
+  def test_rolls_back_read_all_bytes
+    io.readbytes(3).must_equal "abc"
+    io.with_readahead do
+      io.read_all_bytes.must_equal "defghijklmnopqrst"
+    end
+    io.offset.must_equal 3
+  end
+
+  def test_inside_buffer_rolls_back_reads
+    io.with_buffer(10) do
+      io.with_readahead do
+        io.readbytes(4).must_equal "abcd"
+      end
+      io.offset.must_equal 0
+    end
+    io.offset.must_equal 10
+  end
+
+  def test_outside_buffer_rolls_back_reads
+    io.with_readahead do
+      io.with_buffer(10) do
+        io.readbytes(4).must_equal "abcd"
+      end
+      io.offset.must_equal 10
+    end
+    io.offset.must_equal 0
+  end
+end
+
+describe BinData::IO::Read, "#with_readahead" do
+  let(:stream) { StringIO.new "abcdefghijklmnopqrst" }
+  let(:io) { BinData::IO::Read.new(stream) }
+
+  include IOReadWithReadahead
+end
+
+describe BinData::IO::Read, "unseekable stream #with_readahead" do
+  let(:stream) {
+    io = StringIO.new "abcdefghijklmnopqrst"
+    def io.pos
+      raise Errno::EPIPE
+    end
+    io
+  }
+  let(:io) { BinData::IO::Read.new(stream) }
+
+  include IOReadWithReadahead
 end
 
 describe BinData::IO::Write, "writing to non seekable stream" do

data/test/skip_test.rb

@@ -85,3 +85,53 @@ describe BinData::Skip, "with :to_abs_offset" do
     }.must_raise BinData::ValidityError
   end
 end
+
+describe BinData::Skip, "with :until_valid" do
+  let(:io) { StringIO.new("abcdefghij") }
+
+  it "skips to valid match" do
+    skip_obj = [:string, { :read_length => 1, :assert => "f" }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    obj.read(io)
+    io.pos.must_equal 5
+  end
+
+  it "doesn't skip when validator doesn't assert" do
+    skip_obj = [:string, { :read_length => 1 }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    obj.read(io)
+    io.pos.must_equal 0
+  end
+
+  it "raises EOFError when no match" do
+    skip_obj = [:string, { :read_length => 1, :assert => "X" }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    lambda {
+      obj.read(io)
+    }.must_raise EOFError
+  end
+
+  it "raises IOError when validator reads beyond stream" do
+    skip_obj = [:string, { :read_length => 30 }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    lambda {
+      obj.read(io)
+    }.must_raise IOError
+  end
+
+  class DSLSkip < BinData::Record
+    skip :s do
+      string :read_length => 1, :assert => "f"
+    end
+    string :a, :read_length => 1
+  end
+
+  it "uses block form" do
+    obj = DSLSkip.read(io)
+    obj.a.must_equal "f"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bindata
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.3.2
 platform: ruby
 authors:
 - Dion Mendel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-17 00:00:00.000000000 Z
+date: 2016-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake