marshal-structure 1.1.1 → 2.0

data/lib/marshal/structure/tokenizer.rb

@@ -0,0 +1,449 @@
+class Marshal::Structure::Tokenizer
+
+  ##
+  # Supported major Marshal version
+
+  MAJOR_VERSION = 4
+
+  ##
+  # Supported minor Marshal version
+
+  MINOR_VERSION = 8
+
+  ##
+  # nil type prefix
+
+  TYPE_NIL = '0'
+
+  ##
+  # true type prefix
+
+  TYPE_TRUE = 'T'
+
+  ##
+  # false type prefix
+
+  TYPE_FALSE = 'F'
+
+  ##
+  # Fixnum type prefix
+
+  TYPE_FIXNUM = 'i'
+
+  ##
+  # An object that has been extended with a module
+
+  TYPE_EXTENDED = 'e'
+
+  ##
+  # A subclass of a built-in type
+
+  TYPE_UCLASS = 'C'
+
+  ##
+  # A ruby Object
+
+  TYPE_OBJECT = 'o'
+
+  ##
+  # A wrapped C pointer
+
+  TYPE_DATA = 'd'
+
+  ##
+  # An object saved with _dump
+
+  TYPE_USERDEF = 'u'
+
+  ##
+  # An object saved with marshal_dump
+
+  TYPE_USRMARSHAL = 'U'
+
+  ##
+  # A Float
+
+  TYPE_FLOAT = 'f'
+
+  ##
+  # A Bignum
+
+  TYPE_BIGNUM = 'l'
+
+  ##
+  # A String
+
+  TYPE_STRING = '"'
+
+  ##
+  # A Regexp
+
+  TYPE_REGEXP = '/'
+
+  ##
+  # An Array
+
+  TYPE_ARRAY = '['
+
+  ##
+  # A Hash
+
+  TYPE_HASH = '{'
+
+  ##
+  # A Hash with a default value (not proc)
+
+  TYPE_HASH_DEF = '}'
+
+  ##
+  # A Struct
+
+  TYPE_STRUCT = 'S'
+
+  ##
+  # An old-style Module (reference, not content)
+  #
+  # I'm not sure what makes this old.  The byte stream is identical to
+  # TYPE_MODULE
+
+  TYPE_MODULE_OLD = 'M'
+
+  ##
+  # A class (reference, not content)
+
+  TYPE_CLASS = 'c'
+
+  ##
+  # A module (reference, not content)
+
+  TYPE_MODULE = 'm'
+
+  ##
+  # A Symbol
+
+  TYPE_SYMBOL = ':'
+
+  ##
+  # A reference to a previously Symbol
+
+  TYPE_SYMLINK = ';'
+
+  ##
+  # Instance variables for a following object
+
+  TYPE_IVAR = 'I'
+
+  ##
+  # A reference to a previously-stored Object
+
+  TYPE_LINK = '@'
+
+  TYPE_MAP = Hash.new do |_, type| # :nodoc:
+    raise Marshal::Structure::Error, "unknown type #{type.inspect}"
+  end
+
+  TYPE_MAP[TYPE_ARRAY]      = :array
+  TYPE_MAP[TYPE_BIGNUM]     = :bignum
+  TYPE_MAP[TYPE_CLASS]      = :class
+  TYPE_MAP[TYPE_DATA]       = :data
+  TYPE_MAP[TYPE_EXTENDED]   = :extended
+  TYPE_MAP[TYPE_FALSE]      = :false
+  TYPE_MAP[TYPE_FIXNUM]     = :fixnum
+  TYPE_MAP[TYPE_FLOAT]      = :float
+  TYPE_MAP[TYPE_HASH]       = :hash
+  TYPE_MAP[TYPE_HASH_DEF]   = :hash_default
+  TYPE_MAP[TYPE_IVAR]       = :instance_variables
+  TYPE_MAP[TYPE_LINK]       = :link
+  TYPE_MAP[TYPE_MODULE]     = :module
+  TYPE_MAP[TYPE_MODULE_OLD] = :module_old
+  TYPE_MAP[TYPE_NIL]        = :nil
+  TYPE_MAP[TYPE_OBJECT]     = :object
+  TYPE_MAP[TYPE_REGEXP]     = :regexp
+  TYPE_MAP[TYPE_STRING]     = :string
+  TYPE_MAP[TYPE_STRUCT]     = :struct
+  TYPE_MAP[TYPE_SYMBOL]     = :symbol
+  TYPE_MAP[TYPE_SYMLINK]    = :symbol_link
+  TYPE_MAP[TYPE_TRUE]       = :true
+  TYPE_MAP[TYPE_UCLASS]     = :user_class
+  TYPE_MAP[TYPE_USERDEF]    = :user_defined
+  TYPE_MAP[TYPE_USRMARSHAL] = :user_marshal
+
+  ##
+  # Creates a Tokenizer for a Marshal stream in String +stream+.
+
+  def initialize stream
+    @byte_array = stream.bytes.to_a
+    @consumed   = 2
+    @state      = [:any]
+    @stream     = stream
+    @stream.force_encoding Encoding::BINARY
+  end
+
+  ##
+  # Consumes one byte from the marshal stream
+
+  def byte
+    raise Marshal::Structure::EndOfMarshal.new(@consumed, 1) if
+      @consumed >= @byte_array.size
+
+    data = @byte_array[@consumed]
+    @consumed += 1
+
+    data
+  end
+
+  ##
+  # Consumes +count+ bytes from the marshal stream as an Array of bytes
+
+  def byte_array count
+    bytes(count).bytes.to_a
+  end
+
+  ##
+  # Consumes a sequence of bytes from the marshal stream based on the next
+  # integer
+
+  def byte_sequence
+    size = long
+    bytes size
+  end
+
+  ##
+  # Consumes +count+ from the marshal stream
+
+  def bytes count
+    raise Marshal::Structure::EndOfMarshal.new(@consumed, count) if
+      @consumed + count > @stream.size
+
+    data = @stream[@consumed, count]
+    @consumed += count
+    data
+  end
+
+  ##
+  # Consumes one byte from the marshal stream and returns a character
+
+  def character
+    byte.chr
+  end
+
+  ##
+  # Checks if the stream starts with a compatible marshal version
+
+  def check_version
+    major = @stream[0].ord
+    minor = @stream[1].ord
+
+    return if major == MAJOR_VERSION and minor <= MINOR_VERSION
+
+    raise TypeError, "incompatible marshal file format (can't be read)\n\tformat version #{MAJOR_VERSION}.#{MINOR_VERSION} required; #{major}.#{minor} given"
+  end
+
+  ##
+  # Decodes a stored C long
+
+  def long
+    c = byte
+
+    return 0 if c == 0
+
+    # convert to signed integer
+    c = (c ^ 0x80) - 0x80
+
+    if c > 0 then
+      return c - 5 if 4 < c
+
+      x = 0
+
+      c.times do |i|
+        x |= byte << (8 * i)
+      end
+      
+      x
+    else
+      return c + 5 if c < -4
+
+      x = -1
+
+      (-c).times do |i|
+        factor = 8 * i
+        x &= ~(0xff << factor)
+        x |= byte << factor
+      end
+
+      x
+    end
+  end
+
+  ##
+  # Attempts to retrieve the next token from the stream.  You may need to call
+  # next_token twice to receive a token as the current token may be
+  # incomplete.
+
+  def next_token # :nodoc:
+    current_state = @state.pop
+
+    case current_state
+    when :any                         then tokenize_any
+    when :array                       then tokenize_array
+    when :bignum                      then tokenize_bignum
+    when :byte                        then byte
+    when :bytes,
+         :class, :module, :module_old,
+         :float, :string, :symbol     then byte_sequence
+    when :data                        then tokenize_data
+    when :extended                    then tokenize_extended
+    when :fixnum, :link, :symbol_link then long
+    when :hash, :pairs                then tokenize_pairs
+    when :hash_default                then tokenize_hash_default
+    when :instance_variables          then tokenize_instance_variables
+    when :object                      then tokenize_object
+    when :regexp                      then tokenize_regexp
+    when :struct                      then tokenize_struct
+    when :sym                         then tokenize_sym
+    when :user_class                  then tokenize_user_class
+    when :user_defined                then tokenize_user_defined
+    when :user_marshal                then tokenize_user_marshal
+    else
+      raise Marshal::Structure::Error,
+            "bug: unknown state #{current_state.inspect}"
+    end
+  end
+
+  ##
+  # Returns an Enumerator that will tokenize the Marshal stream.
+
+  def tokens
+    check_version
+
+    Enumerator.new do |yielder|
+      until @state.empty? do
+        token = next_token
+
+        yielder << token if token
+      end
+    end
+  end
+
+  def tokenize_any # :nodoc:
+    item_type = TYPE_MAP[character]
+
+    @state.push item_type unless [:nil, :true, :false].include? item_type
+
+    item_type
+  end
+
+  def tokenize_array # :nodoc:
+    size = long
+
+    @state.concat Array.new(size, :any)
+
+    size
+  end
+
+  def tokenize_bignum # :nodoc:
+    sign = byte == 45 ? -1 : 1
+    size = long * 2
+
+    result = 0
+
+    bytes = byte_array size
+
+    bytes.each_with_index do |byte, exp|
+      result += (byte * 2**(exp*8))
+    end
+
+    sign * result
+  end
+
+  def tokenize_data # :nodoc:
+    @state.push :any
+    @state.push :sym
+
+    next_token
+  end
+
+  alias tokenize_extended tokenize_data # :nodoc:
+
+  def tokenize_hash_default # :nodoc:
+    size = long
+
+    @state.push :any
+    @state.push size * 2 if size > 0
+
+    size
+  end
+
+  def tokenize_instance_variables # :nodoc:
+    @state.push :pairs
+    @state.push :any
+
+    next_token
+  end
+
+  ##
+  # For multipart objects like arrays and hashes a count of items is pushed
+  # onto the stack.  This method re-pushes an :any onto the stack until the
+  # correct number of tokens have been created from the stream.
+
+  def tokenize_next_any current_state # :nodoc:
+    next_state = current_state - 1
+    @state.push next_state if current_state > 0
+    @state.push :any
+
+    next_token
+  end
+
+  def tokenize_object # :nodoc:
+    @state.push :pairs
+    @state.push :sym
+
+    next_token
+  end
+
+  def tokenize_pairs # :nodoc:
+    size = long
+
+    @state.concat Array.new(size * 2, :any)
+
+    size
+  end
+
+  def tokenize_regexp # :nodoc:
+    @state.push :byte
+
+    byte_sequence
+  end
+
+  def tokenize_struct # :nodoc:
+    @state.push :pairs
+    @state.push :sym
+
+    next_token
+  end
+
+  def tokenize_sym # :nodoc:
+    item_type = TYPE_MAP[character]
+
+    raise Marshal::Structure::Error,
+          "expected symbol type, got #{item_type.inspect}" unless
+      [:symbol, :symbol_link].include? item_type
+
+    @state.push item_type
+
+    item_type
+  end
+
+  alias tokenize_user_class tokenize_data # :nodoc:
+
+  def tokenize_user_defined # :nodoc:
+    @state.push :bytes
+    @state.push :sym
+
+    next_token
+  end
+
+  alias tokenize_user_marshal tokenize_data # :nodoc:
+
+end
+

data/test/test_marshal_structure.rb

@@ -1,170 +1,36 @@
-require 'minitest/autorun'
-require 'marshal/structure'
-require 'ben_string'
-require 'openssl'
-require 'pp'
+require 'marshal/structure/test_case'
 
-class OpenSSL::X509::Name
-  alias _dump_data to_a
+class TestMarshalStructure < Marshal::Structure::TestCase
 
-  def _load_data ary
-    ary.each do |entry|
-      add_entry(*entry)
+  def test_class_load
+    ary = %W[\x04 \x08 T]
+    def ary.getc
+      shift
     end
-  end
-end
-
-class B; end
-
-module C; end
-
-module E; end
-
-class M
-  def marshal_dump
-    'marshal_dump'
-  end
-
-  def marshal_load o
-  end
-end
-
-class U
-  def self._load str
-    new
-  end
-
-  def _dump limit
-    s = '_dump'
-    s.instance_variable_set :@ivar_on_dump_str, 'value on ivar on dump str'
-    s
-  end
-end
-
-S = Struct.new :f
-
-class TestMarshalStructure < MiniTest::Unit::TestCase
 
-  def mu_pp obj
-    s = ''
-    s = PP.pp obj, s
-    s.chomp
-  end
+    result = @MS.load ary
 
-  def setup
-    @MS = Marshal::Structure
+    assert_equal :true, result
   end
 
-  def test_construct
-    str =
-      "\004\b{\006:\006a[\031c\006Bm\006C\"\006d/\006e\000i\006" \
-     "f\0322.2999999999999998\000ff" \
-     "l+\n\000\000\000\000\000\000\000\000\001\0000TF}\000i\000" \
-     "S:\006S\006:\006fi\000o:\vObject\000@\017" \
-     "U:\006M\"\021marshal_dump" \
-     "Iu:\006U\n_dump\006" \
-     ":\026@ivar_on_dump_str\"\036value on ivar on dump str" \
-     ";\000e:\006Eo;\b\000" \
-     "I\"\025string with ivar\006:\v@value\"\017some value" \
-     "C:\016BenString\"\000"
-
-    structure = @MS.load str
-
-    expected = [
-      :hash,
-      0,
-      1,
-      [:symbol, 0, "a"],
-      [:array,
-        1,
-        20,
-        [:class, 2, "B"],
-        [:module, 3, "C"],
-        [:string, 4, "d"],
-        [:regexp, 5, "e", 0],
-        [:fixnum, 1],
-        [:float, 6, "2.2999999999999998\000ff"],
-        [:bignum, 7, 1, 10, 18446744073709551616],
-        :nil,
-        :true,
-        :false,
-        [:hash_default, 8, 0, [:fixnum, 0]],
-        [:struct, 9, [:symbol, 1, "S"], 1, [:symbol, 2, "f"], [:fixnum, 0]],
-        [:object, 10, [:symbol, 3, "Object"], [0]],
-        [:link, 10],
-        [:user_marshal, 11, [:symbol, 4, "M"], [:string, 12, "marshal_dump"]],
-        [:instance_variables,
-          [:user_defined, 13, [:symbol, 5, "U"], "_dump"],
-          1,
-          [:symbol, 6, "@ivar_on_dump_str"],
-          [:string, 14, "value on ivar on dump str"]],
-          [:symbol_link, 0],
-          [:extended, [:symbol, 7, "E"], [:object, 15, [:symbol_link, 3], [0]]],
-          [:instance_variables,
-            [:string, 16, "string with ivar"],
-            1,
-            [:symbol, 8, "@value"],
-            [:string, 17, "some value"]],
-            [:user_class, [:symbol, 9, "BenString"], [:string, 18, ""]]]]
-
-    assert_equal expected, structure
+  def test_count_allocations
+    assert_equal 1, @MS.new("\x04\x08[\x06T").count_allocations
   end
 
-  def test_construct_data
-    name = OpenSSL::X509::Name.parse 'CN=nobody/DC=example'
-    str = Marshal.dump name
-
-    expected = [
-      :data,
-      0,
-      [:symbol, 0, "OpenSSL::X509::Name"],
-      [:array,
-        1,
-        2,
-        [:array, 2, 3,
-          [:string, 3, "CN"],
-          [:string, 4, "nobody"],
-          [:fixnum, 12]],
-        [:array, 5, 3,
-          [:string, 6, "DC"],
-          [:string, 7, "example"],
-          [:fixnum, 22]]]]
-
-        assert_equal expected, @MS.load(str)
-  end
-
-  def test_construct_module_old
-    assert_equal [:module, 0, "M"], @MS.load("\x04\x08M\x06M")
+  def test_load
+    assert_equal [true], @MS.new("\x04\x08[\x06T").load
   end
 
-  def test_consume
-    ms = @MS.new "\x04\x08\x06M"
-
-    assert_equal "\x06M", ms.consume(2)
+  def test_structure
+    assert_equal [:array, 0, 1, :true], @MS.new("\x04\x08[\x06T").structure
   end
 
-  def test_consume_bytes
-    ms = @MS.new "\x04\x08\x06M"
-
-    assert_equal [6, 77], ms.consume_bytes(2)
-  end
-
-  def test_consume_byte
-    ms = @MS.new "\x04\x08M"
-
-    assert_equal 77, ms.consume_byte
-  end
-
-  def test_consume_character
-    ms = @MS.new "\x04\x08M"
-
-    assert_equal 'M', ms.consume_character
-  end
+  def test_token_stream
+    stream = @MS.new("\x04\x08[\x06T").token_stream
 
-  def test_get_byte_sequence
-    ms = @MS.new "\x04\x08\x06M"
+    assert_kind_of Enumerator, stream
 
-    assert_equal "M", ms.get_byte_sequence
+    assert_equal [:array, 1, :true], stream.to_a
   end
 
 end