marshal-structure 1.1.1 → 2.0

data/lib/marshal/structure/allocation_counter.rb

@@ -0,0 +1,172 @@
+##
+# Counts allocations necessary to load the stream.  The number of allocations
+# may be less as symbols (e.g. for object instance variables) may already
+# exist.
+#
+# Allocation counts are determined as follows:
+#
+# * References to classes or modules are not counted.  They either already
+#   exist or cause an ArgumentError upon load.
+# * true, false, nil and Fixnum are not counted as they are all singletons.
+# * Symbols count as one allocation even if they may already exist.  (Marshal
+#   automatically compresses duplicate mentions of symbols and objects, so
+#   they will only be counted once).
+# * Other objects are counted as one allocation.
+
+class Marshal::Structure::AllocationCounter
+
+  ##
+  # Creates a new AllocationCounter for +tokens+
+
+  def initialize tokens
+    @tokens = tokens
+  end
+
+  ##
+  # Counts objects allocated from the stream.
+
+  def count
+    token = @tokens.next
+
+    case token
+    when :nil, :true, :false          then 0
+    when :array                       then count_array
+    when :bignum                      then count_bignum
+    when :class, :module, :module_old then count_class
+    when :data                        then count_data
+    when :extended                    then count_extended
+    when :fixnum, :link, :symbol_link then @tokens.next; 0
+    when :float                       then count_float
+    when :hash                        then count_hash
+    when :hash_default                then count_hash_default
+    when :object                      then count_object
+    when :regexp                      then count_regexp
+    when :string                      then count_string
+    when :struct                      then count_struct
+    when :symbol                      then count_symbol
+    when :user_class                  then count_extended
+    when :user_defined                then count_user_defined
+    when :user_marshal                then count_user_marshal
+    when :instance_variables          then count + count_instance_variables
+    else
+      raise Marshal::Structure::Error, "bug: unknown token #{token.inspect}"
+    end
+  rescue Marshal::Structure::EndOfMarshal
+    raise ArgumentError, 'marshal data too short'
+  end
+
+  def count_array # :nodoc:
+    allocations = 1
+
+    @tokens.next.times do
+      allocations += count
+    end
+
+    allocations
+  end
+
+  def count_bignum # :nodoc:
+    @tokens.next
+
+    1
+  end
+
+  def count_class # :nodoc:
+    @tokens.next
+
+    0
+  end
+
+  def count_data # :nodoc:
+    get_symbol
+
+    1 + count
+  end
+
+  def count_extended # :nodoc:
+    get_symbol
+
+    count
+  end
+
+  alias count_float count_bignum # :nodoc:
+
+  def count_hash # :nodoc:
+    allocations = 1
+
+    @tokens.next.times do
+      allocations += count
+      allocations += count
+    end
+
+    allocations
+  end
+
+  def count_hash_default # :nodoc:
+    count_hash + count
+  end
+
+  def count_instance_variables # :nodoc:
+    allocations = 0
+
+    @tokens.next.times do
+      allocations += get_symbol
+      allocations += count
+    end
+
+    allocations
+  end
+
+  def count_object # :nodoc:
+    get_symbol + count_instance_variables
+  end
+
+  def count_regexp # :nodoc:
+    @tokens.next
+    @tokens.next
+
+    1
+  end
+
+  alias count_string count_bignum # :nodoc:
+
+  def count_struct # :nodoc:
+    allocations = 1
+    
+    get_symbol
+
+    @tokens.next.times do
+      allocations += get_symbol
+      allocations += count
+    end
+
+    allocations
+  end
+
+  alias count_symbol count_bignum
+
+  def count_user_defined # :nodoc:
+    allocations = get_symbol + 1
+
+    @tokens.next
+
+    allocations
+  end
+
+  def count_user_marshal # :nodoc:
+    get_symbol + count
+  end
+
+  def get_symbol # :nodoc:
+    token = @tokens.next
+
+    case token
+    when :symbol      then count_symbol
+    when :symbol_link then @tokens.next; 0
+    else
+      raise ArgumentError, "expected SYMBOL or SYMLINK, got #{token.inspect}"
+    end
+  end
+
+end
+

data/lib/marshal/structure/parser.rb

@@ -0,0 +1,276 @@
+##
+# Parses a tokenized Marshal stream into a structure that resembles how the
+# stream would be loaded.
+#
+# Marshal can contain references to previous objects.  These references are
+# included in the structure following referenceable items.  For example, a
+# recursive array:
+#
+#    a = []
+#    a << self
+#
+# Has the following Marshal stream:
+#
+#   "\x04\x08[\x06@\x00" # @\x00 is a link to the first Object in the stream
+#
+# And has the following Marshal structure:
+#
+#   [:array, 0, 1,
+#     [:link, 0]]
+#
+# The first item after +:array+, the +0+ is the object's stream ID.  The
+# +:link+ references this ID.
+
+class Marshal::Structure::Parser
+
+  ##
+  # Creates a new Parser using a token stream Enumerator +tokens+.
+
+  def initialize tokens
+    @tokens = tokens
+    @objects = -1
+    @symbols = -1
+  end
+
+  ##
+  # Creates a new object reference
+
+  def object_ref
+    @objects += 1
+  end
+
+  ##
+  # Creates the structure for the remaining stream.
+
+  def parse
+    token = @tokens.next
+
+    return token if [:nil, :true, :false].include? token
+
+    obj = [token]
+
+    rest =
+      case token
+      when :array                       then parse_array
+      when :bignum                      then parse_bignum
+      when :class, :module              then parse_class
+      when :data                        then parse_data
+      when :extended                    then parse_extended
+      when :fixnum, :link, :symbol_link then [@tokens.next]
+      when :float                       then parse_float
+      when :hash                        then parse_hash
+      when :hash_default                then parse_hash_def
+      when :object                      then parse_object
+      when :regexp                      then parse_regexp
+      when :string                      then parse_string
+      when :struct                      then parse_struct
+      when :symbol                      then parse_symbol
+      when :user_class                  then parse_extended
+      when :user_defined                then parse_user_defined
+      when :user_marshal                then parse_user_marshal
+      when :instance_variables          then
+        [parse].concat parse_instance_variables
+      when :module_old                  then
+        obj[0] = :module
+        parse_class
+      else
+        raise Marshal::Structure::Error, "bug: unknown token #{token.inspect}"
+      end
+
+    obj.concat rest
+  rescue Marshal::Structure::EndOfMarshal
+    raise ArgumentError, 'marshal data too short'
+  end
+
+  ##
+  # Creates the body of an +:array+ object
+
+  def parse_array
+    obj = [object_ref]
+
+    items = @tokens.next
+
+    obj << items
+
+    items.times do
+      obj << parse
+    end
+
+    obj
+  end
+
+  ##
+  # Creates the body of a +:bignum+ object
+
+  def parse_bignum
+    result = @tokens.next
+
+    [object_ref, result]
+  end
+
+  ##
+  # Creates the body of a +:class+ object
+
+  def parse_class
+    [object_ref, @tokens.next]
+  end
+
+  ##
+  # Creates the body of a wrapped C pointer object
+
+  def parse_data
+    [object_ref, get_symbol, parse]
+  end
+
+  ##
+  # Creates the body of an extended object
+
+  def parse_extended
+    [get_symbol, parse]
+  end
+
+  ##
+  # Creates the body of a +:float+ object
+
+  def parse_float
+    float = @tokens.next
+
+    [object_ref, float]
+  end
+
+  ##
+  # Creates the body of a +:hash+ object
+
+  def parse_hash
+    obj = [object_ref]
+
+    pairs = @tokens.next
+    obj << pairs
+
+    pairs.times do
+      obj << parse
+      obj << parse
+    end
+
+    obj
+  end
+
+  ##
+  # Creates the body of a +:hash_def+ object
+
+  def parse_hash_def
+    ref, hash = parse_hash
+
+    [ref, hash, parse]
+  end
+
+  ##
+  # Instance variables contain an object followed by a count of instance
+  # variables and their contents
+
+  def parse_instance_variables
+    instance_variables = []
+
+    pairs = @tokens.next
+    instance_variables << pairs
+
+    pairs.times do
+      instance_variables << get_symbol
+      instance_variables << parse
+    end
+
+    instance_variables
+  end
+
+  ##
+  # Creates an Object
+
+  def parse_object
+    [object_ref, get_symbol, parse_instance_variables]
+  end
+
+  ##
+  # Creates a Regexp
+
+  def parse_regexp
+    [object_ref, @tokens.next, @tokens.next]
+  end
+
+  ##
+  # Creates a String
+
+  def parse_string
+    [object_ref, @tokens.next]
+  end
+
+  ##
+  # Creates a Struct
+
+  def parse_struct
+    obj = [object_ref, get_symbol]
+
+    members = @tokens.next
+    obj << members
+
+    members.times do
+      obj << get_symbol
+      obj << parse
+    end
+
+    obj
+  end
+
+  ##
+  # Creates a Symbol
+
+  def parse_symbol
+    sym = @tokens.next
+
+    [symbol_ref, sym]
+  end
+
+  ##
+  # Creates an object saved by _dump
+
+  def parse_user_defined
+    name = get_symbol
+
+    data = @tokens.next
+
+    [object_ref, name, data]
+  end
+
+  ##
+  # Creates an object saved by marshal_dump
+
+  def parse_user_marshal
+    name = get_symbol
+
+    [object_ref, name, parse]
+  end
+
+  ##
+  # Creates a new symbol reference
+
+  def symbol_ref
+    @symbols += 1
+  end
+
+  ##
+  # Constructs a Symbol from the token stream
+
+  def get_symbol
+    token = @tokens.next
+
+    case token
+    when :symbol then
+      [:symbol, *parse_symbol]
+    when :symbol_link then
+      [:symbol_link, @tokens.next]
+    else
+      raise ArgumentError, "expected SYMBOL or SYMLINK, got #{token.inspect}"
+    end
+  end
+
+end
+

data/lib/marshal/structure/test_case.rb

@@ -0,0 +1,95 @@
+require 'minitest/autorun'
+require 'marshal/structure'
+require 'ben_string'
+require 'openssl'
+require 'pp'
+
+# :stopdoc:
+
+class OpenSSL::X509::Name
+  alias _dump_data to_a
+
+  def _load_data ary
+    ary.each do |entry|
+      add_entry(*entry)
+    end
+  end
+end
+
+class B; end
+
+module C; end
+
+module E; end
+
+class M
+  def marshal_dump
+    'marshal_dump'
+  end
+
+  def marshal_load o
+  end
+end
+
+class U
+  def self._load str
+    new
+  end
+
+  def _dump limit
+    s = '_dump'
+    s.instance_variable_set :@ivar_on_dump_str, 'value on ivar on dump str'
+    s
+  end
+end
+
+S = Struct.new :f
+
+# :startdoc:
+
+##
+# A TestCase for writing tests for Marshal::Structure and alternative parsers
+# of Marshal streams.
+
+class Marshal::Structure::TestCase < MiniTest::Unit::TestCase
+
+  ##
+  # A Marshal stream with (almost) every type in it.  The notable absence is
+  # of a Data type.
+
+  EVERYTHING =
+    "\004\b{\006:\006a[\031c\006Bm\006C\"\006d/\006e\000i\006" \
+    "f\0322.2999999999999998\000ff" \
+    "l+\n\000\000\000\000\000\000\000\000\001\0000TF}\000i\000" \
+    "S:\006S\006:\006fi\000o:\vObject\000@\017" \
+    "U:\006M\"\021marshal_dump" \
+    "Iu:\006U\n_dump\006" \
+    ":\026@ivar_on_dump_str\"\036value on ivar on dump str" \
+    ";\000e:\006Eo;\b\000" \
+    "I\"\025string with ivar\006:\v@value\"\017some value" \
+    "C:\016BenString\"\000"
+
+  ##
+  # Pretty-print minitest diff output
+
+  def mu_pp obj # :nodoc:
+    s = ''
+    s = PP.pp obj, s
+    s.chomp
+  end
+
+  ##
+  # Creates the following convenience namespace instance variables:
+  #
+  # @MS:: Marshal::Structure
+  # @MSP:: Marshal::Structure::Tokenizer
+  # @MST:: Marshal::Structure::Parser
+
+  def setup
+    @MS  = Marshal::Structure
+    @MSP = Marshal::Structure::Parser
+    @MST = Marshal::Structure::Tokenizer
+  end
+
+end
+