rlp 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3195187f64cf0bc3c2d9499028b45d66a19591f2
-  data.tar.gz: 1d20e7ef25821a865001a5e59237418749e87aa1
+  metadata.gz: a740accf9d4e050d92d280aacd38699feffb6081
+  data.tar.gz: fdae1f3979c78452f141062f95b3254f9aac4220
 SHA512:
-  metadata.gz: 1c1a5fcb71deda8629cdc4806499cdc050b4900af1ae65e67179e8245ce263db67f89bbf8e3fcc29bdb39c8168b688490a77591a7429d7851ae921e6f3398ce3
-  data.tar.gz: 7bf8bc70835a72adbc41f012c21bed66fee6afdaae78a81fd95a65591b6f6615ba720612fec287e6980b053aa793f33c8c2d6a564b2c23e506aa84230210487e
+  metadata.gz: 5b18bb3dd56837f948f44e7797f5497e3140dedfb98d50698c53470a351db879bacce83b4861c0d1d638f2ade563bcc2450ca60cca8fc372459122a0a45a5c3d
+  data.tar.gz: b143e3ef2289684e754c74406581126c8734845c274f04e8a952e17e65147eabdfde80e8de7e4f8fc801f916dcef024a4ba68597a03c37586f4c158e50a6fb61

data/README.md

@@ -1,3 +1,23 @@
 # ruby-rlp
 
 A Ruby implementation of Ethereum's Recursive Length Prefix encoding (RLP). You can find the specification of the standard in the [Ethereum wiki](https://github.com/ethereum/wiki/wiki/RLP).
+
+## Benchmark
+
+```
+ruby-rlp $ ruby -v -Ilib test/speed.py
+ruby 2.2.2p95 (2015-04-13 revision 50295) [x86_64-linux]
+Block serializations / sec: 2318.21
+Block deserializations / sec: 1704.61
+TX serializations / sec: 30461.76
+TX deserializations / sec: 21378.70
+
+pyrlp $ python -V
+Python 2.7.11
+
+pyrlp $ PYTHONPATH=. python tests/speed.py
+Block serializations / sec: 1225.00
+Block deserializations / sec: 1162.01
+TX serializations / sec: 16468.41
+TX deserializations / sec: 14517.31
+```

data/lib/rlp.rb

@@ -7,9 +7,13 @@ require 'rlp/sedes'
 require 'rlp/encode'
 require 'rlp/decode'
 
+require 'rlp/decode_lazy'
+require 'rlp/lazy_list'
+
 module RLP
   include Encode
   include Decode
+  include DecodeLazy
 
   extend self
 end

data/lib/rlp/decode.rb

@@ -4,10 +4,31 @@ module RLP
     include Error
     include Utils
 
-    def decode(rlp, options={})
-      sedes = options.delete(:sedes)
-      strict = options.delete(:strict) {|k| true }
-
+    ##
+    # Decode an RLP encoded object.
+    #
+    # If the deserialized result `obj` has an attribute `_cached_rlp` (e.g. if
+    # `sedes` is a subclass of {RLP::Sedes::Serializable}), it will be set to
+    # `rlp`, which will improve performance on subsequent {RLP::Encode#encode}
+    # calls. Bear in mind however that `obj` needs to make sure that this value
+    # is updated whenever one of its fields changes or prevent such changes
+    # entirely ({RLP::Sedes::Serializable} does the latter).
+    #
+    # @param sedes [#deserialize] an object implementing a function
+    #   `deserialize(code)` which will be applied after decoding, or `nil` if
+    #   no deserialization should be performed
+    # @param options [Hash] additional keyword arguments that will be passed to
+    #   the deserializer
+    # @param strict [Boolean] if false inputs that are longer than necessary
+    #   don't cause an exception
+    #
+    # @return [Object] the decoded and maybe deserialized object
+    #
+    # @raise [RLP::Error::DecodingError] if the input string does not end after
+    #   the root item and `strict` is true
+    # @raise [RLP::Error::DeserializationError] if the deserialization fails
+    #
+    def decode(rlp, sedes: nil, strict: true, options: {})
       rlp = str_to_bytes(rlp)
 
       begin
@@ -19,8 +40,16 @@ module RLP
       raise DecodingError.new("RLP string ends with #{rlp.size - next_start} superfluous bytes", rlp) if next_start != rlp.size && strict
 
       if sedes
-        obj = sedes.deserialize(item) # TODO: (options)
-        #TODO: cache flow
+        # FIXME: lazy man's kwargs
+        obj = sedes.is_a?(Sedes::Serializable) ?
+          sedes.deserialize(item, **options) :
+          sedes.deserialize(item)
+
+        if obj.respond_to?(:_cached_rlp)
+          obj._cached_rlp = rlp
+          raise "RLP::Sedes::Serializable object must be immutable after decode" if obj.is_a?(Sedes::Serializable) && obj.mutable?
+        end
+
         obj
       else
         item

data/lib/rlp/decode_lazy.rb

@@ -0,0 +1,103 @@
+module RLP
+  module DecodeLazy
+    include Decode
+
+    ##
+    # Decode an RLP encoded object in a lazy fashion.
+    #
+    # If the encoded object is a byte string, this function acts similar to
+    # {RLP::Decode#decode}. If it is a list however, a {LazyList} is returned
+    # instead. This object will decode the string lazily, avoiding both
+    # horizontal and vertical traversing as much as possible.
+    #
+    # The way `sedes` is applied depends on the decoded object: If it is a
+    # string `sedes` deserializes it as a whole; if it is a list, each element
+    # is deserialized individually. In both cases, `sedes_options` are passed
+    # on. Note that, if a deserializer is used, only "horizontal" but not
+    # "vertical lazyness" can be preserved.
+    #
+    # @param rlp [String] the RLP string to decode
+    # @param sedes [Object] an object implementing a method `deserialize(code)`
+    #   which is used as described above, or `nil` if no deserialization should
+    #   be performed
+    # @param sedes_options [Hash] additional keyword arguments that will be
+    #   passed to the deserializers
+    #
+    # @return [Object] either the already decoded and deserialized object (if
+    #   encoded as a string) or an instance of {RLP::LazyList}
+    #
+    def decode_lazy(rlp, sedes: nil, sedes_options: {})
+      item, next_start = consume_item_lazy(rlp, 0)
+
+      raise DecodingError.new("RLP length prefix announced wrong length", rlp) if next_start != rlp.size
+
+      if item.instance_of?(LazyList)
+        item.sedes = sedes
+        item.sedes_options = sedes_options
+        item
+      elsif sedes
+        # FIXME: lazy man's kwargs
+        sedes_options.empty? ?
+          sedes.deserialize(item) :
+          sedes.deserialize(item, **sedes_options)
+      else
+        item
+      end
+    end
+
+    ##
+    # Read an item from an RLP string lazily.
+    #
+    # If the length prefix announces a string, the string is read; if it
+    # announces a list, a {LazyList} is created.
+    #
+    # @param rlp [String] the rlp string to read from
+    # @param start [Integer] the position at which to start reading
+    #
+    # @return [Array] A pair `[item, next_start]` where `item` is the read
+    #   string or a {LazyList} and `next_start` is the position of the first
+    #   unprocessed byte
+    #
+    def consume_item_lazy(rlp, start)
+      t, l, s = consume_length_prefix(rlp, start)
+      if t == :str
+        consume_payload(rlp, s, :str, l)
+      elsif t == :list
+        [LazyList.new(rlp, s, s+l), s+l]
+      else
+        raise "Invalid item type: #{t}"
+      end
+    end
+
+    ##
+    # Get a specific element from an rlp encoded nested list.
+    #
+    # This method uses {RLP::DecodeLazy#decode_lazy} and, thus, decodes only
+    # the necessary parts of the string.
+    #
+    # @example Usage
+    #   rlpdata = RLP.encode([1, 2, [3, [4, 5]]])
+    #   RLP.peek(rlpdata, 0, sedes: RLP::Sedes.big_endian_int) # => 1
+    #   RLP.peek(rlpdata, [2, 0], sedes: RLP::Sedes.big_endian_int) # => 3
+    #
+    # @param rlp [String] the rlp string
+    # @param index [Integer, Array] the index of the element to peek at (can be
+    #   a list for nested data)
+    # @param sedes [#deserialize] a sedes used to deserialize the peeked at
+    #   object, or `nil` if no deserialization should be performed
+    #
+    # @raise [IndexError] if `index` is invalid (out of range or too many levels)
+    #
+    def peek(rlp, index, sedes: nil)
+      ll = decode_lazy(rlp)
+      index = Array(index)
+
+      index.each do |i|
+        raise IndexError, "Too many indices given" if primitive?(ll)
+        ll = ll.fetch(i)
+      end
+
+      sedes ? sedes.deserialize(ll) : ll
+    end
+  end
+end

data/lib/rlp/encode.rb

@@ -39,7 +39,9 @@ module RLP
     # @raise [RLP::SerializationError] if the serialization fails
     #
     def encode(obj, sedes: nil, infer_serializer: true, cache: false)
-      # TODO: cache flow
+      return obj._cached_rlp if obj.is_a?(Sedes::Serializable) && obj._cached_rlp && sedes.nil?
+
+      really_cache = obj.is_a?(Sedes::Serializable) && sedes.nil? && cache
 
       if sedes
         item = sedes.serialize(obj)
@@ -50,6 +52,12 @@ module RLP
       end
 
       result = encode_raw(item)
+
+      if really_cache
+        obj._cached_rlp = result
+        obj.make_immutable!
+      end
+
       result
     end
 

data/lib/rlp/lazy_list.rb

@@ -0,0 +1,86 @@
+module RLP
+##
+  # A RLP encoded list which decodes itself when necessary.
+  #
+  # Both indexing with positive indices and iterating are supported. Getting
+  # the length is possible as well but requires full horizontal encoding.
+  #
+  class LazyList
+    include Enumerable
+    include DecodeLazy
+
+    attr_accessor :sedes, :sedes_options
+
+    ##
+    # @param rlp [String] the rlp string in which the list is encoded
+    # @param start [Integer] the position of the first payload byte of the
+    #   encoded list
+    # @param next_start [Integer] the position of the last payload byte of the
+    #   encoded list
+    # @param sedes [Object] a sedes object which deserializes each element of the
+    #   list, or `nil` for on deserialization
+    # @param sedes_options [Hash] keyword arguments which will be passed on to
+    #   the deserializer
+    #
+    def initialize(rlp, start, next_start, sedes: nil, sedes_options: nil)
+      @rlp = rlp
+      @start = start
+      @next_start = next_start
+      @index = start
+      @elements = []
+      @size = nil
+      @sedes = sedes
+      @sedes_options = sedes_options
+    end
+
+    def next_item
+      if @index == @next_start
+        @size = @elements.size
+        raise StopIteration
+      elsif @index < @next_start
+        item, @index = consume_item_lazy @rlp, @index
+
+        if @sedes
+          # FIXME: lazy man's kwargs
+          item = @sedes_options.empty? ?
+                   @sedes.deserialize(item) :
+                   @sedes.deserialize(item, **@sedes_options)
+        end
+
+        @elements.push item
+        item
+      else
+        raise "Assertion failed: index cannot be larger than next start"
+      end
+    end
+
+    def each(&block)
+      @elements.each(&block)
+      loop { block.call(next_item) }
+    end
+
+    def [](i)
+      fetch(i, nil)
+    end
+
+    def fetch(*args)
+      i = args[0]
+
+      loop do
+        raise StopIteration if @elements.size > i
+        next_item
+      end
+
+      @elements.fetch(*args)
+    end
+
+    def size
+      unless @size
+        loop { next_item }
+      end
+      @size
+    end
+    alias :length :size
+
+  end
+end

data/lib/rlp/sedes/binary.rb

@@ -6,6 +6,10 @@ module RLP
       Infinity = 1.0 / 0.0
 
       class <<self
+        def fixed_length(l, allow_empty: false)
+          new(min_length: l, max_length: l, allow_empty: allow_empty)
+        end
+
         def valid_type?(obj)
           obj.instance_of?(String)
         end

data/lib/rlp/sedes/serializable.rb

@@ -76,6 +76,8 @@ module RLP
         end
       end
 
+      attr_accessor :_cached_rlp
+
       def initialize(*args)
         serializable_initialize(*args)
       end
@@ -101,27 +103,36 @@ module RLP
         raise TypeError, "Not all fields initialized" unless field_set.size == 0
       end
 
-      def ==(other)
-        return false unless other.class.respond_to?(:serialize)
-        self.class.serialize(self) == other.class.serialize(other)
-      end
-
       def _set_field(field, value)
         unless instance_variable_defined?(:@_mutable)
           @_mutable = true
         end
 
-        if _mutable? || !self.class.serializable_fields.has_key?(field)
+        if mutable? || !self.class.serializable_fields.has_key?(field)
           instance_variable_set :"@#{field}", value
         else
           raise ArgumentError, "Tried to mutate immutable object"
         end
       end
 
-      def _mutable?
+      def ==(other)
+        return false unless other.class.respond_to?(:serialize)
+        self.class.serialize(self) == other.class.serialize(other)
+      end
+
+      def mutable?
         @_mutable
       end
 
+      def make_immutable!
+        @_mutable = true
+        self.class.serializable_fields.keys.each do |field|
+          ::RLP::Utils.make_immutable! send(field)
+        end
+
+        @_mutable = false
+        self
+      end
     end
   end
 end

data/lib/rlp/utils.rb

@@ -1,5 +1,30 @@
 module RLP
   module Utils
+    class <<self
+      ##
+      # Do your best to make `obj` as immutable as possible.
+      #
+      # If `obj` is a list, apply this function recursively to all elements and
+      # return a new list containing them. If `obj` is an instance of
+      # {RLP::Sedes::Serializable}, apply this function to its fields, and set
+      # `@_mutable` to `false`. If `obj` is neither of the above, just return
+      # `obj`.
+      #
+      # @return [Object] `obj` after making it immutable
+      #
+      def make_immutable!(obj)
+        if obj.is_a?(Sedes::Serializable)
+          obj.make_immutable!
+        elsif list?(obj)
+          obj.map {|e| make_immutable!(e) }
+        else
+          obj
+        end
+      end
+    end
+
+    extend self
+
     def primitive?(item)
       item.instance_of?(String)
     end

data/lib/rlp/version.rb

@@ -1,3 +1,3 @@
 module RLP
-  VERSION = '0.3.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rlp
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jan Xie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-25 00:00:00.000000000 Z
+date: 2016-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -66,8 +66,10 @@ files:
 - lib/rlp/constant.rb
 - lib/rlp/data.rb
 - lib/rlp/decode.rb
+- lib/rlp/decode_lazy.rb
 - lib/rlp/encode.rb
 - lib/rlp/error.rb
+- lib/rlp/lazy_list.rb
 - lib/rlp/sedes.rb
 - lib/rlp/sedes/big_endian_int.rb
 - lib/rlp/sedes/binary.rb