python-pickle 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea86f43e4b83bacac94635b88e2c9deeb6eb4aaa553f1770e4943554389422fc
-  data.tar.gz: 973327ecbd8f2c1df7d0ac10f37d76628745e00da9317ab655a9f12644fef441
+  metadata.gz: c8fe6820a4b047c2e2eb027c3dbe88b5b8d7e51d201864e9e7b726cb566d687d
+  data.tar.gz: b1b66aa3811a277b63f643eb4c2968bab268ab7c22aa061713c1555f8d66162b
 SHA512:
-  metadata.gz: 8b95a73ff2ebfba7c9ca97648df7c7b3b6be14df61e0e6f4943e3e263a5f300696e79d8cf57aacc7bd9d572bcf3c00a5ec88162e1dd8cc627617afe01414499d
-  data.tar.gz: 73a9a106cfa346d3d74835cf9369732f041fdb2faafdb8a030f1e9ec6b58831e8b7c8aee3a05bec687d11d2d4ea9d48a00d4fcf164f0c686eb84c1248b15f056
+  metadata.gz: b44f4bddc1cfcbdfc4d7fda78a30b9982c268701109bbb92da050c1c226a006859f3a9dacd872c9362bb8c237723b7d8401dcb28626ad9a2e49b83af3c8c32d1
+  data.tar.gz: 8301c1d66a60a6cb7468f32a6f801d3b0ed699d771af7625eba034aafef762815201cb12b9484976248abe84cc32278647991e1cbb76745ebea84994e5ff2f2a

data/ChangeLog.md

@@ -1,3 +1,20 @@
+### 0.2.0 / 2023-02-19
+
+* Added missing support for deserializing Python `set` objects.
+* Added missing support for out-of-band buffers.
+* Added partial support for parsing the `PERSID` and `BINPERSID` instructions.
+* Added missing support for deserializing the `INST` and `OBJ` instructions.
+* Added missing support for deserializing the `EMPTY_SET`, `FROZENSET`, and
+  `ADDITEMS` Pickle instructions.
+* Added missing support for deserializing the `NEXT_BUFFER` and
+  `READONLY_BUFFER` Pickle instructions.
+* Map `__builtin__.set` and `builtins.set` to Ruby's `Set` class.
+
+### 0.1.0 / 2023-02-18
+
+* Changed {Python::Pickle.dump} to raise a `NotImplementedError` exception.
+* Fixed a typo in the method signature of {Python::Pickle.dump}.
+
 ### 0.1.0 / 2023-02-18
 
 * Initial release:

data/README.md

@@ -20,9 +20,12 @@ format.
 * Supports Pickle protocol 0, protocol 1, protocol 2, protocol 3, protocol 4,
   and protocol 5.
   * Can parse both Python 2 and Python 3 Pickled data.
-* Supports deserializing Python `tuple` and `bytearray` objects.
+* Supports deserializing Python `None`, `True`, `False`, `int`, `str`, `tuple`,
+  `set`, `list`, `bytearray`, and other objects.
+* Supports mapping Python extension codes to Ruby classes.
 * Supports mapping Python functions to Ruby methods.
 * Supports mapping Python classes to Ruby classes.
+* Supports out-of-band buffers.
 
 ## TODO
 

data/lib/python/pickle/deserializer.rb

@@ -2,63 +2,10 @@ require 'python/pickle/py_class'
 require 'python/pickle/py_object'
 require 'python/pickle/tuple'
 require 'python/pickle/byte_array'
+require 'python/pickle/instructions'
 require 'python/pickle/exceptions'
 
-require 'python/pickle/instructions/proto'
-require 'python/pickle/instructions/frame'
-require 'python/pickle/instructions/get'
-require 'python/pickle/instructions/bin_get'
-require 'python/pickle/instructions/long_bin_get'
-require 'python/pickle/instructions/mark'
-require 'python/pickle/instructions/pop_mark'
-require 'python/pickle/instructions/dup'
-require 'python/pickle/instructions/put'
-require 'python/pickle/instructions/bin_put'
-require 'python/pickle/instructions/pop'
-require 'python/pickle/instructions/memoize'
-require 'python/pickle/instructions/ext1'
-require 'python/pickle/instructions/ext2'
-require 'python/pickle/instructions/ext4'
-require 'python/pickle/instructions/none'
-require 'python/pickle/instructions/new_true'
-require 'python/pickle/instructions/new_false'
-require 'python/pickle/instructions/float'
-require 'python/pickle/instructions/bin_float'
-require 'python/pickle/instructions/int'
-require 'python/pickle/instructions/bin_int1'
-require 'python/pickle/instructions/long'
-require 'python/pickle/instructions/long1'
-require 'python/pickle/instructions/long4'
-require 'python/pickle/instructions/bin_bytes'
-require 'python/pickle/instructions/short_bin_bytes'
-require 'python/pickle/instructions/bin_bytes8'
-require 'python/pickle/instructions/string'
-require 'python/pickle/instructions/bin_string'
-require 'python/pickle/instructions/short_bin_string'
-require 'python/pickle/instructions/bin_unicode'
-require 'python/pickle/instructions/short_bin_unicode'
-require 'python/pickle/instructions/bin_unicode8'
-require 'python/pickle/instructions/byte_array8'
-require 'python/pickle/instructions/empty_list'
-require 'python/pickle/instructions/empty_tuple'
-require 'python/pickle/instructions/tuple'
-require 'python/pickle/instructions/empty_dict'
-require 'python/pickle/instructions/append'
-require 'python/pickle/instructions/appends'
-require 'python/pickle/instructions/list'
-require 'python/pickle/instructions/tuple1'
-require 'python/pickle/instructions/tuple2'
-require 'python/pickle/instructions/tuple3'
-require 'python/pickle/instructions/dict'
-require 'python/pickle/instructions/global'
-require 'python/pickle/instructions/stack_global'
-require 'python/pickle/instructions/new_obj'
-require 'python/pickle/instructions/new_obj_ex'
-require 'python/pickle/instructions/reduce'
-require 'python/pickle/instructions/build'
-require 'python/pickle/instructions/set_item'
-require 'python/pickle/instructions/set_items'
-require 'python/pickle/instructions/stop'
+require 'set'
 
 module Python
   module Pickle
@@ -94,6 +41,11 @@ module Python
       # @return [Hash{Integer => Object}]
       attr_reader :extensions
 
+      # An enumerable list of out-of-band buffers.
+      #
+      # @return [Enumerator, nil]
+      attr_reader :buffers
+
       # The Python `object` class.
       OBJECT_CLASS = PyClass.new('__builtins__','object')
 
@@ -107,7 +59,10 @@ module Python
       #   An optional mapping of custom Python constant names to Ruby classes
       #   or methods.
       #
-      def initialize(constants: nil, extensions: nil)
+      # @param [Enumerable, nil] buffers
+      #   An enumerable list of out-of-band buffers.
+      #
+      def initialize(constants: nil, extensions: nil, buffers: nil)
         @meta_stack = []
         @stack = []
         @memo  = []
@@ -120,12 +75,14 @@ module Python
 
           '__builtin__' => {
             'object'    => OBJECT_CLASS,
+            'set'       => Set,
             'bytearray' => ByteArray
           },
 
           # Python 3.x
           'builtins' => {
             'object'    => OBJECT_CLASS,
+            'set'       => Set,
             'bytearray' => ByteArray
           }
         }
@@ -133,6 +90,8 @@ module Python
 
         @extensions = {}
         @extensions.merge!(extensions) if extensions
+
+        @buffers = buffers.each if buffers
       end
 
       #
@@ -199,26 +158,33 @@ module Python
              Instructions::ShortBinUnicode,
              Instructions::BinUnicode8
           @stack.push(instruction.value)
-        when Instructions::ByteArray8   then execute_byte_array8(instruction)
-        when Instructions::EMPTY_LIST   then execute_empty_list
-        when Instructions::EMPTY_TUPLE  then execute_empty_tuple
-        when Instructions::TUPLE        then execute_tuple
-        when Instructions::EMPTY_DICT   then execute_empty_dict
-        when Instructions::APPEND       then execute_append
-        when Instructions::APPENDS      then execute_appends
-        when Instructions::LIST         then execute_list
-        when Instructions::TUPLE1       then execute_tuple1
-        when Instructions::TUPLE2       then execute_tuple2
-        when Instructions::TUPLE3       then execute_tuple3
-        when Instructions::DICT         then execute_dict
-        when Instructions::Global       then execute_global(instruction)
-        when Instructions::STACK_GLOBAL then execute_stack_global
-        when Instructions::NEWOBJ       then execute_newobj
-        when Instructions::NEWOBJ_EX    then execute_newobj_ex
-        when Instructions::REDUCE       then execute_reduce
-        when Instructions::BUILD        then execute_build
-        when Instructions::SETITEM      then execute_setitem
-        when Instructions::SETITEMS     then execute_setitems
+        when Instructions::ByteArray8      then execute_byte_array8(instruction)
+        when Instructions::EMPTY_LIST      then execute_empty_list
+        when Instructions::EMPTY_TUPLE     then execute_empty_tuple
+        when Instructions::TUPLE           then execute_tuple
+        when Instructions::EMPTY_DICT      then execute_empty_dict
+        when Instructions::EMPTY_SET       then execute_empty_set
+        when Instructions::FROZENSET       then execute_frozenset
+        when Instructions::APPEND          then execute_append
+        when Instructions::APPENDS         then execute_appends
+        when Instructions::ADDITEMS        then execute_additems
+        when Instructions::LIST            then execute_list
+        when Instructions::TUPLE1          then execute_tuple1
+        when Instructions::TUPLE2          then execute_tuple2
+        when Instructions::TUPLE3          then execute_tuple3
+        when Instructions::DICT            then execute_dict
+        when Instructions::Global          then execute_global(instruction)
+        when Instructions::STACK_GLOBAL    then execute_stack_global
+        when Instructions::Inst            then execute_inst(instruction)
+        when Instructions::OBJ             then execute_obj
+        when Instructions::NEWOBJ          then execute_newobj
+        when Instructions::NEWOBJ_EX       then execute_newobj_ex
+        when Instructions::REDUCE          then execute_reduce
+        when Instructions::BUILD           then execute_build
+        when Instructions::SETITEM         then execute_setitem
+        when Instructions::SETITEMS        then execute_setitems
+        when Instructions::NEXT_BUFFER     then execute_next_buffer
+        when Instructions::READONLY_BUFFER then execute_readonly_buffer
         when Instructions::STOP
           return :halt, @stack.pop
         else
@@ -361,6 +327,29 @@ module Python
         @stack.push({})
       end
 
+      #
+      # Executes an `EMPTY_SET` instruction.
+      #
+      # @since 0.2.0
+      #
+      def execute_empty_set
+        @stack.push(Set.new)
+      end
+
+      #
+      # Executes a `FROZENSET` instruction.
+      #
+      # @since 0.2.0
+      #
+      def execute_frozenset
+        items = pop_meta_stack
+
+        set = Set.new(items)
+        set.freeze
+
+        @stack.push(set)
+      end
+
       #
       # Executes an `APPEND` instruction.
       #
@@ -368,11 +357,11 @@ module Python
         item = @stack.pop
         list = @stack.last
 
-        unless list.kind_of?(Array)
+        unless (list.kind_of?(Array) || list.kind_of?(Set))
           raise(DeserializationError,"cannot append element #{item.inspect} onto a non-Array: #{list.inspect}")
         end
 
-        list.push(item)
+        list << item
       end
 
       #
@@ -382,11 +371,34 @@ module Python
         items = pop_meta_stack
         list  = @stack.last
 
-        unless list.kind_of?(Array)
+        case list
+        when Array
+          list.concat(items)
+        when Set
+          items.each do |item|
+            list << item
+          end
+        else
           raise(DeserializationError,"cannot append elements #{items.inspect} onto a non-Array: #{list.inspect}")
         end
+      end
+
+      #
+      # Executes a `ADDITEMS` instruction.
+      #
+      # @since 0.2.0
+      #
+      def execute_additems
+        items = pop_meta_stack
+        set   = @stack.last
+
+        unless set.kind_of?(Set)
+          raise(DeserializationError,"cannot add items #{items.inspect} to a non-Set object: #{set.inspect}")
+        end
 
-        list.concat(items)
+        items.each do |item|
+          set << item
+        end
       end
 
       #
@@ -500,6 +512,33 @@ module Python
         @stack.push(constant)
       end
 
+      #
+      # Executes an `INST` instruction.
+      #
+      # @since 0.2.0
+      #
+      def execute_inst(instruction)
+        namespace = instruction.namespace
+        name      = instruction.name
+        py_class  = resolve_constant(namespace,name)
+        args      = pop_meta_stack
+        py_object = py_class.new(*args)
+
+        @stack.push(py_object)
+      end
+
+      #
+      # Executes an `OBJ` instruction.
+      #
+      # @since 0.2.0
+      #
+      def execute_obj
+        py_class, *args = pop_meta_stack
+        py_object = py_class.new(*args)
+
+        @stack.push(py_object)
+      end
+
       #
       # Executes a `NEWOBJ` instruction.
       #
@@ -590,6 +629,29 @@ module Python
         end
       end
 
+      #
+      # Executes a `NEXT_BUFFER` instruction.
+      #
+      # @since 0.2.0
+      #
+      def execute_next_buffer
+        unless @buffers
+          raise(DeserializationError,"pickle stream includes a NEXT_BUFFER instruction, but no buffers were provided")
+        end
+
+        @stack.push(@buffers.next)
+      end
+
+      #
+      # Executes a `READONLY_BUFFER` instruction.
+      #
+      # @since 0.2.0
+      #
+      def execute_readonly_buffer
+        buffer = @stack.last
+        buffer.freeze
+      end
+
     end
   end
 end

data/lib/python/pickle/instructions/bin_persid.rb

@@ -0,0 +1,31 @@
+require 'python/pickle/instruction'
+require 'python/pickle/instructions/has_value'
+
+module Python
+  module Pickle
+    module Instructions
+      #
+      # Represents the `BINPERSID` instruction.
+      #
+      # @note introduced in protocol 0.
+      #
+      # @since 0.2.0
+      #
+      class BinPersID < Instruction
+
+        #
+        # Initializes the `BINPERSID` instruction.
+        #
+        def initialize
+          super(:BINPERSID)
+        end
+
+      end
+
+      # Represents the `BINPERSID` instruction.
+      #
+      # @since 0.2.0
+      BINPERSID = BinPersID.new
+    end
+  end
+end

data/lib/python/pickle/instructions/global.rb

@@ -1,59 +1,29 @@
 require 'python/pickle/instruction'
-require 'python/pickle/instructions/has_value'
+require 'python/pickle/instructions/has_namespace_and_name'
 
 module Python
   module Pickle
     module Instructions
+      #
+      # Represents the `GLOBAL` instruction.
+      #
+      # @note introduced in protocol 0.
+      #
       class Global < Instruction
 
-        # The global object namespace.
-        #
-        # @return [String]
-        attr_reader :namespace
-
-        # The global object name.
-        #
-        # @return [String]
-        attr_reader :name
+        include HasNamespaceAndName
 
         #
-        # Initializes the `GLOBAL` instruction.
+        # Initializes a `GLOBAL` instruction.
         #
         # @param [String] namespace
-        #   The namespace name for the global object.
+        #   The namespace name for the constant.
         #
         # @param [String] name
-        #   The name of the global object.
+        #   The name of the constant.
         #
         def initialize(namespace,name)
-          super(:GLOBAL)
-
-          @namespace = namespace
-          @name      = name
-        end
-
-        #
-        # Compares the `GLOBAL` instruction to another instruction.
-        #
-        # @param [Instruction] other
-        #   The other instruction to compare against.
-        #
-        # @return [Boolean]
-        #   Indicates whether the other instruction matches this one.
-        #
-        def ==(other)
-          super(other) && \
-            (@namespace == other.namespace) && \
-            (@name == other.name)
-        end
-
-        #
-        # Converts the `GLOBAL` instructions to a String.
-        #
-        # @return [String]
-        #
-        def to_s
-          "#{super} #{@namespace}.#{@name}"
+          super(:GLOBAL,namespace,name)
         end
 
       end

data/lib/python/pickle/instructions/has_namespace_and_name.rb

@@ -0,0 +1,61 @@
+module Python
+  module Pickle
+    module Instructions
+      module HasNamespaceAndName
+        # The constant's namespace.
+        #
+        # @return [String]
+        attr_reader :namespace
+
+        # The constant name.
+        #
+        # @return [String]
+        attr_reader :name
+
+        #
+        # Initializes the instruction.
+        #
+        # @param [Symbol] opcode
+        #   The instruction's opcode.
+        #
+        # @param [String] namespace
+        #   The namespace name for the constant.
+        #
+        # @param [String] name
+        #   The name of the constant.
+        #
+        def initialize(opcode,namespace,name)
+          super(opcode)
+
+          @namespace = namespace
+          @name      = name
+        end
+
+        #
+        # Compares the instruction to another instruction.
+        #
+        # @param [Instruction] other
+        #   The other instruction to compare against.
+        #
+        # @return [Boolean]
+        #   Indicates whether the other instruction matches this one.
+        #
+        def ==(other)
+          super(other) && \
+            (@namespace == other.namespace) && \
+            (@name == other.name)
+        end
+
+        #
+        # Converts the instructions to a String.
+        #
+        # @return [String]
+        #
+        def to_s
+          "#{super} #{@namespace}.#{@name}"
+        end
+
+      end
+    end
+  end
+end

data/lib/python/pickle/instructions/inst.rb

@@ -0,0 +1,34 @@
+require 'python/pickle/instruction'
+require 'python/pickle/instructions/has_namespace_and_name'
+
+module Python
+  module Pickle
+    module Instructions
+      #
+      # Represents the `INST` instruction.
+      #
+      # @note introduced in protocol 0.
+      #
+      # @since 0.2.0
+      #
+      class Inst < Instruction
+
+        include HasNamespaceAndName
+
+        #
+        # Initializes a `INST` instruction.
+        #
+        # @param [String] namespace
+        #   The namespace name for the constant.
+        #
+        # @param [String] name
+        #   The name of the constant.
+        #
+        def initialize(namespace,name)
+          super(:INST,namespace,name)
+        end
+
+      end
+    end
+  end
+end

data/lib/python/pickle/instructions/next_buffer.rb

@@ -6,7 +6,9 @@ module Python
       #
       # Represents a pickle `NEXT_BUFFER` instruction.
       #
-      # @note introduced in protocol 4.
+      # @note introduced in protocol 5.
+      #
+      # @since 0.2.0
       #
       class NextBuffer < Instruction
 
@@ -20,6 +22,8 @@ module Python
       end
 
       # The `NEXT_BUFFER` instruction.
+      #
+      # @since 0.2.0
       NEXT_BUFFER = NextBuffer.new
     end
   end

data/lib/python/pickle/instructions/obj.rb

@@ -0,0 +1,30 @@
+require 'python/pickle/instruction'
+
+module Python
+  module Pickle
+    module Instructions
+      #
+      # Represents a pickle `OBJ` instruction.
+      #
+      # @note introduced in protocol 0.
+      #
+      # @since 0.2.0
+      #
+      class Obj < Instruction
+
+        #
+        # Initializes the `OBJ` instruction.
+        #
+        def initialize
+          super(:OBJ)
+        end
+
+      end
+
+      # The `OBJ` instruction.
+      #
+      # @since 0.2.0
+      OBJ = Obj.new
+    end
+  end
+end

data/lib/python/pickle/instructions/persid.rb

@@ -0,0 +1,31 @@
+require 'python/pickle/instruction'
+require 'python/pickle/instructions/has_value'
+
+module Python
+  module Pickle
+    module Instructions
+      #
+      # Represents the `PERSID` instruction.
+      #
+      # @note introduced in protocol 0.
+      #
+      # @since 0.2.0
+      #
+      class PersID < Instruction
+
+        include HasValue
+
+        #
+        # Initializes the `PERSID` instruction.
+        #
+        # @param [String] value
+        #   The `PERSID` instruction's value.
+        #
+        def initialize(value)
+          super(:PERSID,value)
+        end
+
+      end
+    end
+  end
+end

data/lib/python/pickle/instructions/readonly_buffer.rb

@@ -8,6 +8,8 @@ module Python
       #
       # @note introduced in protocol 5.
       #
+      # @since 0.2.0
+      #
       class ReadonlyBuffer < Instruction
 
         #
@@ -20,6 +22,8 @@ module Python
       end
 
       # The `READONLY_BUFFER` instruction.
+      #
+      # @since 0.2.0
       READONLY_BUFFER = ReadonlyBuffer.new
     end
   end