python-pickle 0.1.0 → 0.2.0

data/spec/deserializer_spec.rb

@@ -2,6 +2,11 @@ require 'spec_helper'
 require 'python/pickle/deserializer'
 
 describe Python::Pickle::Deserializer do
+  module TestDeserializer
+    class MyClass
+    end
+  end
+
   describe "#initialize" do
     it "must initialize #meta_stack to an empty Array" do
       expect(subject.meta_stack).to eq([])
@@ -26,6 +31,10 @@ describe Python::Pickle::Deserializer do
         expect(subject.constants['__builtin__']['object']).to be(described_class::OBJECT_CLASS)
       end
 
+      it "must contain '__builtin__.set' for Python 2.x support" do
+        expect(subject.constants['__builtin__']['set']).to be(Set)
+      end
+
       it "must contain '__builtin__.bytearray' for Python 2.x support" do
         expect(subject.constants['__builtin__']['bytearray']).to be(Python::Pickle::ByteArray)
       end
@@ -34,6 +43,10 @@ describe Python::Pickle::Deserializer do
         expect(subject.constants['builtins']['object']).to be(described_class::OBJECT_CLASS)
       end
 
+      it "must contain 'builtins.set' for Python 2.x support" do
+        expect(subject.constants['builtins']['set']).to be(Set)
+      end
+
       it "must contain 'builtins.bytearray' for Python 2.x support" do
         expect(subject.constants['builtins']['bytearray']).to be(Python::Pickle::ByteArray)
       end
@@ -44,9 +57,75 @@ describe Python::Pickle::Deserializer do
     end
 
     context "when initialized with the `extensions:` keyword argument" do
+      let(:extensions) do
+        {
+          0x41 => Object.new,
+          0x42 => Object.new,
+          0x43 => Object.new
+        }
+      end
+
+      subject { described_class.new(extensions: extensions) }
+
+      it "must add the extensions: values to #extensions" do
+        expect(subject.extensions).to eq(extensions)
+      end
     end
 
     context "when initialized with the `constants:` keyword argument" do
+      let(:constants) do
+        {
+          '__main__' => {
+            'MyClass' => TestDeserializer::MyClass
+          }
+        }
+      end
+
+      subject { described_class.new(constants: constants) }
+
+      it "must merge the constants: keyword argument with the default constants" do
+        expect(subject.constants).to eq(
+          {
+            'copy_reg' => {
+              '_reconstructor' => subject.method(:copyreg_reconstructor)
+            },
+
+            '__builtin__' => {
+              'object'    => described_class::OBJECT_CLASS,
+              'set'       => Set,
+              'bytearray' => Python::Pickle::ByteArray
+            },
+
+            'builtins' => {
+              'object'    => described_class::OBJECT_CLASS,
+              'set'       => Set,
+              'bytearray' => Python::Pickle::ByteArray
+            },
+
+            '__main__' => {
+              'MyClass' => TestDeserializer::MyClass
+            }
+          }
+        )
+      end
+    end
+
+    context "when initialized with the `buffers:` keyword argument" do
+      let(:buffer1) { "hello world" }
+      let(:buffer2) { "foo bar"     }
+      let(:buffers) do
+        [
+          buffer1,
+          buffer2
+        ]
+      end
+
+      subject { described_class.new(buffers: buffers) }
+
+      it "must set #buffers to an Enumerator of the buffers" do
+        expect(subject.buffers).to be_kind_of(Enumerator)
+        expect(subject.buffers.to_a).to eq(buffers)
+      end
     end
   end
 
@@ -364,6 +443,37 @@ describe Python::Pickle::Deserializer do
       end
     end
 
+    context "when given a Python::Pickle::Instructions::EMPTY_SET" do
+      let(:instruction) { Python::Pickle::Instructions::EMPTY_SET }
+
+      before do
+        subject.execute(instruction)
+      end
+
+      it "must push an empty Set object onto the #stack" do
+        expect(subject.stack).to eq([ Set.new ])
+      end
+    end
+
+    context "when given a Python::Pickle::Instructions::FROZENSET" do
+      let(:instruction) { Python::Pickle::Instructions::FROZENSET }
+
+      before do
+        subject.meta_stack << []
+        subject.stack << 1 << 2 << 3
+        subject.execute(instruction)
+      end
+
+      it "must pop the #meta_stack and create a frozen Set from the previous #stack and push the frozen Set onto the new #stack" do
+        expect(subject.stack.length).to eq(1)
+
+        set = subject.stack[-1]
+
+        expect(set).to be_frozen
+        expect(set).to eq(Set[1,2,3])
+      end
+    end
+
     context "when given a Python::Pickle::Instructions::APPEND" do
       context "and when the previous element on the stack is an Array" do
         let(:instruction) { Python::Pickle::Instructions::APPEND }
@@ -378,6 +488,19 @@ describe Python::Pickle::Deserializer do
         end
       end
 
+      context "and when the previous element on the stack is a Set" do
+        let(:instruction) { Python::Pickle::Instructions::APPEND }
+
+        before do
+          subject.stack << Set.new << 2
+          subject.execute(instruction)
+        end
+
+        it "must pop the last element from the #stack and push it onto the next list element" do
+          expect(subject.stack).to eq([ Set[2] ])
+        end
+      end
+
       context "but when the previous element on the stack is not an Array" do
         let(:instruction) { Python::Pickle::Instructions::APPEND }
         let(:item) { 2 }
@@ -410,6 +533,20 @@ describe Python::Pickle::Deserializer do
         end
       end
 
+      context "and when the previous element on the stack is a Set" do
+        let(:instruction) { Python::Pickle::Instructions::APPENDS }
+
+        before do
+          subject.meta_stack << [ Set[1,2,3] ]
+          subject.stack << 4 << 5 << 6
+          subject.execute(instruction)
+        end
+
+        it "must pop the #meta_stack, store the #stack, and concat the previous #stack onto the last element of the new #stack" do
+          expect(subject.stack).to eq([ Set[1,2,3,4,5,6] ])
+        end
+      end
+
       context "but when the previous element on the stack is not an Array" do
         let(:instruction) { Python::Pickle::Instructions::APPENDS }
         let(:items) { [3,4,5] }
@@ -428,6 +565,39 @@ describe Python::Pickle::Deserializer do
       end
     end
 
+    context "when given a Python::Pickle::Instructions::ADDITEMS" do
+      context "and when the previous element on the stack is a Set" do
+        let(:instruction) { Python::Pickle::Instructions::ADDITEMS }
+
+        before do
+          subject.meta_stack << [ Set[1,2,3] ]
+          subject.stack << 4 << 5 << 6
+          subject.execute(instruction)
+        end
+
+        it "must pop the #meta_stack, store the #stack, and concat the previous #stack onto the last element of the new #stack" do
+          expect(subject.stack).to eq([ Set[1,2,3,4,5,6] ])
+        end
+      end
+
+      context "but when the previous element on the stack is not a Set" do
+        let(:instruction) { Python::Pickle::Instructions::ADDITEMS }
+        let(:items) { [3,4,5] }
+        let(:set)   { [] }
+
+        before do
+          subject.meta_stack << [ set ]
+          subject.stack << items[0] << items[1] << items[2]
+        end
+
+        it do
+          expect {
+            subject.execute(instruction)
+          }.to raise_error(Python::Pickle::DeserializationError,"cannot add items #{items.inspect} to a non-Set object: #{set.inspect}")
+        end
+      end
+    end
+
     context "when given a Python::Pickle::Instructions::LIST" do
       let(:instruction) { Python::Pickle::Instructions::LIST }
 
@@ -578,6 +748,142 @@ describe Python::Pickle::Deserializer do
       end
     end
 
+    context "when given a Python::Pickle::Instructions::Inst object" do
+      let(:namespace)   { '__main__' }
+      let(:name)        { 'MyClass'  }
+      let(:instruction) { Python::Pickle::Instructions::Inst.new(namespace,name) }
+
+      before do
+        subject.meta_stack << []
+        subject.stack << 1 << 2
+        subject.execute(instruction)
+      end
+
+      context "when the constant can be resolved" do
+        module TestInstInstruction
+          class MyClass
+            attr_reader :x, :y
+
+            def initialize(x,y)
+              @x = x
+              @y = y
+            end
+          end
+        end
+
+        subject do
+          described_class.new(
+            constants: {
+              '__main__' => {
+                'MyClass' => TestInstInstruction::MyClass
+              }
+            }
+          )
+        end
+
+        it "must resolve the class from the INST instruction's #namespace and #name, pop the meta stack, use the previous sstack as the initialization arguments, initialize a new instance of the class, and push it onto the new #stack" do
+          expect(subject.stack.length).to eq(1)
+
+          object = subject.stack[-1]
+
+          expect(object).to be_kind_of(TestInstInstruction::MyClass)
+          expect(object.x).to eq(1)
+          expect(object.y).to eq(2)
+        end
+      end
+
+      context "but the constant cannot be resolved" do
+        it "must push a new Python::Pickle::PyClass object onto the #stack" do
+          py_object = subject.stack[-1]
+
+          expect(py_object).to be_kind_of(Python::Pickle::PyObject)
+          expect(py_object.py_class).to be_kind_of(Python::Pickle::PyClass)
+          expect(py_object.py_class.namespace).to eq(namespace)
+          expect(py_object.py_class.name).to eq(name)
+          expect(py_object.init_args).to eq([1,2])
+        end
+      end
+    end
+
+    context "when given a Python::Pickle::Instructions::OBJ" do
+      let(:instruction) { Python::Pickle::Instructions::OBJ }
+
+      context "and when the constant on the #stack is a Ruby class" do
+        module TestObjInstruction
+          class MyClass
+          end
+        end
+
+        context "but there are no additional arguments on the #stack after the class" do
+          before do
+            subject.stack << TestObjInstruction::MyClass
+            subject.execute(instruction)
+          end
+
+          it "must pop off first element, and initialize a new instance of the class, and push the new instance onto the #stack" do
+            expect(subject.stack.length).to eq(1)
+            expect(subject.stack[-1]).to be_kind_of(TestObjInstruction::MyClass)
+          end
+        end
+
+        context "but there are additional arguments on the #stack after the class" do
+          module TestObjInstruction
+            class MyClassWithArgs
+              attr_reader :x, :y
+
+              def initialize(x,y)
+                @x = x
+                @y = y
+              end
+            end
+          end
+
+          before do
+            subject.stack << TestObjInstruction::MyClassWithArgs << 1 << 2
+            subject.execute(instruction)
+          end
+
+          it "must call #initialize with the splatted tuple's arguments" do
+            object = subject.stack[-1]
+
+            expect(object.x).to eq(1)
+            expect(object.y).to eq(2)
+          end
+        end
+      end
+
+      context "and when the constant on the #stack is a PyClass" do
+        let(:namespace) { '__main__' }
+        let(:name)      { 'MyClass'  }
+        let(:py_class)  { Python::Pickle::PyClass.new(namespace,name) }
+
+        context "but there are no additional arguments on the #stack after the class" do
+          before do
+            subject.stack << py_class
+            subject.execute(instruction)
+          end
+
+          it "must pop off the two last elements and push the new Python::Pickle::PyObject onto the #stack" do
+            expect(subject.stack.length).to eq(1)
+            expect(subject.stack[-1]).to be_kind_of(Python::Pickle::PyObject)
+          end
+        end
+
+        context "but there are additional arguments on the #stack after the class" do
+          before do
+            subject.stack << py_class << 1 << 2
+            subject.execute(instruction)
+          end
+
+          it "must set the object's #init_args to the tuple's elements" do
+            object = subject.stack[-1]
+
+            expect(object.init_args).to eq([1,2])
+          end
+        end
+      end
+    end
+
     context "when given a Python::Pickle::Instructions::NEWOBJ" do
       let(:instruction) { Python::Pickle::Instructions::NEWOBJ }
 
@@ -1150,6 +1456,59 @@ describe Python::Pickle::Deserializer do
         end
       end
     end
+
+    context "when given a Python::Pickle::Instructions::NEXT_BUFFER" do
+      let(:instruction) { Python::Pickle::Instructions::NEXT_BUFFER }
+
+      context "and the #{described_class} was initialized with the buffers: keyword argument" do
+        let(:buffer1) { String.new("hello world") }
+        let(:buffer2) { String.new("foo bar")     }
+        let(:buffers) do
+          [
+            buffer1,
+            buffer2
+          ]
+        end
+
+        subject { described_class.new(buffers: buffers) }
+
+        before do
+          subject.execute(instruction)
+        end
+
+        it "must take the next element from #buffers and push it onto the #stack" do
+          expect(subject.stack).to eq([buffer1])
+        end
+
+        it "must not modify the underlying buffers Array" do
+          expect(buffers).to eq([buffer1, buffer2])
+        end
+      end
+
+      context "but the #{described_class} was not initialized with the buffers: keyword argument" do
+        it do
+          expect {
+            subject.execute(instruction)
+          }.to raise_error(Python::Pickle::DeserializationError,"pickle stream includes a NEXT_BUFFER instruction, but no buffers were provided")
+        end
+      end
+    end
+
+    context "when given a Python::Pickle::Instructions::READONLY_BUFFER" do
+      let(:instruction) { Python::Pickle::Instructions::READONLY_BUFFER }
+
+      let(:buffer1) { String.new("hello world") }
+      let(:buffer2) { String.new("foo bar")     }
+
+      before do
+        subject.stack << buffer1 << buffer2
+        subject.execute(instruction)
+      end
+
+      it "must freeze the buffer at the top of the #stack" do
+        expect(subject.stack[-1]).to be_frozen
+      end
+    end
   end
 
   describe "#copyreg_reconstructor" do

data/spec/fixtures/set_v0.pkl

@@ -0,0 +1,11 @@
+c__builtin__
+set
+p0
+((lp1
+I1
+aI2
+aI3
+aI4
+atp2
+Rp3
+.

data/spec/generate_pickles2.py

@@ -26,6 +26,7 @@ objects = {
 		"bin_str": b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 		"list": [None, True, False, 42, 'ABC'],
 		"nested_list": [1, [2, [3, [4]]]],
+		"set": set([1,2,3,4]),
 		"dict": {"foo": "bar"},
 		"nested_dict": {"a": {"b": {"c": "d"}}},
 		"function": func,

data/spec/generate_pickles3.py

@@ -25,6 +25,7 @@ objects = {
 		"bin_str": b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 		"list": [None, True, False, 42, 'ABC'],
 		"nested_list": [1, [2, [3, [4]]]],
+		"set": set([1,2,3,4]),
 		"dict": {"foo": "bar"},
 		"nested_dict": {"a": {"b": {"c": "d"}}},
 		"function": func,

data/spec/integration/load/protocol0_spec.rb

@@ -104,6 +104,16 @@ describe Python::Pickle do
         end
       end
 
+      context "and it contains a set type" do
+        let(:file) { File.join(fixtures_dir,'set_v0.pkl') }
+
+        it "must return a Set object" do
+          expect(subject.load(io)).to eq(
+            Set[1, 2, 3, 4]
+          )
+        end
+      end
+
       context "and it contains a dict type" do
         let(:file) { File.join(fixtures_dir,'dict_v0.pkl') }
 

data/spec/integration/load/protocol1_spec.rb

@@ -104,6 +104,16 @@ describe Python::Pickle do
         end
       end
 
+      context "and it contains a set type" do
+        let(:file) { File.join(fixtures_dir,'set_v1.pkl') }
+
+        it "must return a Set object" do
+          expect(subject.load(io)).to eq(
+            Set[1, 2, 3, 4]
+          )
+        end
+      end
+
       context "and it contains a dict type" do
         let(:file) { File.join(fixtures_dir,'dict_v1.pkl') }
 

data/spec/integration/load/protocol2_spec.rb

@@ -104,6 +104,16 @@ describe Python::Pickle do
         end
       end
 
+      context "and it contains a set type" do
+        let(:file) { File.join(fixtures_dir,'set_v2.pkl') }
+
+        it "must return a Set object" do
+          expect(subject.load(io)).to eq(
+            Set[1, 2, 3, 4]
+          )
+        end
+      end
+
       context "and it contains a dict type" do
         let(:file) { File.join(fixtures_dir,'dict_v2.pkl') }
 

data/spec/integration/load/protocol3_spec.rb

@@ -104,6 +104,16 @@ describe Python::Pickle do
         end
       end
 
+      context "and it contains a set type" do
+        let(:file) { File.join(fixtures_dir,'set_v3.pkl') }
+
+        it "must return a Set object" do
+          expect(subject.load(io)).to eq(
+            Set[1, 2, 3, 4]
+          )
+        end
+      end
+
       context "and it contains a dict type" do
         let(:file) { File.join(fixtures_dir,'dict_v3.pkl') }
 

data/spec/integration/load/protocol4_spec.rb

@@ -104,6 +104,16 @@ describe Python::Pickle do
         end
       end
 
+      context "and it contains a set type" do
+        let(:file) { File.join(fixtures_dir,'set_v4.pkl') }
+
+        it "must return a Set object" do
+          expect(subject.load(io)).to eq(
+            Set[1, 2, 3, 4]
+          )
+        end
+      end
+
       context "and it contains a dict type" do
         let(:file) { File.join(fixtures_dir,'dict_v4.pkl') }
 

data/spec/integration/load/protocol5_spec.rb

@@ -104,6 +104,16 @@ describe Python::Pickle do
         end
       end
 
+      context "and it contains a set type" do
+        let(:file) { File.join(fixtures_dir,'set_v5.pkl') }
+
+        it "must return a Set object" do
+          expect(subject.load(io)).to eq(
+            Set[1, 2, 3, 4]
+          )
+        end
+      end
+
       context "and it contains a dict type" do
         let(:file) { File.join(fixtures_dir,'dict_v5.pkl') }
 

data/spec/pickle_spec.rb

@@ -52,6 +52,33 @@ describe Python::Pickle do
     it "must deserialize the Python Pickle data in the given file" do
       expect(subject.load(data)).to eq({"foo" => "bar"})
     end
+
+    context "when the constants: keyword argument is given" do
+      let(:path) { File.join(fixtures_dir,'object_v4.pkl') }
+
+      module TestLoad
+        class MyClass
+          attr_reader :x, :y
+
+          def __setstate__(attributes)
+            @x = attributes['x']
+            @y = attributes['y']
+          end
+        end
+      end
+
+      let(:constants) do
+        {
+          '__main__' => {
+            'MyClass' => TestLoad::MyClass
+          }
+        }
+      end
+
+      it "must map the Python classes to the Ruby classes" do
+        expect(subject.load(data, constants: constants)).to be_kind_of(TestLoad::MyClass)
+      end
+    end
   end
 
   describe ".load_file" do
@@ -60,9 +87,43 @@ describe Python::Pickle do
     it "must deserialize the Python Pickle data in the given file" do
       expect(subject.load_file(path)).to eq({"foo" => "bar"})
     end
+
+    context "when the constants: keyword argument is given" do
+      let(:path) { File.join(fixtures_dir,'object_v4.pkl') }
+
+      module TestLoad
+        class MyClass
+          attr_reader :x, :y
+
+          def __setstate__(attributes)
+            @x = attributes['x']
+            @y = attributes['y']
+          end
+        end
+      end
+
+      let(:constants) do
+        {
+          '__main__' => {
+            'MyClass' => TestLoad::MyClass
+          }
+        }
+      end
+
+      it "must map the Python classes to the Ruby classes" do
+        expect(subject.load_file(path, constants: constants)).to be_kind_of(TestLoad::MyClass)
+      end
+    end
   end
 
   describe ".dump" do
+    let(:object) { Object.new }
+
+    it do
+      expect {
+        subject.dump(object)
+      }.to raise_error(NotImplementedError,"pickle serializing is currently not supported")
+    end
   end
 
   describe ".infer_protocol_version" do