unpickle 0.0.1

data/README.md

@@ -0,0 +1,37 @@
+# unpickle
+
+This is a very limited tool to unpickle python 'pickle' objects.
+
+# Usage
+
+    require 'unpickle'
+    
+    fh = File.open('some.pickle')
+    o = unpickle(fh.read)
+
+unpickle() will raise Unpickle::UnpickleException if it doesn't support an
+opcode in the picklestream, or encounters some kind of problem or invalid 
+sequence.
+
+# Limitations
+
+Currently unpickle only supports protocol 0.
+
+It only supports Integers (not Longs), Strings (non-unicode),
+booleans, dictionaries, lists, tuples and None.
+
+Tuples will be returned as a frozen array.
+
+None will be returned as nil.
+
+# TODO
+
+ * Support more of protocol 0.
+ * Support newer protocols.
+ * Test object identity mapping.
+ * Package as a gem
+ * fix namespace.
+
+# Author
+
+Chris Collins <chris.collins@anchor.net.au>

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/lib/unpickle.rb

@@ -0,0 +1,213 @@
+# unpickle.rb
+# 
+# baby-unpickler to handle limited protocol 0 crap.
+#
+# vim:et sts=4 sw=4 ts=8:
+#
+module Unpickle
+    class UnpickleException < RuntimeError
+    end
+
+    class Mark
+    end
+
+    class PickleMachine
+        def initialize(input)
+            @stack = []
+            @memo = {}
+            @input = input
+            @idx = 0
+        end
+
+        def at_end?
+            @idx >= @input.length
+        end
+
+        def next_char
+            rv = @input[@idx..@idx]
+            @idx += 1
+            return rv
+        end
+
+        def peek_char
+            @input[@idx..@idx]
+        end
+
+        def read_int
+            strout = ""
+            while peek_char != "\n"
+                strout += next_char
+            end
+            next_char
+            case strout
+            when '00'
+                return false
+            when '01'
+                return true
+            else
+                return strout.to_i
+            end
+        end
+
+        def marker
+            idx = @stack.length-1
+            while idx >= 0
+                if @stack[idx].is_a?(Mark)
+                    return idx
+                end
+                idx -= 1
+            end
+            raise UnpickleException, "Couldn't find Mark"
+        end
+
+        # read from the input stream to read the python string.
+        #
+        # returns the value.
+        def read_string
+            strout = ''
+            if next_char != '\''
+                raise UnpickleException, "Couldn't find leading quote for string"
+            end
+            while not at_end?
+                c = next_char
+                case c
+                when "\\"
+                    opt = next_char
+                    case opt
+                    when 'x'
+                        num = ''
+                        while peek_char.match(/[\dA-Fa-f]/)
+                            num += next_char
+                            if num.length >= 2
+                                break
+                            end
+                        end
+                        unless (1..2).include?(num.length)
+                            raise UnpickleException, "Bad \\x sequence in string"
+                        end
+                        strout += num.to_i(16).chr
+                    when '0'
+                        num = ''
+                        while peek_char.match(/[0-7]/)
+                            num += next_char
+                            if num.length >= 3
+                                break
+                            end
+                        end
+                        unless (1..3).include?(num.length)
+                            raise UnpickleException, "Bad \\0 sequence in string"
+                        end
+                        strout += num.to_i(8).chr
+                    when 'n'
+                        strout += "\n"
+                    when "\\"
+                        strout += "\\"
+                    when 't'
+                        strout += "\t"
+                    when "'"
+                        strout += "'"
+                    else
+                        raise UnpickleException, "Unexpected \\ escape: \\#{opt}"
+                    end
+                when "'"
+                    # valid end of string...
+                    break
+                else
+                    strout += c
+                end
+            end
+            if next_char != "\n"
+                raise UnpickleException, "Expected \\n after string"
+            end
+            return strout
+        end
+
+        def unpickle
+            while not at_end?
+                op = next_char
+                case op
+                when '(' # MARK
+                    @stack.push(Mark.new)
+                when 'd' # DICT
+                    newdict = {}
+                    while true
+                        if @stack.empty?
+                            raise UnpickleException, "Stack empty during 'd'"
+                        end
+                        v = @stack.pop
+                        if v.is_a?(Mark)
+                            break
+                        end
+                        if @stack.empty?
+                            raise UnpickleException, "Stack empty during 'd'"
+                        end
+                        k = stack.pop
+                        if k.is_a?(Mark)
+                            raise UnpickleException, "Odd number of elements during 'd' stack walk"
+                        end
+                        newdict[k] = v
+                    end
+                    @stack.push(newdict)
+                when 'S' # STRING
+                    newstr = read_string
+                    @stack.push(newstr)
+                when 'p' # PUT (string)
+                    index = read_int
+                    @memo[index] = @stack[-1]
+                when 'g' # GET (string)
+                    index = read_int
+                    @stack.push(@memo[index])
+                when 'I' # INT
+                    intarg = read_int
+                    @stack.push(intarg)
+                when 's' # SETITEM
+                    value = @stack.pop
+                    key = @stack.pop
+                    dict = @stack[-1]
+                    dict[key] = value
+                when 't' # TUPLE
+                    midx = marker
+                    tuple = @stack[midx+1..-1]
+                    @stack = @stack[0...midx]
+                    tuple.freeze
+                    @stack.push(tuple)
+                when 'l' # LIST
+                    midx = marker
+                    list = @stack[midx+1..-1]
+                    @stack = @stack[0...midx]
+                    @stack.push(list)
+                when 'N' # NONE
+                    @stack.push(nil)
+                when 'a' # APPEND
+                    e = @stack.pop
+                    @stack[-1].push(e)
+                when '0' # POP
+                    @stack.pop
+                when '2' # DUP
+                    @stack.push(@stack[-1])
+                when '.' # STOP
+                    return @stack.pop
+                else
+                    raise UnpickleException, "Unsupported unpickle operation '#{op}'"
+                end
+            end
+            raise UnpickleException, "Hit end of input stream"
+        end
+    end
+
+    # Unpickle the python object pickled into str.
+    # 
+    # At this time, this ONLY works with a limited set of constructs 
+    # (dicts, lists, tuples, strings, ints, bools, None) and only with
+    # protocol 0.
+    # 
+    # Object uniqueness should obey the python semantics but is largely 
+    # untested.
+    #
+    # Raises an UnpickleException if anything goes wrong.
+    def Unpickle.loads(str)
+        p = Unpickle::PickleMachine.new(str)
+        return p.unpickle
+    end
+end
+

data/test/test_unpickle.rb

@@ -0,0 +1,99 @@
+# tests for Unpickle.loads.rb
+#
+# vim:et sts=4 sw=4 ts=8:
+require 'test/unit'
+require 'unpickle'
+
+class UnpickleTests < Test::Unit::TestCase
+    # all pickle_str's were generated using python 2.5
+    # and pickle.dumps(...., 0)
+    
+    def test_simple_string
+        # >>> pickle_obj = 'abcdefg'
+        pickle_str = "S'abcdefg'\np0\n."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal('abcdefg', o)
+    end
+
+    def test_simple_bool_true
+        # >>> pickle_obj = True
+        pickle_str = "I01\n."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal(true, o)
+    end
+
+    def test_simple_bool_false
+        # >>> pickle_obj = False
+        pickle_str = "I00\n."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal(false, o)
+    end
+
+    def test_simple_list
+        # >>> pickle_obj = [1,2,3]
+        pickle_str = "(lp0\nI1\naI2\naI3\na."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal([1,2,3], o)
+    end
+
+    def test_simple_dict
+        # >>> pickle_obj = {'a': 1, 'b': 2, 'c': 3}
+        pickle_str = "(dp0\nS'a'\np1\nI1\nsS'c'\np2\nI3\nsS'b'\np3\nI2\ns."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal({'a' => 1, 'b' => 2, 'c' => 3}, o)
+    end
+
+    def test_simple_int_zero
+        # >>> pickle_obj = 0
+        pickle_str = "I0\n."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal(0, o)
+    end
+
+    def test_none
+        # >>> pickle_obj = None
+        pickle_str = 'N.'
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal(nil, o)
+    end
+
+    def test_mixed
+        # >>> pickle_obj = {'a': [1,2,3,4], 'b': (1,2,3), 'c': None, 'd': 'abcd'}
+        pickle_str = "(dp0\nS'a'\np1\n(lp2\nI1\naI2\naI3\naI4\nasS'c'\np3\nNsS'b'\np4\n(I1\nI2\nI3\ntp5\nsS'd'\np6\nS'abcd'\np7\ns."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert_equal({'a' => [1,2,3,4], 'b' => [1,2,3], 'c' => nil, 'd' => 'abcd'}, o)
+    end
+
+    def test_recursive
+        # >>> aobj = {}
+        # >>> bobj = {'a': aobj}
+        # >>> aobj['b'] = bobj
+        # >>> pickle.dumps(aobj, 0)
+        pickle_str = "(dp0\nS'b'\np1\n(dp2\nS'a'\np3\ng0\nss."
+
+        o = Unpickle.loads(pickle_str)
+
+        assert(o.is_a?(Hash))
+        assert(o.include?('b'))
+        assert(o['b'].is_a?(Hash))
+        assert(o['b'].include?('a'))
+        # check that the object identity is correctly respected.
+        assert(o.object_id == o['b']['a'].object_id)
+    end
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: unpickle
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Chris Collins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-01 00:00:00.000000000 Z
+dependencies: []
+description: A library to unpickle simple python objects directly into ruby
+email: kuroneko-rubygems@sysadninjas.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/unpickle.rb
+- README.md
+- Rakefile
+- test/test_unpickle.rb
+homepage: http://github.com/kuroneko/unpickle-rb
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: unpickle python objects in ruby
+test_files:
+- test/test_unpickle.rb