rmarshal 0.1

data/lib/rmarshal.rb

@@ -0,0 +1,2 @@
+require 'rmarshal/unmarshal'
+require 'rmarshal/pyc'

data/lib/rmarshal/disasm.rb

@@ -0,0 +1,56 @@
+require 'rmarshal/opcodes'
+
+class Disasm
+	def initialize(code, consts, varnames, names)
+		@code = code
+		@consts = consts
+		@varnames = varnames
+		@names = names
+		@insts = []
+		@off = 0
+		
+		while @off != @code.size
+			@insts[@insts.size] = parse
+		end
+	end
+	
+	def insts() @insts end
+	def byte()
+		val = @code[@off].ord
+		@off += 1
+		val
+	end
+	def short()
+		val = @code[@off...@off+2].unpack('s')[0]
+		@off += 2
+		val
+	end
+	
+	def parse
+		off = @off
+		opcd = byte
+		if not $opcodes.has_key? opcd
+			raise "Unknown opcode #{opcd}"
+		end
+		
+		name, arg = $opcodes[opcd]
+		if arg == nil
+			[off, name]
+		else
+			val = short
+			arg = case arg
+					when :const then @consts[val]
+					when :name then @names[val]
+					when :jrel then @off + val
+					when :local then @varnames[val]
+					else val
+				end
+			
+			[off, name, arg]
+		end
+	end
+end
+
+def disasm(*args)
+	Disasm.new(*args).insts
+end

data/lib/rmarshal/opcodes.rb

@@ -0,0 +1,115 @@
+$opcodes = {
+		0x00 => [:STOP_CODE, nil], 
+		0x01 => [:POP_TOP, nil], 
+		0x02 => [:ROT_TWO, nil], 
+		0x03 => [:ROT_THREE, nil], 
+		0x04 => [:DUP_TOP, nil], 
+		0x05 => [:ROT_FOUR, nil], 
+		0x09 => [:NOP, nil], 
+		0x0A => [:UNARY_POSITIVE, nil], 
+		0x0B => [:UNARY_NEGATIVE, nil], 
+		0x0C => [:UNARY_NOT, nil], 
+		0x0D => [:UNARY_CONVERT, nil], 
+		0x0F => [:UNARY_INVERT, nil], 
+		0x12 => [:LIST_APPEND, nil], 
+		0x13 => [:BINARY_POWER, nil], 
+		0x14 => [:BINARY_MULTIPLY, nil], 
+		0x15 => [:BINARY_DIVIDE, nil], 
+		0x16 => [:BINARY_MODULO, nil], 
+		0x17 => [:BINARY_ADD, nil], 
+		0x18 => [:BINARY_SUBTRACT, nil], 
+		0x19 => [:BINARY_SUBSCR, nil], 
+		0x1A => [:BINARY_FLOOR_DIVIDE, nil], 
+		0x1B => [:BINARY_TRUE_DIVIDE, nil], 
+		0x1C => [:INPLACE_FLOOR_DIVIDE, nil], 
+		0x1D => [:INPLACE_TRUE_DIVIDE, nil], 
+		0x1E => [:SLICE_0, nil], 
+		0x1F => [:SLICE_1, nil], 
+		0x20 => [:SLICE_2, nil], 
+		0x21 => [:SLICE_3, nil], 
+		0x28 => [:STORE_SLICE_0, nil], 
+		0x29 => [:STORE_SLICE_1, nil], 
+		0x2A => [:STORE_SLICE_2, nil], 
+		0x2B => [:STORE_SLICE_3, nil], 
+		0x32 => [:DELETE_SLICE_0, nil], 
+		0x33 => [:DELETE_SLICE_1, nil], 
+		0x34 => [:DELETE_SLICE_2, nil], 
+		0x35 => [:DELETE_SLICE_3, nil], 
+		0x36 => [:STORE_MAP, nil], 
+		0x37 => [:INPLACE_ADD, nil], 
+		0x38 => [:INPLACE_SUBTRACT, nil], 
+		0x39 => [:INPLACE_MULTIPLY, nil], 
+		0x3A => [:INPLACE_DIVIDE, nil], 
+		0x3B => [:INPLACE_MODULO, nil], 
+		0x3C => [:STORE_SUBSCR, nil], 
+		0x3D => [:DELETE_SUBSCR, nil], 
+		0x3E => [:BINARY_LSHIFT, nil], 
+		0x3F => [:BINARY_RSHIFT, nil], 
+		0x40 => [:BINARY_AND, nil], 
+		0x41 => [:BINARY_XOR, nil], 
+		0x42 => [:BINARY_OR, nil], 
+		0x43 => [:INPLACE_POWER, nil], 
+		0x44 => [:GET_ITER, nil], 
+		0x46 => [:PRINT_EXPR, nil], 
+		0x47 => [:PRINT_ITEM, nil], 
+		0x48 => [:PRINT_NEWLINE, nil], 
+		0x49 => [:PRINT_ITEM_TO, nil], 
+		0x4A => [:PRINT_NEWLINE_TO, nil], 
+		0x4B => [:INPLACE_LSHIFT, nil], 
+		0x4C => [:INPLACE_RSHIFT, nil], 
+		0x4D => [:INPLACE_AND, nil], 
+		0x4E => [:INPLACE_XOR, nil], 
+		0x4F => [:INPLACE_OR, nil], 
+		0x50 => [:BREAK_LOOP, nil], 
+		0x51 => [:WITH_CLEANUP, nil], 
+		0x52 => [:LOAD_LOCALS, nil], 
+		0x53 => [:RETURN_VALUE, nil], 
+		0x54 => [:IMPORT_STAR, nil], 
+		0x55 => [:EXEC_STMT, nil], 
+		0x56 => [:YIELD_VALUE, nil], 
+		0x57 => [:POP_BLOCK, nil], 
+		0x58 => [:END_FINALLY, nil], 
+		0x59 => [:BUILD_CLASS, nil], 
+		0x5A => [:STORE_NAME, :name], 
+		0x5B => [:DELETE_NAME, :name], 
+		0x5C => [:UNPACK_SEQUENCE, nil], 
+		0x5D => [:FOR_ITER, :jrel], 
+		0x5F => [:STORE_ATTR, :name], 
+		0x60 => [:DELETE_ATTR, :name], 
+		0x61 => [:STORE_GLOBAL, :name], 
+		0x62 => [:DELETE_GLOBAL, :name], 
+		0x63 => [:DUP_TOPX, nil], 
+		0x64 => [:LOAD_CONST, :const], 
+		0x65 => [:LOAD_NAME, :name], 
+		0x66 => [:BUILD_TUPLE, nil], 
+		0x67 => [:BUILD_LIST, nil], 
+		0x68 => [:BUILD_MAP, nil], 
+		0x69 => [:LOAD_ATTR, :name], 
+		0x6A => [:COMPARE_OP, :compare], 
+		0x6B => [:IMPORT_NAME, :name], 
+		0x6C => [:IMPORT_FROM, :name], 
+		0x6E => [:JUMP_FORWARD, :jrel], 
+		0x6F => [:JUMP_IF_FALSE, :jrel], 
+		0x70 => [:JUMP_IF_TRUE, :jrel], 
+		0x71 => [:JUMP_ABSOLUTE, :jabs], 
+		0x74 => [:LOAD_GLOBAL, :name], 
+		0x77 => [:CONTINUE_LOOP, :jabs], 
+		0x78 => [:SETUP_LOOP, :jrel], 
+		0x79 => [:SETUP_EXCEPT, :jrel], 
+		0x7A => [:SETUP_FINALLY, :jrel], 
+		0x7C => [:LOAD_FAST, :local], 
+		0x7D => [:STORE_FAST, :local], 
+		0x7E => [:DELETE_FAST, :local], 
+		0x82 => [:RAISE_VARARGS, nil], 
+		0x83 => [:CALL_FUNCTION, nil], 
+		0x84 => [:MAKE_FUNCTION, nil], 
+		0x85 => [:BUILD_SLICE, nil], 
+		0x86 => [:MAKE_CLOSURE, nil], 
+		0x87 => [:LOAD_CLOSURE, :free], 
+		0x88 => [:LOAD_DEREF, :free], 
+		0x89 => [:STORE_DEREF, :free], 
+		0x8C => [:CALL_FUNCTION_VAR, nil], 
+		0x8D => [:CALL_FUNCTION_KW, nil], 
+		0x8E => [:CALL_FUNCTION_VAR_KW, nil], 
+		0x8F => [:EXTENDED_ARG, nil], 
+	}

data/lib/rmarshal/pyc.rb

@@ -0,0 +1,9 @@
+require 'rmarshal'
+
+def pyc(fn)
+	fp = File.open fn
+	
+	magic, timestamp = fp.read(8).unpack('ii')
+	
+	unmarshal fp
+end

data/lib/rmarshal/unmarshal.rb

@@ -0,0 +1,113 @@
+require 'rmarshal/disasm'
+require 'set'
+
+class Code
+	def initialize(
+			argcount, nlocals, stacksize, flags, code, consts, names, varnames, freevars, 
+			cellvars, filename, name, firstlineno, lnotab
+		)
+		
+		@argcount, @nlocals, @stacksize, @flags = argcount, nlocals, stacksize, flags
+		@code, @consts, @names, @varnames = code, consts, names, varnames
+		@freevars, @cellvars, @filename, @name = freevars, cellvars, filename, name
+		@firstlineno, @lnotab = firstlineno, lnotab
+	end
+	
+	def method_missing(sym)
+		instance_variable_get sym
+	end
+	
+	def disassemble
+		disasm @code, @consts, @varnames, @names
+	end
+end
+
+class Unmarshal
+	def initialize(fp)
+		@fp = fp
+		@interned = []
+		
+		@value = unmarshal
+	end
+	
+	def value() @value end
+	def byte() @fp.read 1 end
+	def long() @fp.read(4).unpack('i')[0] end
+	def double() @fp.read(8).unpack('E')[0] end
+	
+	def unmarshal
+		type = byte
+		case type
+			when '.' then :Ellipsis
+			when '0' then :null # Special!
+			when 'N' then nil
+			when 'T' then true
+			when 'F' then false
+			
+			when 'c'
+				argcount, nlocals, stacksize, flags = long, long, long, long
+				
+				code_, consts, names, varnames, freevars = unmarshal, unmarshal, unmarshal, unmarshal, unmarshal
+				cellvars, filename, name = unmarshal, unmarshal, unmarshal
+				firstlineno, lnotab = long, unmarshal
+				
+				Code.new( 
+						argcount, nlocals, stacksize, flags, 
+						code_, consts, names, varnames, freevars, 
+						cellvars, filename, name, firstlineno, lnotab
+					)
+			
+			when 'f' then @fp.read(byte).to_f
+			when 'g' then double
+			when 'i' then long
+			when 'I' then @fp.read(8).unpack('q')[0]
+			
+			when 'R'
+				@interned[long]
+			
+			when 's' then @fp.read long
+			
+			when 'S' then :StopIteration
+			
+			when 't'
+				str = @fp.read long
+				@interned[@interned.size] = str
+				str
+			
+			when 'u' then @fp.read(long).force_encoding 'utf-8'
+			
+			when 'x'
+				Complex @fp.read(byte).to_f(), @fp.read(byte).to_f()
+			when 'y'
+				Complex double, double
+			
+			when '(', '['
+				(0...long).map { unmarshal }
+			
+			when '{'
+				val = {}
+				
+				while true
+					key = unmarshal
+					break if key == :null
+					
+					val[key] = unmarshal
+				end
+				
+				val
+			
+			when '<', '>'
+				Set.new((0...long).map { unmarshal })
+			
+			when '?'
+				raise 'TYPE_UNKNOWN encountered.  Incomplete marshal.'
+			
+			else
+				raise "Unknown type '#{type}'"
+		end
+	end
+end
+
+def unmarshal(fp)
+	Unmarshal.new(fp).value
+end

data/test/exhaustive.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+# Creates a file containing all possible marshal types
+
+import marshal
+
+def testfunc():
+	print 'Test'
+
+def exhaustive():
+	return (
+			None, # TYPE_NONE 'N'
+			False, # TYPE_FALSE 'F'
+			True, # TYPE_TRUE 'T'
+			0, # TYPE_INT 'i'
+			#1L, # TYPE_LONG 'l'
+			0.0, # TYPE_BINARY_FLOAT 'g'
+			1j, # TYPE_BINARY_COMPLEX 'y'
+			'foo', # TYPE_STRING 's'
+			'bar', # TYPE_INTERNED 't'
+			'bar', # TYPE_STRINGREF 'R'
+			u'unicode €', # TYPE_UNICODE 'u'
+			[0, 'test', True], # TYPE_LIST '['
+			{0 : 1, 'bar' : 'baz'}, # TYPE_DICT '{'
+			testfunc.func_code, # TYPE_CODE 'c'
+			
+			set([0, 1, 2]), # TYPE_SET
+			frozenset([3, 4, 5]), # TYPE_FROZENSET
+			StopIteration, # TYPE_STOPITER
+			Ellipsis, # TYPE_ELLIPSIS
+		)
+
+marshal.dump(exhaustive(), file('tests\exhaustive.mrsh', 'wb'))

data/test/test.rb

@@ -0,0 +1,5 @@
+require 'rmarshal'
+require 'pp'
+
+fp = File.open 'exhaustive.mrsh'
+pp unmarshal fp

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+name: rmarshal
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- Cody Brocious
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-20 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: 
+email: cody.brocious@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/rmarshal/disasm.rb
+- lib/rmarshal/opcodes.rb
+- lib/rmarshal/pyc.rb
+- lib/rmarshal/unmarshal.rb
+- lib/rmarshal.rb
+- test/exhaustive.py
+- test/test.rb
+has_rdoc: true
+homepage: http://github.com/daeken/RMarshal
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Ruby gem to support the Python marshal format
+test_files: []
+