metasm 1.0.0 → 1.0.5

data/{lib/metasm → metasm}/exe_format/nds.rb

@@ -30,7 +30,7 @@ class NDS < ExeFormat
 		mem :secareadisable, 8
 		words :endoff, :headersz
 		mem :reserved4, 56
-	       	mem :ninlogo, 156
+		mem :ninlogo, 156
 		half :logoCRC, 0xcf56
 		half :headerCRC
 	end
@@ -70,14 +70,17 @@ class NDS < ExeFormat
 	def decode_byte(edata = @encoded) edata.decode_imm(:u8,  @endianness) end
 	def decode_half(edata = @encoded) edata.decode_imm(:u16, @endianness) end
 	def decode_word(edata = @encoded) edata.decode_imm(:u32, @endianness) end
+	def sizeof_byte ; 1 ; end
+	def sizeof_half ; 2 ; end
+	def sizeof_word ; 4 ; end
 
 
 	attr_accessor :header, :icon, :arm9, :arm7
 	attr_accessor :files, :fat
 
-	def initialize(endianness=:little)
-		@endianness = endianness
-		@encoded = EncodedData.new
+	def initialize(cpu=nil)
+		@endianness = (cpu ? cpu.endianness : :little)
+		super(cpu)
 	end
 
 	# decodes the header from the current offset in self.encoded

data/{lib/metasm → metasm}/exe_format/pe.rb

@@ -11,8 +11,9 @@ require 'metasm/exe_format/coff'
 module Metasm
 class PE < COFF
 	MAGIC = "PE\0\0"	# 0x50450000
+	MAGIC.force_encoding('BINARY') if MAGIC.respond_to?(:force_encoding)
 
-	attr_accessor :coff_offset, :signature, :mz
+	attr_accessor :coff_offset, :signature, :mz, :productid
 
 	def initialize(*a)
 		super(*a)
@@ -26,7 +27,14 @@ class PE < COFF
 	def decode_header
 		@cursection ||= self
 		@encoded.ptr = 0x3c
-		@encoded.ptr = decode_word(@encoded)
+		lfanew = decode_word(@encoded)
+
+		try_rich_sz = [0x400, lfanew].min
+		rich_ary = []
+		rich_ary << decode_word(@encoded) while @encoded.ptr < try_rich_sz
+		@productid = decode_productid(rich_ary)
+
+		@encoded.ptr = lfanew
 		@signature = @encoded.read(4)
 		raise InvalidExeFormat, "Invalid PE signature #{@signature.inspect}" if @signature != MAGIC
 		@coff_offset = @encoded.ptr
@@ -38,6 +46,20 @@ class PE < COFF
 		super()
 	end
 
+	RICH_MAGIC = 0x68636952	# "Rich"
+	def decode_productid(ary)
+		return unless idx = ary.index(RICH_MAGIC) and xorkey = ary[idx+1]
+		ary = ary[0, idx-1].map { |dw| dw ^ xorkey }
+		return unless idx = ary.rindex(0x536E6144)	# "DanS"
+		ary = ary[idx+2..-1]
+		out = []
+		until ary.empty?
+			ar1 = ary.shift
+			out << { :id => ar1 >> 16, :ver => ar1 & 0xffff, :count => ary.shift.to_i }
+		end
+		out
+	end
+
 	# creates a default MZ file to be used in the PE header
 	# this one is specially crafted to fit in the 0x3c bytes before the signature
 	def encode_default_mz_header
@@ -215,7 +237,7 @@ EOS
 	# TODO seh prototype (args => context)
 	# TODO hook on (non)resolution of :w xref
 	def get_xrefs_x(dasm, di)
-		if @cpu.shortname =~ /ia32|x64/ and a = di.instruction.args.first and a.kind_of? Ia32::ModRM and a.seg and a.seg.val == 4 and
+		if @cpu.shortname =~ /^ia32|^x64/ and a = di.instruction.args.first and a.kind_of?(Ia32::ModRM) and a.seg and a.seg.val == 4 and
 				w = get_xrefs_rw(dasm, di).find { |type, ptr, len| type == :w and ptr.externals.include? 'segment_base_fs' } and
 				dasm.backtrace(Expression[w[1], :-, 'segment_base_fs'], di.address).to_a.include?(Expression[0])
 			sehptr = w[1]
@@ -225,8 +247,8 @@ EOS
 			puts "backtrace seh from #{di} => #{a.map { |addr| Expression[addr] }.join(', ')}" if $VERBOSE
 			a.each { |aa|
 				next if aa == Expression::Unknown
-				l = dasm.auto_label_at(aa, 'seh', 'loc', 'sub')
-				dasm.addrs_todo << [aa]
+				dasm.auto_label_at(aa, 'seh', 'loc', 'sub')
+				dasm.addrs_todo << { :addr => aa }
 			}
 			super(dasm, di)
 		else
@@ -243,17 +265,19 @@ EOS
 			old_cp = d.c_parser
 			d.c_parser = nil
 			d.parse_c '__stdcall void *GetProcAddress(int, char *);'
-			d.c_parser.lexer.define_weak('__MS_X86_64_ABI__') if @cpu.kind_of? X86_64
+			d.parse_c '__stdcall void ExitProcess(int) __attribute__((noreturn));'
+			d.c_parser.lexer.define_weak('__MS_X86_64_ABI__') if @cpu.shortname == 'x64'
 			gpa = @cpu.decode_c_function_prototype(d.c_parser, 'GetProcAddress')
+			epr = @cpu.decode_c_function_prototype(d.c_parser, 'ExitProcess')
 			d.c_parser = old_cp
 			d.parse_c ''
-			d.c_parser.lexer.define_weak('__MS_X86_64_ABI__') if @cpu.kind_of? X86_64
+			d.c_parser.lexer.define_weak('__MS_X86_64_ABI__') if @cpu.shortname == 'x64'
 			@getprocaddr_unknown = []
 			gpa.btbind_callback = lambda { |dasm, bind, funcaddr, calladdr, expr, origin, maxdepth|
 				break bind if @getprocaddr_unknown.include? [dasm, calladdr] or not Expression[expr].externals.include? :eax
 				sz = @cpu.size/8
 				break bind if not dasm.decoded[calladdr]
-				if @cpu.kind_of? X86_64
+				if @cpu.shortname == 'x64'
 					arg2 = :rdx
 				else
 					arg2 = Indirection[[:esp, :+, 2*sz], sz, calladdr]
@@ -268,6 +292,7 @@ EOS
 				bind
 			}
 			d.function[Expression['GetProcAddress']] = gpa
+			d.function[Expression['ExitProcess']] = epr
 			d.function[:default] = @cpu.disassembler_default_func
 		end
 		d
@@ -294,6 +319,62 @@ EOS
 		} if export
 		syms
 	end
+
+	# compute the pe-sha1 or pe-sha256 of the binary
+	# argument should be a Digest::SHA1 (from digest/sha1) or a Digest::SHA256 (from digest/sha2)
+	# returns the hex checksum
+	def pehash(digest)
+		off0 = 0
+		off1 = @coff_offset + @header.sizeof(self) + @optheader.offsetof(self, :checksum)
+
+		dir_ct_idx = DIRECTORIES.index('certificate_table')
+		if @optheader.numrva > dir_ct_idx
+			off2 = @coff_offset + @header.sizeof(self) + @optheader.sizeof(self) + 8*dir_ct_idx
+			ct_size = @encoded.data[off2, 8].unpack('V*')[1]
+			off3 = @encoded.length - ct_size
+		else
+			off4 = @encoded.length
+		end
+
+		digest << @encoded.data[off0 ... off1].to_str
+		digest << @encoded.data[off1+4 ... off2].to_str if off2
+		digest << @encoded.data[off2+8 ... off3].to_str if off2 and off3 > off2+8
+		digest << @encoded.data[off1+4 ... off4].to_str if off4
+		digest << ("\0" * (8 - (@encoded.length & 7))) if @encoded.length & 7 != 0
+
+		digest.hexdigest
+	end
+
+	def self.pehash(path, digest)
+		decode_file_header(path).pehash(digest)
+	end
+
+	# compute Mandiant "importhash"
+	def imphash
+		lst = []
+		@imports.to_a.each { |id|
+			ln = id.libname.downcase.sub(/.(dll|sys|ocx)$/, '')
+			id.imports.each { |i|
+				if not i.name and ordtable = WindowsExports::IMPORT_HASH[ln]
+					iname = ordtable[i.ordinal]
+				else
+					iname = i.name
+				end
+				iname ||= "ord#{i.ordinal}"
+
+				lst << "#{ln}.#{iname}"
+			}
+		}
+
+		require 'digest/md5'
+		Digest::MD5.hexdigest(lst.join(',').downcase)
+	end
+
+	def self.imphash(path)
+		pe = decode_file_header(path)
+		pe.decode_imports
+		pe.imphash
+	end
 end
 
 # an instance of a PE file, loaded in memory
@@ -312,7 +393,7 @@ class LoadedPE < PE
 
 	# reads a loaded PE from memory, returns a PE object
 	# dumps the header, optheader and all sections ; try to rebuild IAT (#memdump_imports)
-	def self.memdump(memory, baseaddr, entrypoint = nil, iat_p=nil)
+	def self.memdump(memory, baseaddr, entrypoint=nil, iat_p=nil)
 		loaded = LoadedPE.load memory[baseaddr, 0x1000_0000]
 		loaded.load_address = baseaddr
 		loaded.decode
@@ -372,7 +453,6 @@ class LoadedPE < PE
 			else
 				# read imported pointer from the import structure
 				while not ptr = imports.first.iat.shift
-					load_dll = nil
 					imports.shift
 					break if imports.empty?
 					iat_p = imports.first.iat_p
@@ -415,6 +495,7 @@ class LoadedPE < PE
 					puts 'unknown ptr %x' % ptr if $DEBUG
 					# allow holes in the unk_iat_p table
 					break if not unk_iat_p or failcnt > 4
+					loaded_dll = nil
 					failcnt += 1
 					next
 				end
@@ -422,7 +503,7 @@ class LoadedPE < PE
 			end
 
 			# dumped last importdirectory is correct, append the import field
- 			i = ImportDirectory::Import.new
+			i = ImportDirectory::Import.new
 			if e.name
 				puts e.name if $DEBUG
 				i.name = e.name
@@ -433,5 +514,9 @@ class LoadedPE < PE
 			dump.imports.last.imports << i
 		end
 	end
+
+	def pehash(digest)
+		raise "cannot compute a PEhash from memory image"
+	end
 end
 end

data/metasm/exe_format/pyc.rb

@@ -0,0 +1,167 @@
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+require 'metasm/exe_format/main'
+require 'metasm/encode'
+require 'metasm/decode'
+
+
+module Metasm
+# Python preparsed module (.pyc)
+class PYC < ExeFormat
+	# 1 magic per python version...
+	# file = MAGIC(u16) \r \n timestamp(u32) data
+	MAGICS = [
+		62211 # 62211 = python2.7a0
+	]
+
+	class Header < SerialStruct
+		half :version
+		half :rn
+		word :timestamp
+	end
+
+	def decode_half(edata=@encoded) edata.decode_imm(:u16, @endianness) end
+	def decode_word(edata=@encoded) edata.decode_imm(:u32, @endianness) end
+	def decode_long(edata=@encoded) edata.decode_imm(:i32, @endianness) end
+	def sizeof_half ; 2 ; end
+	def sizeof_word ; 4 ; end
+	def sizeof_long ; 4 ; end
+
+	# file header
+	attr_accessor :header
+	# the marshalled object
+	attr_accessor :root
+	# list of all code objects
+	attr_accessor :all_code
+
+	def initialize()
+		@endianness = :little
+		@encoded = EncodedData.new
+		super()
+	end
+
+	def decode_header
+		@header = Header.decode(self)
+	end
+
+	def decode_pymarshal
+		case c = @encoded.read(1)
+		when '0' # NULL
+			:null
+		when 'N' # None
+			nil
+		when 'F' # False
+			false
+		when 'T' # True
+			true
+		#when 'S' # stopiter TODO
+		#when '.' # ellipsis TODO
+		when 'i' # long (i32)
+			decode_long
+		when 'I' # long (i64)
+			decode_word | (decode_long << 32)
+		when 'f' # float (ascii)
+			@encoded.read(@encoded.read(1).unpack('C').first).to_f
+		when 'g' # float (binary)
+			@encoded.read(8).unpack('d').first	# XXX check
+		when 'x' # complex (f f)
+			{ :type => :complex,
+			  :real => @encoded.read(@encoded.read(1).unpack('C').first).to_f,
+			  :imag => @encoded.read(@encoded.read(1).unpack('C').first).to_f }
+		when 'y' # complex (g g)
+			{ :type => :complex,
+			  :real => @encoded.read(8).unpack('d').first,
+			  :imag => @encoded.read(8).unpack('d').first }
+		when 'l' # long (i32?)
+			decode_long
+		when 's' # string: len (long), data
+			@encoded.read(decode_long)
+		when 't' # 'interned': string with possible backreference later
+			s = @encoded.read(decode_long)
+			@references << s
+			s
+		when 'R' # stringref (see 't')
+			@references[decode_long]
+		when '(' # tuple (frozen Array): length l*objs
+			obj = []
+			decode_long.times { obj << decode_pymarshal }
+			obj
+		when '[' # list (Array)
+			obj = []
+			decode_long.times { obj << decode_pymarshal }
+			obj
+		when '{' # dict (Hash)
+			obj = {}
+			loop do
+				k = decode_pymarshal
+				break if k == :null
+				obj[k] = decode_pymarshal
+			end
+			{ :type => hash, :hash => obj }	# XXX to avoid confusion with code, etc
+		when 'c' # code
+			# XXX format varies with version (header.signature)
+			obj = {}
+			obj[:type] = :code
+			obj[:argcount] = decode_long
+			#obj[:kwonly_argcount] = decode_long	# not in py2.7
+			obj[:nlocals] = decode_long
+			obj[:stacksize] = decode_long
+			obj[:flags] = decode_long	# TODO bit-decode this one
+
+			obj[:fileoff] = @encoded.ptr + 5	# XXX assume :code is a 's'
+			obj[:code] = decode_pymarshal
+			obj[:consts] = decode_pymarshal
+			obj[:names] = decode_pymarshal
+			obj[:varnames] = decode_pymarshal
+			obj[:freevars] = decode_pymarshal
+			obj[:cellvars] = decode_pymarshal
+			obj[:filename] = decode_pymarshal
+			obj[:name] = decode_pymarshal
+			obj[:firstlineno] = decode_long
+			obj[:lnotab] = decode_pymarshal
+			@all_code << obj
+			obj
+		when 'u' # unicode
+			@encoded.read(decode_long)
+		#when '?' # unknown TODO
+		#when '<' # set TODO
+		#when '>' # set (frozen) TODO
+		else
+			raise "unsupported python marshal #{c.inspect}"
+		end
+	end
+
+	def decode
+		decode_header
+		@all_code = []
+		@references = []
+		@root = decode_pymarshal
+		@references = nil
+	end
+
+	def cpu_from_headers
+		Python.new(self)
+	end
+
+	def each_section
+		yield @encoded, 0
+	end
+
+	def get_default_entrypoints
+		if @root.kind_of? Hash and @root[:type] == :code
+			[@root[:fileoff]]
+		else
+			[]
+		end
+	end
+
+	# return the :code part which contains off
+	def code_at_off(off)
+		@all_code.find { |c| c[:fileoff] <= off and c[:fileoff] + c[:code].length > off }
+	end
+end
+end

data/{lib/metasm → metasm}/exe_format/serialstruct.rb

@@ -13,19 +13,20 @@ class SerialStruct
 	NAME=0
 	DECODE=1
 	ENCODE=2
-	DEFVAL=3
-	ENUM=4
-	BITS=5
+	SIZEOF=3
+	DEFVAL=4
+	ENUM=5
+	BITS=6
 
 class << self
 	# defines a new field
 	# adds an accessor
-	def new_field(name, decode, encode, defval, enum=nil, bits=nil)
+	def new_field(name, decode, encode, sizeof, defval, enum=nil, bits=nil)
 		if name
 			attr_accessor name
 			name = "@#{name}".to_sym
 		end
-		(@@fields[self] ||= []) << [name, decode, encode, defval, enum, bits]
+		(@@fields[self] ||= []) << [name, decode, encode, sizeof, defval, enum, bits]
 	end
 
 	# creates a field constructor for a simple integer
@@ -34,7 +35,7 @@ class << self
 		recv = class << self ; self ; end
 		types.each { |type|
 			recv.send(:define_method, type) { |name, *args|
-				new_field(name, "decode_#{type}".to_sym, "encode_#{type}".to_sym, args[0] || 0, args[1])
+				new_field(name, "decode_#{type}".to_sym, "encode_#{type}".to_sym, "sizeof_#{type}".to_sym, args[0] || 0, args[1])
 			}
 
 			# shortcut to define multiple fields of this type with default values
@@ -46,24 +47,33 @@ class << self
 
 	# standard fields:
 
+	# virtual field, handled explicitly in a custom encode/decode
+	def virtual(*a)
+		a.each { |f|
+			new_field(f, nil, nil, nil, nil)
+		}
+	end
+
 	# a fixed-size memory chunk
 	def mem(name, len, defval='')
-		new_field(name, lambda { |exe, me| exe.curencoded.read(len) }, lambda { |exe, me, val| val[0, len].ljust(len, 0.chr) }, defval)
+		new_field(name, lambda { |exe, me| exe.curencoded.read(len) }, lambda { |exe, me, val| d = val[0, len].ljust(len, 0.chr) ; d.force_encoding('BINARY') if d.respond_to?(:force_encoding) ; d }, lambda { |exe, me| len }, defval)
 	end
 	# a fixed-size string, 0-padded
 	def str(name, len, defval='')
-		e = lambda { |exe, me, val| val[0, len].ljust(len, 0.chr) }
 		d = lambda { |exe, me| v = exe.curencoded.read(len) ; v = v[0, v.index(?\0)] if v.index(?\0) ; v }
-		new_field(name, d, e, defval)
+		e = lambda { |exe, me, val| val[0, len].ljust(len, 0.chr) }
+		s = lambda { |exe, me| len }
+		new_field(name, d, e, s, defval)
 	end
 	# 0-terminated string
 	def strz(name, defval='')
 		d = lambda { |exe, me|
-		       	ed = exe.curencoded
+			ed = exe.curencoded
 			ed.read(ed.data.index(?\0, ed.ptr)-ed.ptr+1).chop
 		}
 		e = lambda { |exe, me, val| val + 0.chr }
-		new_field(name, d, e, defval)
+		s = lambda { |exe, val| val.length + 1 }
+		new_field(name, d, e, s, defval)
 	end
 
 	# field access
@@ -93,7 +103,7 @@ class << self
 		d = lambda { |exe, me| @bitfield_val = exe.send("decode_#{inttype}") }
 		# reset a temp var
 		e = lambda { |exe, me, val| @bitfield_val = 0 ; nil }
-		new_field(nil, d, e, nil)
+		new_field(nil, d, e, 0, nil)
 
 		h = h.sort
 		h.length.times { |i|
@@ -107,7 +117,7 @@ class << self
 			d = lambda { |exe, me| (@bitfield_val >> off) & mask }
 			# update the temp var with the field value, return nil
 			e = lambda { |exe, me, val| @bitfield_val |= (val & mask) << off ; nil }
-		       	new_field(name, d, e, 0)
+			new_field(name, d, e, 0, 0)
 		}
 
 		# free the temp var
@@ -118,11 +128,13 @@ class << self
 			@bitfield_val = nil
 			exe.send("encode_#{inttype}", val)
 		}
-		new_field(nil, d, e, nil)
+		s = lambda { |exe, me| exe.send("sizeof_#{inttype}") }
+		new_field(nil, d, e, s, nil)
 	end
 
 	# inject a hook to be run during the decoding process
 	def decode_hook(before=nil, &b)
+		@@fields[self] ||= []
 		idx = (before ? @@fields[self].index(fld_get(before)) : -1)
 		@@fields[self].insert(idx, [nil, b])
 	end
@@ -209,6 +221,39 @@ end	# class methods
 		ed
 	end
 
+	# size of the structure = fields.sum { size of field }
+	def sizeof(exe)
+		struct_fields(exe).inject(0) { |off, f|
+			case sz = f[SIZEOF]
+			when Proc; sz = sz[exe, self]
+			when Symbol; sz = exe.send(sz)
+			when Array; sz = exe.send(*sz)
+			when nil; sz = 0
+			end
+			off + sz
+		}
+	end
+
+	# offset (in bytes) of the structure member
+	# for bitfields, return the byte offset of the whole bitfield
+	def offsetof(exe, fld)
+		fld2 = fld
+		fld2 = "@#{fld}".to_sym if fld.to_s[0] != ?@
+		off = 0
+		struct_fields(exe).each { |f|
+			return off if f[NAME] == fld or f[NAME] == fld2
+
+			case sz = f[SIZEOF]
+			when Proc; sz = sz[exe, self]
+			when Symbol; sz = exe.send(sz)
+			when Array; sz = exe.send(*sz)
+			when nil; sz = 0
+			end
+			off += sz
+		}
+		raise 'unknown field'
+	end
+
 	# shortcut to create a new instance and decode it
 	def self.decode(*a)
 		s = new
@@ -216,6 +261,14 @@ end	# class methods
 		s
 	end
 
+	def self.sizeof(exe)
+		new.sizeof(exe)
+	end
+
+	def self.offsetof(exe, fld)
+		new.offsetof(exe, fld)
+	end
+
 	def dump(e, a)
 		case e
 		when Integer; e >= 0x100 ? '0x%X'%e : e