metasm 1.0.0

data/lib/metasm/x86_64/main.rb

@@ -0,0 +1,135 @@
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+require 'metasm/main'
+require 'metasm/ia32'
+
+module Metasm
+
+# The x86_64, 64-bit extension of the x86 CPU (x64, em64t, amd64...)
+class X86_64 < Ia32
+	# FpReg, SegReg, Farptr unchanged
+	# XXX ST(15) ?
+
+	# Simd extended to 16 regs, xmm only (mmx gone with 80387)
+	class SimdReg < Ia32::SimdReg
+		double_map  64 => (0..15).map { |n| "mm#{n}" },
+   			   128 => (0..15).map { |n| "xmm#{n}" }
+	end
+
+	# general purpose registers, all sizes
+	# 8 new gprs (r8..r15), set bit R in the REX prefix to reference them (or X/B if in ModRM)
+	# aonethusaontehsanothe with 8bit subreg: with no rex prefix, refers to ah ch dh bh (as usual)
+	#  but whenever the prefix is present, those become unavailable and encodie spl..dil (low byte of rsp/rdi)
+	class Reg < Ia32::Reg
+		double_map  8 => %w{ al  cl  dl  bl spl bpl sil dil r8b r9b r10b r11b r12b r13b r14b r15b ah ch dh bh},
+			   16 => %w{ ax  cx  dx  bx  sp  bp  si  di r8w r9w r10w r11w r12w r13w r14w r15w},
+			   32 => %w{eax ecx edx ebx esp ebp esi edi r8d r9d r10d r11d r12d r13d r14d r15d eip},
+			   64 => %w{rax rcx rdx rbx rsp rbp rsi rdi r8  r9  r10  r11  r12  r13  r14  r15  rip}
+
+		Sym = @i_to_s[64].map { |s| s.to_sym }
+
+		# returns a symbolic representation of the register:
+		# cx => :rcx & 0xffff
+		# ah => (:rax >> 8) & 0xff
+		# XXX in x64, 32bits operations are zero-extended to 64bits (eg mov rax, 0x1234_ffff_ffff ; add eax, 1 => rax == 0
+		def symbolic(di=nil)
+			s = Sym[@val]
+			s = di.next_addr if s == :rip and di
+			if @sz == 8 and to_s[-1] == ?h
+				Expression[[Sym[@val-16], :>>, 8], :&, 0xff]
+			elsif @sz == 8
+				Expression[s, :&, 0xff]
+			elsif @sz == 16
+				Expression[s, :&, 0xffff]
+			elsif @sz == 32
+				Expression[s, :&, 0xffffffff]
+			else
+				s
+			end
+		end
+
+		# checks if two registers have bits in common
+		def share?(other)
+			raise 'TODO'
+			# XXX TODO wtf does formula this do ?
+			other.val % (other.sz >> 1) == @val % (@sz >> 1) and (other.sz != @sz or @sz != 8 or other.val == @val)
+		end
+
+		# returns the part of @val to encode in an instruction field
+		def val_enc
+			if @sz == 8 and @val >= 16; @val-12	# ah, bh, ch, dh
+			elsif @val >= 16			# rip
+			else @val & 7				# others
+			end
+		end
+
+		# returns the part of @val to encode in an instruction's rex prefix
+		def val_rex
+			if @sz == 8 and @val >= 16		# ah, bh, ch, dh: rex forbidden
+			elsif @val >= 16			# rip
+			else @val >> 3				# others
+			end
+		end
+	end
+
+	# ModRM represents indirections (eg dword ptr [eax+4*ebx+12h])
+	# 16bit mode unavailable in x64
+	# opcodes use 64bit addressing by default, use adsz override (67h) prefix to switch to 32
+	# immediate values are encoded as :i32 sign-extended to 64bits
+	class ModRM < Ia32::ModRM
+		# mod 0/1/2 m 4 => sib
+		# mod 0 m 5 => rip+imm
+		# sib: i 4 => no index, b 5 => no base
+	end
+
+	class DbgReg < Ia32::DbgReg
+		simple_map((0..15).map { |i| [i, "dr#{i}"] })
+	end
+
+	class CtrlReg < Ia32::CtrlReg
+		simple_map((0..15).map { |i| [i, "cr#{i}"] })
+	end
+
+	# Create a new instance of an X86 cpu
+	# arguments (any order)
+	# - instruction set (386, 486, sse2...) [latest]
+	# - endianness [:little]
+	def initialize(*a)
+		super(:latest)
+		@size = 64
+		a.delete @size
+		@endianness = (a & [:big, :little]).first || :little
+		a.delete @endianness
+		@family = a.pop || :latest
+		raise "Invalid arguments #{a.inspect}" if not a.empty?
+		raise "Invalid X86_64 family #{@family.inspect}" if not respond_to?("init_#@family")
+	end
+
+	# defines some preprocessor macros to say who we are:
+	# TODO
+	def tune_prepro(pp)
+		super(pp, :itsmeX64)	# ask Ia32's to just call super()
+		pp.define_weak('_M_AMD64')
+		pp.define_weak('_M_X64')
+		pp.define_weak('__amd64__')
+		pp.define_weak('__x86_64__')
+	end
+
+	def str_to_reg(str)
+		# X86_64::Reg != Ia32::Reg
+		Reg.from_str(str) if Reg.s_to_i.has_key? str
+	end
+
+	def shortname
+		"x64#{'_be' if @endianness == :big}"
+	end
+end
+
+X64 = X86_64
+AMD64 = X86_64
+
+end

data/lib/metasm/x86_64/opcodes.rb

@@ -0,0 +1,118 @@
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+require 'metasm/x86_64/main'
+require 'metasm/ia32/opcodes'
+
+module Metasm
+class X86_64
+	def init_cpu_constants
+		super()
+		@valid_args.concat [:i32, :u32, :i64, :u64] - @valid_args
+	end
+
+	def init_386_common_only
+		super()
+		# :imm64 => accept a real int64 as :i argument
+		# :auto64 => ignore rex_w, always 64-bit op
+		# :op32no64 => if write to a 32-bit reg, dont zero the top 32-bits of dest
+		@valid_props |= [:imm64, :auto64, :op32no64]
+		@opcode_list.delete_if { |o| o.bin[0].to_i & 0xf0 == 0x40 }	# now REX prefix
+		@opcode_list.each { |o|
+			o.props[:imm64] = true if o.bin == [0xB8]	# mov reg, <true imm64>
+			o.props[:auto64] = true if o.name =~ /^(j|loop|(call|enter|leave|lgdt|lidt|lldt|ltr|pop|push|ret)$)/
+			#o.props[:op32no64] = true if o.name =~ //	# TODO are there any instr here ?
+		}
+		addop 'movsxd', [0x63], :mrm
+	end
+
+	# all x86_64 cpu understand <= sse2 instrs
+	def init_x8664_only
+		init_386_common_only
+		init_386_only
+		init_387_only	# 387 indeed
+		init_486_only
+		init_pentium_only
+		init_p6_only
+		init_sse_only
+		init_sse2_only
+
+		@opcode_list.delete_if { |o|
+			o.args.include?(:seg2) or
+			o.args.include?(:seg2A) or
+			%w[aaa aad aam aas bound daa das
+			 lds les loadall arpl pusha pushad popa popad].include?(o.name)
+		}
+
+		addop 'swapgs',  [0x0F, 0x01, 0xF8]
+	end
+
+	def init_sse3
+		init_x8664_only
+		init_sse3_only
+	end
+
+	def init_vmx
+		init_sse3
+		init_vmx_only
+	end
+	
+	def init_all
+		init_vmx
+		init_sse42_only
+		init_3dnow_only
+	end
+
+	alias init_latest init_all
+
+
+	def addop_macrostr(name, bin, type)
+		super(name, bin, type)
+		bin = bin.dup
+		bin[0] |= 1
+		addop(name+'q', bin) { |o| o.props[:opsz] = 64 ; o.props[type] = true }
+	end
+
+	def addop_post(op)
+		if op.fields[:d] or op.fields[:w] or op.fields[:s] or op.args.first == :regfp0
+			return super(op)
+		end
+
+		dupe = lambda { |o|
+			dop = Opcode.new o.name.dup, o.bin.dup
+ 			dop.fields, dop.props, dop.args = o.fields.dup, o.props.dup, o.args.dup
+			dop
+		}
+
+		@opcode_list << op
+
+		if op.args == [:i] or op.args == [:farptr] or op.name[0, 3] == 'ret'
+			# define opsz-override version for ambiguous opcodes
+			op16 = dupe[op]
+			op16.name << '.i16'
+			op16.props[:opsz] = 16
+			@opcode_list << op16
+			# push call ret jz  can't 32bit
+			op64 = dupe[op]
+			op64.name << '.i64'
+			op64.props[:opsz] = 64
+			@opcode_list << op64
+		elsif op.props[:strop] or op.props[:stropz] or op.args.include? :mrm_imm or
+				op.args.include? :modrm or op.args.include? :modrmA or op.name =~ /loop|xlat/
+			# define adsz-override version for ambiguous opcodes (movsq)
+			# XXX loop pfx 67 = rip+ecx, 66/rex ignored
+			op32 = dupe[op]
+			op32.name << '.a32'
+			op32.props[:adsz] = 32
+			@opcode_list << op32
+			op64 = dupe[op]
+			op64.name << '.a64'
+			op64.props[:adsz] = 64
+			@opcode_list << op64
+		end
+	end
+end
+end

data/lib/metasm/x86_64/parse.rb

@@ -0,0 +1,68 @@
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+require 'metasm/x86_64/opcodes'
+require 'metasm/x86_64/encode'
+require 'metasm/parse'
+
+module Metasm
+class X86_64
+	def parse_parser_instruction(lexer, instr)
+		case instr.raw.downcase
+		when '.mode', '.bits'
+			if tok = lexer.readtok and tok.type == :string and tok.raw == '64'
+				lexer.skip_space
+				raise instr, 'syntax error' if ntok = lexer.nexttok and ntok.type != :eol
+			else
+				raise instr, 'invalid cpu mode, 64bit only'
+			end
+		else super(lexer, instr)
+		end
+	end
+
+	def parse_prefix(i, pfx)
+		super(i, pfx) or (i.prefix[:sz] = 64 if pfx == 'code64')
+	end
+
+	# needed due to how ruby inheritance works wrt constants
+	def parse_argregclasslist
+		[Reg, SimdReg, SegReg, DbgReg, CtrlReg, FpReg]
+	end
+	# same inheritance sh*t
+	def parse_modrm(lex, tok, cpu)
+		ModRM.parse(lex, tok, cpu)
+	end
+
+	def parse_instruction_checkproto(i)
+		# check ah vs rex prefix
+		return if i.args.find { |a| a.kind_of? Reg and a.sz == 8 and a.val >= 16 and
+				op = opcode_list.find { |op_| op_.name == i.opname } and
+				((not op.props[:auto64] and i.args.find { |aa| aa.respond_to? :sz and aa.sz == 64 }) or
+				 i.args.find { |aa| aa.kind_of? Reg and aa.val >= 8 and aa.val < 16 } or	# XXX mov ah, cr12...
+				 i.args.grep(ModRM).find { |aa| (aa.b and aa.b.val >= 8 and aa.b.val < 16) or (aa.i and aa.i.val >= 8 and aa.i.val < 16) })
+			}
+		super(i)
+	end
+
+	# check if the argument matches the opcode's argument spec
+	def parse_arg_valid?(o, spec, arg)
+		return if arg.kind_of? ModRM and ((arg.b and arg.b.val == 16 and arg.i) or (arg.i and arg.i.val == 16 and (arg.b or arg.s != 1)))
+		return if arg.kind_of? Reg and arg.sz >= 32 and arg.val == 16	# eip/rip only in modrm
+		return if o.props[:auto64] and arg.respond_to? :sz and arg.sz == 32
+		if o.name == 'movsxd'
+			return if not arg.kind_of? Reg and not arg.kind_of? ModRM
+			arg.sz ||= 32
+			if spec == :reg
+				return if not arg.kind_of? Reg
+				return arg.sz >= 32
+			else
+				return arg.sz == 32
+			end
+		end
+		super(o, spec, arg)
+	end
+end
+end

data/misc/bottleneck.rb

@@ -0,0 +1,61 @@
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+# A script to help finding performance bottlenecks:
+#
+# $ ruby-prof myscript.rb
+#  => String#+ gets called 50k times and takes 30s
+#
+# $ LOGCALLER='String#+' ruby -r bottleneck myscript.rb
+#  => String#+ called 40k times from:
+#      stuff.rb:42 in Myclass#uglymethod from
+#      stuff.rb:32 in Myclass#initialize
+#
+# now you know what to rewrite
+
+
+
+def log_caller(cls, meth, singleton=false, histlen=nil)
+	histlen ||= ENV.fetch('LOGCALLER_MAXHIST', 16).to_i
+	dec_meth = 'm_' + meth.to_s.gsub(/[^\w]/) { |c| c.unpack('H*')[0] }
+	malias = dec_meth + '_log_caller'
+	mcntr = '$' + dec_meth + '_counter'
+	eval <<EOS
+
+#{cls.kind_of?(Class) ? 'class' : 'module'} #{cls}
+#{'class << self' if singleton}
+ alias #{malias} #{meth}
+
+ def #{meth}(*a, &b)
+  #{mcntr}[caller[0, #{histlen}]] += 1
+  #{malias}(*a, &b)
+ end
+
+#{'end' if singleton}
+end
+
+#{mcntr} = Hash.new(0)
+
+at_exit {
+	total = #{mcntr}.inject(0) { |a, (k, v)| a+v } 
+	puts "\#{total} callers of #{cls} #{meth}:"
+	#{mcntr}.sort_by { |k, v|
+		-v
+	}[0, 4].each { |k, v|
+		puts " \#{'%.2f%%' % (100.0*v/total)} - \#{v} times from", k, ''
+	}
+}
+
+EOS
+
+end
+
+ENV['LOGCALLER'].to_s.split(';').map { |lc|
+	next if not lc =~ /^(.*)([.#])(.*)$/
+	cls, sg, meth = $1, $2, $3.to_sym
+	sg = { '.' => true, '#' => false }[sg]
+	cls = cls.split('::').inject(::Object) { |o, cst| o.const_get(cst) }
+	log_caller(cls, meth, sg)
+}

data/misc/cheader-findpppath.rb

@@ -0,0 +1,58 @@
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+#
+# shows the preprocessor path to find a specific line
+# usage: ruby chdr-find.rb 'regex pattern' list of files.h
+#
+
+def gets
+	l = $ungets
+	$ungets = nil
+	l || super()
+end
+
+def parse(root=false)
+	want = false
+	ret = []
+	while l = gets
+		case l = l.strip
+		when /^#if/
+			ret << l
+			r = parse(true)
+			if r.empty?
+				ret.pop
+			else
+				want = true
+				rr = r.pop
+				ret.concat r.map { |l_| (l_[0,3] == '#el' ? ' ' : '    ') << l_ }
+				ret << rr
+			end
+		when /^#el/
+			if not root
+				$ungets = l
+				break
+			end
+			ret << l
+			r = parse
+			want = true if not r.empty?
+			ret.concat r
+		when /^#endif/
+			if not root
+				$ungets = l
+				break
+			end
+			ret << l
+			break
+		when /#$srch/ #, /^#include/
+			want = true
+			ret << l
+		end
+	end
+	want ? ret : []
+end
+
+$srch = ARGV.shift
+puts parse

data/misc/hexdiff.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+
+synclen = 6
+ctxlen = 16
+
+file1 = ('x'*ctxlen) + File.read(ARGV.shift)
+file2 = ('x'*ctxlen) + File.read(ARGV.shift)
+
+count1 = count2 = ctxlen
+
+# prints the string in 80 cols
+# with the first column filled with +pfx+
+def show(pfx, str)
+	loop do
+		if str.length > 79
+			len = 79 - str[0...79][/\S+$/].to_s.length
+			len = 79 if len == 0
+			puts pfx + str[0...len]
+			str = str[len..-1]
+		else
+			puts pfx + str
+			break
+		end
+	end
+end
+
+loop do
+	w1 = file1[count1]
+	w2 = file2[count2]
+	break if not w1 and not w2
+	if w1 == w2
+		count1 += 1
+		count2 += 1
+	else
+		diff1 = diff2 = nil
+		catch(:resynced) {
+		1000.times { |depth|
+			(-depth..depth).map { |d|
+				if d == 0
+					[depth, depth]
+				elsif d < 0
+					[depth, depth+d]
+				elsif d > 0
+					[depth-d, depth]
+				end
+			}.each { |dc1, dc2|
+				next if !(0...synclen).all? { |i| file1[count1 + dc1 + i] == file2[count2 + dc2 + i] }
+
+				puts "@#{(count1-ctxlen).to_s 16} #{(count2-ctxlen).to_s 16}"
+				show ' ', file1[count1-ctxlen, ctxlen].inspect
+				if dc1 > 0
+					show '-', file1[count1, dc1].inspect
+				end
+				if dc2 > 0
+					show '+', file2[count2, dc2].inspect
+				end
+				count1 += dc1
+				count2 += dc2
+				show ' ', file1[count1, ctxlen].inspect
+				puts
+
+				throw :resynced
+			}
+		}
+		raise 'nomatch..'
+		}
+	end
+end