metasm 1.0.1 → 1.0.2

data/{lib/metasm → metasm}/preprocessor.rb

@@ -84,7 +84,7 @@ class Preprocessor
 		# modifies the list, returns an array of list of tokens/nil
 		# handles nesting
 		def self.parse_arglist(lexer, list=nil)
-			readtok = lambda { list ? list.shift : lexer.readtok(false) }
+			readtok = lambda { list ? list.shift : lexer.readtok_nopp }
 			unreadtok = lambda { |t| list ? (list.unshift(t) if t) : lexer.unreadtok(t) }
 			tok = nil
 			unreadlist = []
@@ -378,6 +378,7 @@ class Preprocessor
 	# hash filename => file content
 	attr_accessor :hooked_include
 	attr_accessor :warn_redefinition
+	attr_accessor :may_preprocess
 
 	# global default search directory for #included <files>
 	@@include_search_path = ['/usr/include']
@@ -385,18 +386,14 @@ class Preprocessor
 	def self.include_search_path=(np) @@include_search_path=np end
 
 	def initialize(text='')
-		@queue = []
 		@backtrace = []
 		@definition = %w[__FILE__ __LINE__ __COUNTER__ __DATE__ __TIME__].inject({}) { |h, n| h.update n => SpecialMacro.new(n) }
 		@include_search_path = @@include_search_path.dup
 		# stack of :accept/:discard/:discard_all/:testing, represents the current nesting of #if..#endif
 		@ifelse_nesting = []
-		@text = text
-		@pos = 0
-		@filename = 'unknown'
-		@lineno = 1
 		@warn_redefinition = true
 		@hooked_include = {}
+		@may_preprocess = false
 		@pragma_once = {}
 		@pragma_callback = lambda { |otok|
 			tok = otok
@@ -405,6 +402,7 @@ class Preprocessor
 			unreadtok tok
 			puts otok.exception("unhandled pragma #{str.inspect}").message if $VERBOSE
 		}
+		feed!(text)
 		define '__METASM__', VERSION
 	end
 
@@ -493,11 +491,16 @@ class Preprocessor
 	def feed!(text, filename='unknown', lineno=1)
 		raise ArgumentError, 'need something to parse!' if not text
 		@text = text
+		if not @may_preprocess and (@text =~ /^\s*(#|\?\?=)/ or (not @definition.empty? and
+				 @text =~ /#{@definition.keys.map { |k| Regexp.escape(k) }.join('|')}/))
+			@may_preprocess = true
+		end
 		# @filename[-1] used in trace_macros to distinguish generic/specific files
 		@filename = "\"#{filename}\""
 		@lineno = lineno
 		@pos = 0
 		@queue = []
+		@backtrace = []
 		self
 	end
 
@@ -512,7 +515,7 @@ class Preprocessor
 
 	# reads one character from self.text
 	# updates self.lineno
-	# handles trigraphs and \-continued lines
+	# handles \-continued lines
 	def getchar
 		@ungetcharpos = @pos
 		@ungetcharlineno = @lineno
@@ -520,11 +523,11 @@ class Preprocessor
 		@pos += 1
 
 		# check trigraph
-		if c == ?? and @text[@pos] == ?? and Trigraph[@text[@pos+1]]
-			puts "can i has trigraf plox ??#{c.chr} (#@filename:#@lineno)" if $VERBOSE
-			c = Trigraph[@text[@pos+1]]
-			@pos += 2
-		end
+		#if c == ?? and @text[@pos] == ?? and Trigraph[@text[@pos+1]]
+		#	puts "can i has trigraf plox ??#{c.chr} (#@filename:#@lineno)" if $VERBOSE
+		#	c = Trigraph[@text[@pos+1]]
+		#	@pos += 2
+		#end
 
 		# check line continuation
 		# TODO portability
@@ -567,9 +570,9 @@ class Preprocessor
 	end
 
 	# calls readtok_nopp and handles preprocessor directives
-	def readtok(expand_macros = true)
-		lastpos = @pos
+	def readtok
 		tok = readtok_nopp
+		return tok if not @may_preprocess	# shortcut
 
 		if not tok
 			# end of file: resume parent
@@ -579,32 +582,41 @@ class Preprocessor
 				tok = readtok
 			end
 
-		elsif (tok.type == :eol or lastpos == 0) and @ifelse_nesting.last != :testing
-			unreadtok tok if lastpos == 0
-			# detect preprocessor directive
-			# state = 1 => seen :eol, 2 => seen #
+		elsif tok.type == :punct and tok.raw == '#' and not tok.expanded_from and @ifelse_nesting.last != :testing
+			# backward check for :eol (skip the '#' itself)
+			pos = @pos-2
+			while pos >= 0		# if reach start of file, proceed
+				case @text[pos, 1]
+				when "\n"
+					pos -= 1 if pos > 0 and @text[pos-1] == ?\r
+					return tok if pos > 0 and @text[pos-1] == ?\\	# check if the newline was a line-continuation
+					return tok if pos > 2 and @text[pos-3, 3] == '??/'	# trigraph
+					break	# proceed
+				when /\s/	# beware switch order, this matches "\n" too
+				else return tok	# false alarm
+				end
+				pos -= 1
+			end
 			pretok = []
 			rewind = true
-			state = 1
-			loop do
-				pretok << (ntok = readtok_nopp)
-				break if not ntok
+			while ntok = readtok_nopp
+				pretok << ntok
 				if ntok.type == :space	# nothing
-				elsif state == 1 and ntok.type == :punct and ntok.raw == '#' and not ntok.expanded_from
-					state = 2
-				elsif state == 2 and ntok.type == :string and not ntok.expanded_from
+					next
+				elsif ntok.type == :string and not ntok.expanded_from
 					rewind = false if preprocessor_directive(ntok)
-					break
-				else break
 				end
+				break
 			end
 			if rewind
 				# false alarm: revert
 				pretok.reverse_each { |t| unreadtok t }
+			else
+				# XXX return :eol ?
+				tok = readtok
 			end
-			tok = readtok if lastpos == 0	# else return the :eol
 
-		elsif expand_macros and tok.type == :string and m = @definition[tok.raw] and not tok.expanded_from.to_a.find { |ef| ef.raw == m.name.raw } and
+		elsif tok.type == :string and m = @definition[tok.raw] and not tok.expanded_from.to_a.find { |ef| ef.raw == m.name.raw } and
 				((m.args and margs = Macro.parse_arglist(self)) or not m.args)
 
 			if defined? @traced_macros and tok.backtrace[-2].to_s[0] == ?" and m.name and m.name.backtrace[-2].to_s[0] == ?<
@@ -637,21 +649,20 @@ class Preprocessor
 		when ?a..?z, ?A..?Z, ?0..?9, ?$, ?_
 			tok.type = :string
 			raw = tok.raw << c
-			loop do
-				case c = getchar
-				when nil; ungetchar; break		# avoids 'no method "coerce" for nil' warning
+			while c = getchar
+				case c
 				when ?a..?z, ?A..?Z, ?0..?9, ?$, ?_
-					raw << c
-				else ungetchar; break
+				else break
 				end
+				raw << c
 			end
+			ungetchar
 
 		when ?\ , ?\t, ?\r, ?\n, ?\f
 			tok.type = ((c == ?\  || c == ?\t) ? :space : :eol)
 			raw = tok.raw << c
-			loop do
-				case c = getchar
-				when nil; break
+			while c = getchar
+				case c
 				when ?\ , ?\t
 				when ?\n, ?\f, ?\r; tok.type = :eol
 				else break
@@ -676,8 +687,7 @@ class Preprocessor
 				tok.type = :space
 				raw << c
 				seenstar = false
-				loop do
-					raise tok, 'unterminated c++ comment' if not c = getchar
+				while c = getchar
 					raw << c
 					case c
 					when ?*; seenstar = true
@@ -685,6 +695,7 @@ class Preprocessor
 					else seenstar = false
 					end
 				end
+				raise tok, 'unterminated c++ comment' if not c
 			else
 				# just a slash
 				ungetchar
@@ -705,6 +716,7 @@ class Preprocessor
 		tok.type = :quoted
 		tok.raw << delimiter
 		tok.value = ''
+		tok.value.force_encoding('binary') if tok.value.respond_to?(:force_encoding)
 		c = nil
 		loop do
 			raise tok, 'unterminated string' if not c = getchar
@@ -767,6 +779,9 @@ class Preprocessor
 	def define(name, value=nil, from=caller.first)
 		from =~ /^(.*?):(\d+)/
 		btfile, btlineno = $1, $2.to_i
+		if not @may_preprocess and @text =~ /#{Regexp.escape name}/
+			@may_preprocess = true
+		end
 		t = Token.new([btfile, btlineno])
 		t.type = :string
 		t.raw = name.dup
@@ -1095,7 +1110,7 @@ class Preprocessor
 			nil while dir = readtok and dir.type == :space
 			raise cmd, 'qstring expected' if not dir or dir.type != :quoted
 			dir = ::File.expand_path dir.value
-			raise cmd, 'invalid path' if not ::File.directory? dir
+			raise cmd, "invalid path #{dir.inspect}" if not ::File.directory? dir
 			@include_search_path.unshift dir
 
 		when 'push_macro', 'pop_macro'

data/{lib/metasm → metasm}/render.rb

@@ -19,8 +19,10 @@ module Renderable
 		r = proc { |e|
 			case e
 			when Expression
-				yield e
 				r[e.lexpr] ; r[e.rexpr]
+				yield e
+			when ExpressionType
+				yield e
 			when Renderable
 				e.render.each { |re| r[re] }
 			end
@@ -64,45 +66,13 @@ end
 
 class Expression
 	include Renderable
-	attr_accessor :render_info
-
-	# this is an accessor to @@render_int, the lambda used to render integers > 10
-	# usage: Expression.render_int = lambda { |e| '0x%x' % e }
-	#  or    Expression.render_int { |e| '0x%x' % e }
-	# XXX the returned string should be suitable for inclusion in a label name etc
-	def self.render_int(&b)
-		if b
-			@@render_int = b
-		else
-			@@render_int
-		end
-	end
-	def self.render_int=(p)
- 		@@render_int = p
-	end
-	@@render_int = nil
 
 	def render_integer(e)
-		if render_info and @render_info[:char]
-			ee = e
-			v = []
-			while ee > 0
-				v << (ee & 0xff)
-				ee >>= 8
-			end
-			v.reverse! if @render_info[:char] == :big
-			if not v.empty? and v.all? { |c| c < 0x7f }
-				# XXX endianness
-				return "'" + v.pack('C*').inspect.gsub("'") { '\\\'' }[1...-1] + "'"
-			end
-		end
 		if e < 0
 			neg = true
 			e = -e
 		end
 		if e < 10; e = e.to_s
-		elsif @@render_int
-			e = @@render_int[e]
 		else
 			e = '%xh' % e
 			e = '0' << e unless (?0..?9).include? e[0]
@@ -114,17 +84,23 @@ class Expression
 	NOSQ1 = NOSQ2 = {:* => [:*], :+ => [:+, :-, :*], :- => [:+, :-, :*]}
 	NOSQ2[:-] = [:*]
 	def render
-		l = @lexpr.kind_of?(Integer) ? render_integer(@lexpr) : @lexpr
-		r = @rexpr.kind_of?(Integer) ? render_integer(@rexpr) : @rexpr
-		l = ['(', l, ')'] if @lexpr.kind_of? Expression and (not oa = NOSQ1[@op] or not oa.include?(@lexpr.op))
-		r = ['(', r, ')'] if @rexpr.kind_of? Expression and (not oa = NOSQ2[@op] or not oa.include?(@rexpr.op))
+		l = @lexpr.kind_of?(::Integer) ? render_integer(@lexpr) : @lexpr
+		r = @rexpr.kind_of?(::Integer) ? render_integer(@rexpr) : @rexpr
+		l = ['(', l, ')'] if @lexpr.kind_of?(Expression) and (not oa = NOSQ1[@op] or not oa.include?(@lexpr.op))
+		r = ['(', r, ')'] if @rexpr.kind_of?(Expression) and (not oa = NOSQ2[@op] or not oa.include?(@rexpr.op))
 		op = @op if l or @op != :+
 		if op == :+
 			r0 = [r].flatten.first
 			r0 = r0.render.flatten.first while r0.kind_of? Renderable
-			op = nil if (r0.kind_of? Integer and r0 < 0) or (r0.kind_of? String and r0[0] == ?-) or r0 == :-
+			op = nil if (r0.kind_of?(::Integer) and r0 < 0) or (r0.kind_of?(::String) and r0[0] == ?-) or r0 == :-
 		end
 		[l, op, r].compact
 	end
 end
+
+class ExpressionString
+	include Renderable
+
+	def render; hide_str ? @expr.render : render_str ; end
+end
 end

data/misc/hexdump.rb

@@ -51,5 +51,6 @@ if $0 == __FILE__
 	fmt << 'd' if ARGV.delete '-D'
 	fmt << 'a' if ARGV.delete '-A'
 	fmt = ['c', 'd', 'a'] if ARGV.delete '-a'
-	File.open(ARGV.first, 'rb').hexdump(:fmt => fmt)
+	infd = ARGV.empty? ? $stdin : File.open(ARGV.first, 'rb')
+	infd.hexdump(:fmt => fmt)
 end

data/misc/lint.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/ruby
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+# this is a ruby code cleaner tool
+# it passes its argument to ruby -v -c, which displays warnings (eg unused variable)
+# it shows the incriminated line along the warning, to help identify false positives
+# probably linux-only, and need ruby-1.9.1 or newer
+
+def lint(tg)
+	if File.symlink?(tg)
+		# nothing
+	elsif File.directory?(tg)
+		Dir.entries(tg).each { |ent|
+			next if ent == '.' or ent == '..'
+			ent = File.join(tg, ent)
+			lint(ent) if File.directory?(ent) or ent =~ /\.rb$/
+		}
+	else
+		lint_file(tg)
+	end
+end
+
+def lint_file(tg)
+	flines = nil
+	compile_warn(tg).each_line { |line|
+		file, lineno, warn = line.split(/\s*:\s*/, 3)
+		if file == tg
+			if not flines
+				puts "#{tg}:"
+				flines = File.readlines(file) #File.open(file, 'rb') { |fd| fd.readlines }
+			end
+			puts " l.#{lineno}: #{warn.strip}: #{flines[lineno.to_i-1].strip.inspect}"
+		end
+	}
+	puts if flines
+end
+
+def compile_warn(tg)
+	r, w = IO.pipe('binary')
+	if !fork
+		r.close
+		$stderr.reopen w
+		$stdout.reopen '/dev/null'
+		exec 'ruby', '-v', '-c', tg
+		exit!
+	else
+		w.close
+	end
+	r
+end
+
+ARGV << '.' if ARGV.empty?
+ARGV.each { |arg| lint arg }
+

data/misc/txt2html.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# encoding: binary (rage)
 #    This file is part of Metasm, the Ruby assembly manipulation suite
 #    Copyright (C) 2006-2009 Yoann GUILLOT
 #
@@ -48,7 +49,7 @@ class Elem
 			if e.class.ancestors.include? Elem
 				@content << e
 			else
-				@content << e.to_s.gsub(Regexp.new("(#{@@quotechars.keys.join('|')})")) { |x| @@quotechars[x] }
+				@content << e.to_s.gsub(Regexp.new("(#{@@quotechars.keys.join('|')})", 'm')) { |x| @@quotechars[x] }
 			end
 		}
 		self
@@ -273,7 +274,7 @@ class Txt2Html
 		puts "compiling #{outf}..." if $VERBOSE
 
 		@pathfix = outf.split('/')[0...-1].map { '../' }.join
-		out = compile(File.read(f).gsub("\r", '') + "\n\n")
+		out = compile(File.open(f, 'rb') { |fd| fd.read }.gsub("\r", '') + "\n\n")
 		File.open(outf, 'wb') { |fd| fd.write out.to_s.gsub("\r", '').gsub("\n", "\r\n") }
 	end
 
@@ -368,13 +369,13 @@ class Txt2Html
 		out
 	end
 
-	# handle **bold_words** *italic* `fixed` <links>
+	# handle **bold_words** *italic* `fixed` <links> **bold__word__with__underscore**
 	def compile_string(str)
 		o = [str]
 		on = []
 		o.each { |s|
 			while s.kind_of? String and o1 = s.index('**') and o2 = s.index('**', o1+2) and not s[o1..o2].index(' ')
-				on << s[0...o1] << Html::Elem.new('b').add(s[o1+2...o2].tr('_', ' '))
+				on << s[0...o1] << Html::Elem.new('b').add(s[o1+2...o2].tr('_', ' ').gsub('  ', '_'))
 				s = s[o2+2..-1]
 			end
 			on << s
@@ -383,7 +384,7 @@ class Txt2Html
 		on = []
 		o.each { |s|
 			while s.kind_of? String and o1 = s.index('*') and o2 = s.index('*', o1+1) and not s[o1..o2].index(' ')
-				on << s[0...o1] << Html::Elem.new('i').add(s[o1+1...o2].tr('_', ' '))
+				on << s[0...o1] << Html::Elem.new('i').add(s[o1+1...o2].tr('_', ' ').gsub('  ', '_'))
 				s = s[o2+1..-1]
 			end
 			on << s
@@ -409,19 +410,20 @@ class Txt2Html
 					when 'txt'
 						tg = outfilename(lnk)
 						Txt2Html.new(lnk)
-						on << Html::A.new(@pathfix + tg, File.basename(lnk, '.txt').tr('_', ' '))
+						on << Html::A.new(@pathfix + tg, File.basename(lnk, '.txt').tr('_', ' ').gsub('  ', '_'))
 					when 'jpg', 'png'
 						on << Html::Img.new(lnk)
 					end
 				else
+					on << Html::A.new(lnk, lnk)
 					if lnk =~ /\.txt$/
 						@@seen_nofile ||= []
 						if not @@seen_nofile.include? lnk
 							@@seen_nofile << lnk
 							puts "reference to missing #{lnk.inspect}"
 						end
+						on.last.hclass('brokenlink')
 					end
-					on << Html::A.new(lnk, lnk)
 				end
 			end
 			on << s

data/samples/bindiff.rb

@@ -294,7 +294,6 @@ class BinDiffWidget < Gui::DrawableWidget
 		set_status('match funcs') {
 		# refine the layout matching with actual function matching
 		already_matched = []
-		match_score = {}
 		layout_match.each { |f1, list|
 puts "matching #{Expression[f1]}" if $VERBOSE
 begin
@@ -429,8 +428,8 @@ end
 	# show in window 1 the match of the function found in win 2
 	def sync1
 		c2 = curfunc2
-		if a1 = match_funcs.find { |k, (a2, s)| a2 == c2 }
-			@dasm1.gui.focus_addr(a1[0])
+		if a1 = match_funcs.find_key { |k| match_funcs[k][0] == c2 }
+			@dasm1.gui.focus_addr(a1)
 		end
 	end
 
@@ -491,7 +490,7 @@ $VERBOSE = true
 # parse arguments
 opts = {}
 OptionParser.new { |opt|
-	opt.banner = 'Usage: bindiff.rb [options] <executable> [<entrypoints>]'
+	opt.banner = 'Usage: bindiff.rb [options] <exe1> <exe2> [<entrypoints>]'
 	opt.on('-P <plugin>', '--plugin <plugin>', 'load a metasm disassembler plugin') { |h| (opts[:plugin] ||= []) << h }
 	opt.on('-e <code>', '--eval <code>', 'eval a ruby code') { |h| (opts[:hookstr] ||= []) << h }
 	opt.on('--map1 <mapfile>', 'load a map file (addr <-> name association)') { |f| opts[:map1] = f }