metasm 1.0.0 → 1.0.5

data/{lib/metasm → metasm}/preprocessor.rb

@@ -84,7 +84,7 @@ class Preprocessor
 		# modifies the list, returns an array of list of tokens/nil
 		# handles nesting
 		def self.parse_arglist(lexer, list=nil)
-			readtok = lambda { list ? list.shift : lexer.readtok(false) }
+			readtok = lambda { list ? list.shift : lexer.readtok_nopp }
 			unreadtok = lambda { |t| list ? (list.unshift(t) if t) : lexer.unreadtok(t) }
 			tok = nil
 			unreadlist = []
@@ -378,6 +378,7 @@ class Preprocessor
 	# hash filename => file content
 	attr_accessor :hooked_include
 	attr_accessor :warn_redefinition
+	attr_accessor :may_preprocess
 
 	# global default search directory for #included <files>
 	@@include_search_path = ['/usr/include']
@@ -385,18 +386,14 @@ class Preprocessor
 	def self.include_search_path=(np) @@include_search_path=np end
 
 	def initialize(text='')
-		@queue = []
 		@backtrace = []
 		@definition = %w[__FILE__ __LINE__ __COUNTER__ __DATE__ __TIME__].inject({}) { |h, n| h.update n => SpecialMacro.new(n) }
 		@include_search_path = @@include_search_path.dup
 		# stack of :accept/:discard/:discard_all/:testing, represents the current nesting of #if..#endif
 		@ifelse_nesting = []
-		@text = text
-		@pos = 0
-		@filename = 'unknown'
-		@lineno = 1
 		@warn_redefinition = true
 		@hooked_include = {}
+		@may_preprocess = false
 		@pragma_once = {}
 		@pragma_callback = lambda { |otok|
 			tok = otok
@@ -405,6 +402,7 @@ class Preprocessor
 			unreadtok tok
 			puts otok.exception("unhandled pragma #{str.inspect}").message if $VERBOSE
 		}
+		feed!(text)
 		define '__METASM__', VERSION
 	end
 
@@ -493,11 +491,16 @@ class Preprocessor
 	def feed!(text, filename='unknown', lineno=1)
 		raise ArgumentError, 'need something to parse!' if not text
 		@text = text
+		if not @may_preprocess and (@text =~ /^\s*(#|\?\?=)/ or (not @definition.empty? and
+				 @text =~ /#{@definition.keys.map { |k| Regexp.escape(k) }.join('|')}/))
+			@may_preprocess = true
+		end
 		# @filename[-1] used in trace_macros to distinguish generic/specific files
 		@filename = "\"#{filename}\""
 		@lineno = lineno
 		@pos = 0
 		@queue = []
+		@backtrace = []
 		self
 	end
 
@@ -512,7 +515,7 @@ class Preprocessor
 
 	# reads one character from self.text
 	# updates self.lineno
-	# handles trigraphs and \-continued lines
+	# handles \-continued lines
 	def getchar
 		@ungetcharpos = @pos
 		@ungetcharlineno = @lineno
@@ -520,11 +523,11 @@ class Preprocessor
 		@pos += 1
 
 		# check trigraph
-		if c == ?? and @text[@pos] == ?? and Trigraph[@text[@pos+1]]
-			puts "can i has trigraf plox ??#{c.chr} (#@filename:#@lineno)" if $VERBOSE
-			c = Trigraph[@text[@pos+1]]
-			@pos += 2
-		end
+		#if c == ?? and @text[@pos] == ?? and Trigraph[@text[@pos+1]]
+		#	puts "can i has trigraf plox ??#{c.chr} (#@filename:#@lineno)" if $VERBOSE
+		#	c = Trigraph[@text[@pos+1]]
+		#	@pos += 2
+		#end
 
 		# check line continuation
 		# TODO portability
@@ -567,9 +570,9 @@ class Preprocessor
 	end
 
 	# calls readtok_nopp and handles preprocessor directives
-	def readtok(expand_macros = true)
-		lastpos = @pos
+	def readtok
 		tok = readtok_nopp
+		return tok if not @may_preprocess	# shortcut
 
 		if not tok
 			# end of file: resume parent
@@ -579,32 +582,41 @@ class Preprocessor
 				tok = readtok
 			end
 
-		elsif (tok.type == :eol or lastpos == 0) and @ifelse_nesting.last != :testing
-			unreadtok tok if lastpos == 0
-			# detect preprocessor directive
-			# state = 1 => seen :eol, 2 => seen #
+		elsif tok.type == :punct and tok.raw == '#' and not tok.expanded_from and @ifelse_nesting.last != :testing
+			# backward check for :eol (skip the '#' itself)
+			pos = @pos-2
+			while pos >= 0		# if reach start of file, proceed
+				case @text[pos, 1]
+				when "\n"
+					pos -= 1 if pos > 0 and @text[pos-1] == ?\r
+					return tok if pos > 0 and @text[pos-1] == ?\\	# check if the newline was a line-continuation
+					return tok if pos > 2 and @text[pos-3, 3] == '??/'	# trigraph
+					break	# proceed
+				when /\s/	# beware switch order, this matches "\n" too
+				else return tok	# false alarm
+				end
+				pos -= 1
+			end
 			pretok = []
 			rewind = true
-			state = 1
-			loop do
-				pretok << (ntok = readtok_nopp)
-				break if not ntok
+			while ntok = readtok_nopp
+				pretok << ntok
 				if ntok.type == :space	# nothing
-				elsif state == 1 and ntok.type == :punct and ntok.raw == '#' and not ntok.expanded_from
-					state = 2
-				elsif state == 2 and ntok.type == :string and not ntok.expanded_from
+					next
+				elsif ntok.type == :string and not ntok.expanded_from
 					rewind = false if preprocessor_directive(ntok)
-					break
-				else break
 				end
+				break
 			end
 			if rewind
 				# false alarm: revert
 				pretok.reverse_each { |t| unreadtok t }
+			else
+				# XXX return :eol ?
+				tok = readtok
 			end
-			tok = readtok if lastpos == 0	# else return the :eol
 
-		elsif expand_macros and tok.type == :string and m = @definition[tok.raw] and not tok.expanded_from.to_a.find { |ef| ef.raw == m.name.raw } and
+		elsif tok.type == :string and m = @definition[tok.raw] and not tok.expanded_from.to_a.find { |ef| ef.raw == m.name.raw } and
 				((m.args and margs = Macro.parse_arglist(self)) or not m.args)
 
 			if defined? @traced_macros and tok.backtrace[-2].to_s[0] == ?" and m.name and m.name.backtrace[-2].to_s[0] == ?<
@@ -637,21 +649,20 @@ class Preprocessor
 		when ?a..?z, ?A..?Z, ?0..?9, ?$, ?_
 			tok.type = :string
 			raw = tok.raw << c
-			loop do
-				case c = getchar
-				when nil; ungetchar; break		# avoids 'no method "coerce" for nil' warning
+			while c = getchar
+				case c
 				when ?a..?z, ?A..?Z, ?0..?9, ?$, ?_
-					raw << c
-				else ungetchar; break
+				else break
 				end
+				raw << c
 			end
+			ungetchar
 
 		when ?\ , ?\t, ?\r, ?\n, ?\f
 			tok.type = ((c == ?\  || c == ?\t) ? :space : :eol)
 			raw = tok.raw << c
-			loop do
-				case c = getchar
-				when nil; break
+			while c = getchar
+				case c
 				when ?\ , ?\t
 				when ?\n, ?\f, ?\r; tok.type = :eol
 				else break
@@ -676,8 +687,7 @@ class Preprocessor
 				tok.type = :space
 				raw << c
 				seenstar = false
-				loop do
-					raise tok, 'unterminated c++ comment' if not c = getchar
+				while c = getchar
 					raw << c
 					case c
 					when ?*; seenstar = true
@@ -685,6 +695,7 @@ class Preprocessor
 					else seenstar = false
 					end
 				end
+				raise tok, 'unterminated c++ comment' if not c
 			else
 				# just a slash
 				ungetchar
@@ -705,6 +716,7 @@ class Preprocessor
 		tok.type = :quoted
 		tok.raw << delimiter
 		tok.value = ''
+		tok.value.force_encoding('binary') if tok.value.respond_to?(:force_encoding)
 		c = nil
 		loop do
 			raise tok, 'unterminated string' if not c = getchar
@@ -767,6 +779,9 @@ class Preprocessor
 	def define(name, value=nil, from=caller.first)
 		from =~ /^(.*?):(\d+)/
 		btfile, btlineno = $1, $2.to_i
+		if not @may_preprocess and @text =~ /#{Regexp.escape name}/
+			@may_preprocess = true
+		end
 		t = Token.new([btfile, btlineno])
 		t.type = :string
 		t.raw = name.dup
@@ -1095,7 +1110,7 @@ class Preprocessor
 			nil while dir = readtok and dir.type == :space
 			raise cmd, 'qstring expected' if not dir or dir.type != :quoted
 			dir = ::File.expand_path dir.value
-			raise cmd, 'invalid path' if not ::File.directory? dir
+			raise cmd, "invalid path #{dir.inspect}" if not ::File.directory? dir
 			@include_search_path.unshift dir
 
 		when 'push_macro', 'pop_macro'
@@ -1161,7 +1176,7 @@ class Preprocessor
 				op = op.dup
 				op.raw << ntok.raw
 			# ok
-			when '^', '+', '-', '*', '/', '%', '>>', '<<', '>=', '<=', '||', '&&', '!=', '=='
+			when '^', '+', '-', '*', '/', '%', '>>', '<<', '>=', '<=', '||', '&&', '!=', '==', '?'
 			# unknown
 			else
 				lexer.unreadtok tok
@@ -1258,6 +1273,21 @@ class Preprocessor
 					stack << Expression.new(opstack.pop, stack.pop, stack.pop)
 				end
 
+				if op.value == :'?'
+					a1 = parse(lexer)
+					if not tok = lexer.readtok or tok.type != :punct or tok.raw != ':'
+						raise op, 'expected ":" ternary operator'
+					end
+					a2 = parse(lexer)
+					case Expression[stack.pop].reduce
+					when 0;         stack << a2
+					when ::Integer; stack << a1
+					else;           stack << a2
+					end
+
+					next
+				end
+
 				opstack << op.value
 
 				raise op, 'need rhs' if not e = parse_value(lexer)

data/{lib/metasm → metasm}/render.rb

@@ -19,8 +19,10 @@ module Renderable
 		r = proc { |e|
 			case e
 			when Expression
-				yield e
 				r[e.lexpr] ; r[e.rexpr]
+				yield e
+			when ExpressionType
+				yield e
 			when Renderable
 				e.render.each { |re| r[re] }
 			end
@@ -64,45 +66,13 @@ end
 
 class Expression
 	include Renderable
-	attr_accessor :render_info
-
-	# this is an accessor to @@render_int, the lambda used to render integers > 10
-	# usage: Expression.render_int = lambda { |e| '0x%x' % e }
-	#  or    Expression.render_int { |e| '0x%x' % e }
-	# XXX the returned string should be suitable for inclusion in a label name etc
-	def self.render_int(&b)
-		if b
-			@@render_int = b
-		else
-			@@render_int
-		end
-	end
-	def self.render_int=(p)
- 		@@render_int = p
-	end
-	@@render_int = nil
 
 	def render_integer(e)
-		if render_info and @render_info[:char]
-			ee = e
-			v = []
-			while ee > 0
-				v << (ee & 0xff)
-				ee >>= 8
-			end
-			v.reverse! if @render_info[:char] == :big
-			if not v.empty? and v.all? { |c| c < 0x7f }
-				# XXX endianness
-				return "'" + v.pack('C*').inspect.gsub("'") { '\\\'' }[1...-1] + "'"
-			end
-		end
 		if e < 0
 			neg = true
 			e = -e
 		end
 		if e < 10; e = e.to_s
-		elsif @@render_int
-			e = @@render_int[e]
 		else
 			e = '%xh' % e
 			e = '0' << e unless (?0..?9).include? e[0]
@@ -114,17 +84,23 @@ class Expression
 	NOSQ1 = NOSQ2 = {:* => [:*], :+ => [:+, :-, :*], :- => [:+, :-, :*]}
 	NOSQ2[:-] = [:*]
 	def render
-		l = @lexpr.kind_of?(Integer) ? render_integer(@lexpr) : @lexpr
-		r = @rexpr.kind_of?(Integer) ? render_integer(@rexpr) : @rexpr
-		l = ['(', l, ')'] if @lexpr.kind_of? Expression and (not oa = NOSQ1[@op] or not oa.include?(@lexpr.op))
-		r = ['(', r, ')'] if @rexpr.kind_of? Expression and (not oa = NOSQ2[@op] or not oa.include?(@rexpr.op))
+		l = @lexpr.kind_of?(::Integer) ? render_integer(@lexpr) : @lexpr
+		r = @rexpr.kind_of?(::Integer) ? render_integer(@rexpr) : @rexpr
+		l = ['(', l, ')'] if @lexpr.kind_of?(Expression) and (not oa = NOSQ1[@op] or not oa.include?(@lexpr.op))
+		r = ['(', r, ')'] if @rexpr.kind_of?(Expression) and (not oa = NOSQ2[@op] or not oa.include?(@rexpr.op))
 		op = @op if l or @op != :+
 		if op == :+
 			r0 = [r].flatten.first
 			r0 = r0.render.flatten.first while r0.kind_of? Renderable
-			op = nil if (r0.kind_of? Integer and r0 < 0) or (r0.kind_of? String and r0[0] == ?-) or r0 == :-
+			op = nil if (r0.kind_of?(::Integer) and r0 < 0) or (r0.kind_of?(::String) and r0[0] == ?-) or r0 == :-
 		end
 		[l, op, r].compact
 	end
 end
+
+class ExpressionString
+	include Renderable
+
+	def render; hide_str ? @expr.render : render_str ; end
+end
 end

data/misc/hexdump.rb

@@ -29,7 +29,7 @@ def hexdump(ctx={})
 			print s.unpack('C*').map { |b| '%02x' % b }.join(' ').ljust(3*16-1) + '  ' if fmt.include? 'c'
 			print s.unpack('v*').map { |b| '%04x' % b }.join(' ').ljust(5*8-1)  + '  ' if fmt.include? 'w'
 			print s.unpack('L*').map { |b| '%08x' % b }.join(' ').ljust(9*4-1)  + '  ' if fmt.include? 'd'
-			print s.tr("\0-\x1f\x7f-\xff", '.') if fmt.include? 'a'
+			print s.tr("\0-\x1f\x7f-\xff".force_encoding('BINARY'), '.') if fmt.include? 'a'
 			puts
 		elsif not ctx[:lastdup]
 			ctx[:lastdup] = true
@@ -50,6 +50,7 @@ if $0 == __FILE__
 	fmt << 'w' if ARGV.delete '-W'
 	fmt << 'd' if ARGV.delete '-D'
 	fmt << 'a' if ARGV.delete '-A'
-	fmt = ['c', 'd', 'a'] if ARGV.delete '-a'
-	File.open(ARGV.first, 'rb').hexdump(:fmt => fmt)
+	fmt = ['c', 'd', 'a'] if ARGV.delete '-a' or fmt.empty?
+	infd = ARGV.empty? ? $stdin : File.open(ARGV.first, 'rb')
+	infd.hexdump(:fmt => fmt)
 end

data/misc/lint.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/ruby
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+# this is a ruby code cleaner tool
+# it passes its argument to ruby -v -c, which displays warnings (eg unused variable)
+# it shows the incriminated line along the warning, to help identify false positives
+# probably linux-only, and need ruby-1.9.1 or newer
+
+def lint(tg)
+	if File.symlink?(tg)
+		# nothing
+	elsif File.directory?(tg)
+		Dir.entries(tg).each { |ent|
+			next if ent == '.' or ent == '..'
+			ent = File.join(tg, ent)
+			lint(ent) if File.directory?(ent) or ent =~ /\.rb$/
+		}
+	else
+		lint_file(tg)
+	end
+end
+
+def lint_file(tg)
+	flines = nil
+	compile_warn(tg).each_line { |line|
+		file, lineno, warn = line.split(/\s*:\s*/, 3)
+		if file == tg
+			if not flines
+				puts "#{tg}:"
+				flines = File.readlines(file) #File.open(file, 'rb') { |fd| fd.readlines }
+			end
+			puts " l.#{lineno}: #{warn.strip}: #{flines[lineno.to_i-1].strip.inspect}"
+		end
+	}
+	puts if flines
+end
+
+def compile_warn(tg)
+	r, w = IO.pipe('binary')
+	if !fork
+		r.close
+		$stderr.reopen w
+		$stdout.reopen '/dev/null'
+		exec 'ruby', '-v', '-c', tg
+		exit!
+	else
+		w.close
+	end
+	r
+end
+
+ARGV << '.' if ARGV.empty?
+ARGV.each { |arg| lint arg }
+

data/misc/objdiff.rb

@@ -22,7 +22,10 @@ def Object.diff(o1, o2)
 				h["[#{k.inspect}]"] = d if not d.empty?
 			}
 		else
-			a = (@@diff_accessor_cache ||= {})[o1.class] ||= (im = o1.class.public_instance_methods.grep(/^[a-z]/) ; (im & im.map { |m| m + '=' }).map { |m| m.chop }.find_all { |m| o1.instance_variable_get('@'+m) })
+			a = ($diff_accessor_cache ||= {})[o1.class] ||= (
+				im = o1.class.public_instance_methods.map { |m| m.to_s }.grep(/^[a-z]/)
+				(im & im.map { |m| m+'=' }).map { |m| m.chop }.find_all { |m| o1.instance_variable_get('@'+m) }
+			)
 			if a.empty?
 				return o1 == o2 ? h : [o1, o2]
 			end

data/misc/objscan.rb

@@ -26,7 +26,7 @@ class Object
 		end
 		scan_iter { |v, p|
 			case v
-			when Fixnum, Symbol; next
+			when Integer, Symbol; next
 			end
 			p = path+p
 			if done[v.object_id]

data/misc/openrisc-parser.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/ruby
+
+require 'xml'
+
+xml = Xml.parse_file(ARGV.shift || 'openrisc-insn.html')
+
+# [name, bin, args]
+addop = []
+# arg => [flds]
+valid_args = {}
+# field => [bitoff, bitmask]
+fields = {}
+
+xml.each('ul') { |ul|
+	syntax = nil
+	bits = []
+	vals = []
+	trno = 0
+	ul.each('li') { |li|
+		if li.children[0] == 'syntax:'
+			# <li>syntax:<tt><font>l.add $rd, $ra, $rb</font></tt></li>
+			syntax = li.children[1].children[0].children[0]
+		end
+	}
+	next if not syntax
+	ul.each('tr') { |tr|
+		case trno
+		when 0; tr.each('td') { |td| bits << td.children[0].split.map { |b| b.to_i } }
+		when 2; tr.each('td') { |td| vals << td.children.map { |v| v =~ /^0x/ ? v.to_i(16) : v.gsub('-', '') }.first }
+		end
+		trno += 1
+	}
+
+	iname = syntax.split[0].sub(/^l\./, '')
+	iargs = syntax.split[1].to_s.split(',').map { |a| a.gsub(/[${}-]/, '').gsub(/(\w+)\((\w+)\)/, '\2_\1') }
+	bin = bits.zip(vals).inject(0) { |b, (bt, bv)| bv.kind_of?(Integer) ? b | (bv << bt.last) : b }
+	addop << [iname, bin]
+
+	flds = bits.zip(vals).inject({}) { |h, (bt, bv)|
+		next h if bv.kind_of?(Integer)
+		blen = bt.first + 1 - bt.last
+		h.update bv => [bt.last, (1 << blen) - 1]
+	}
+	flds.each { |n, (o, m)|
+		if not fields[n]
+			fields[n] = [o, m]
+		elsif fields[n] != [o, m]
+			puts "# fields mismatch in #{iname} #{n}"
+		end
+	}
+
+	addop.last << iargs
+
+	iargs.each { |a|
+		a.split('_').each { |f|
+			if not flds.delete(f)
+				puts "# no field #{f} for arg #{a} in #{iname}"
+			end
+		}
+		valid_args[a] ||= a.split('_')
+	}
+	flds.each_key { |f|
+		puts "# no arg using #{f} in #{iname}"
+		a_i = "#{f}_ign"
+		valid_args[a_i] ||= [f]
+		addop.last.last << a_i
+	}
+}
+
+puts "\tdef init_cpu"
+puts "\t\t@opcode_list = []"
+puts "\t\t@valid_args = { #{valid_args.map { |a, f| ":#{a} => [#{f.map { |ff| ':' + ff }.join(', ')}]" }.join(', ')} }"
+puts "\t\t@fields_off = { #{fields.map { |k, v| ":#{k} => #{v[0]}" }.join(', ')} }"
+puts "\t\t@fields_mask = { #{fields.map { |k, v| ":#{k} => #{'0x%02X' % v[1]}" }.join(', ')} }"
+puts
+addop.each { |op|
+	puts "\t\taddop '#{op[0]}', #{'0x%08X' % op[1]}#{op[2].map { |a| ", :#{a}" }.join('')}"
+}
+puts "\tend"