metasm 1.0.0

data/misc/ppc_pdf2oplist.rb

@@ -0,0 +1,192 @@
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+#
+# parses the PPC specification PDF to generate the opcode list
+#
+
+require 'pdfparse'
+
+$field_mask = {}
+$field_shift = {}
+$opcodes = []
+def make_instr(bins, bits, text)
+	# calc bitfields length from their offset
+	last = 32
+	bitlen = []
+	bits.reverse_each { |bit|
+		bitlen.unshift last-bit
+		last = bit
+	}
+
+	# the opcode binary value (w/o fields)
+	bin = 0
+	fields = []
+
+	# parse the data
+	bins.zip(bits, bitlen).each { |val, off, len|
+		off = 32-(off+len)
+		msk = (1 << len) - 1
+		case val
+		when '/', '//', '///'	# reserved field, value unspecified
+		when /^\d+$/; bin |= val.to_i << off	# constant field
+		when /^[A-Za-z]+$/
+			fld = val.downcase.to_sym
+			fld = "#{fld}_".to_sym while $field_mask[fld] and ($field_mask[fld] != msk or $field_shift[fld] != off)
+			fields << fld
+			$field_mask[fld] ||= msk
+			$field_shift[fld] ||= off
+		end
+	}
+
+	text.each { |txt|
+		# fnabs FRT,FRB (Rc=0)
+		curbin = bin
+		curfields = fields.dup
+		txt.sub!('  Rc=1)', '  (Rc=1)') if txt.include? 'fdiv.'		# typo: fdiv. has no '('
+		if txt =~ /(.*\S)\s*\((\w+=.*)\)/
+			txt = $1
+			$2.split.each { |e|
+ 				raise e if e !~ /(\w+)=(\d+)/
+				name, val = $1.downcase, $2.to_i
+				raise "bad bit #{name} in #{txt}" if not fld = curfields.find { |fld_| fld_.to_s.delete('_') == name }
+				curfields.delete fld
+				curbin |= val << $field_shift[fld]
+			}
+		end
+		opname, args = txt.split(/\s+/, 2)
+		args = args.to_s.downcase.split(/\s*,\s*/).map { |arg| fld = curfields.find { |fld_| fld_.to_s.delete('_') == arg } ; curfields.delete fld ; fld }
+		if args.include? nil and curfields.length == 2 and (curfields - [:ra, :d]).empty?
+			args[args.index(nil)] = :ra_i16
+			curfields.clear
+		elsif args.include? nil and curfields.length == 2 and (curfields - [:ra, :ds]).empty?
+			args[args.index(nil)] = :ra_i16s
+			curfields.clear
+		elsif args.include? nil and curfields.length == 2 and (curfields - [:ra, :dq]).empty?
+			args[args.index(nil)] = :ra_i16q
+			curfields.clear
+		elsif args.include? nil and curfields.length == 1
+			args[args.index(nil)] = curfields.shift
+		end
+		raise "bad args #{args.inspect} (#{curfields.inspect}) in #{txt}" if args.include? nil
+		$opcodes << [opname, curbin, args]
+
+		n = (opname.inspect << ',').ljust(10) + '0x%08X' % curbin
+		n << ', ' if not args.empty?
+		puts "\taddop " + n + args.map { |e| e.inspect }.join(', ')
+	}
+end
+
+# handle instruction aliases
+# NOT WORKING
+# should be implemented in the parser/displayer instead of opcode list
+# manual work needed for eg conditionnal jumps
+def make_alias(newop, newargs, oldop, oldargs)
+	raise "unknown alias #{newop} => #{oldop}" if not op = $opcodes.reverse.find { |op_| op_[0] == oldop }
+	op2 = op.dup
+	op2[0] = newop
+	oldargs.each_with_index { |oa, i|
+		# XXX bcctr 4, 6  ->  bcctr 4, 6, 0 => not the work
+		if oa =~ /^[0-9]+$/ or oa =~ /^0x[0-9a-f]+$/i
+			fld = op[2][i]
+			op2[1] |= Integer(oa) << $field_shift[fld]
+		end
+	}
+	puts "#\talias #{newop} #{newargs.join(', ')}  ->  #{oldop} #{oldargs.join(', ')}".downcase
+end
+
+require 'enumerator'
+def epilog
+	puts "\n\t@field_shift = {"
+	puts $field_shift.sort_by { |k, v| k.to_s }.enum_slice(6).map { |slc|
+		"\t\t" + slc.map { |k, v| "#{k.inspect} => #{v}" }.join(', ')
+	}.join(",\n")
+	puts "\t}"
+	puts "\n\t@field_mask = {"
+	puts $field_mask.sort_by { |k, v| k.to_s }.enum_slice(6).map { |slc|
+		"\t\t" + slc.map { |k, v| "#{k.inspect} => #{v > 1000 ? '0x%X' % v : v}" }.join(', ')
+	}.join(",\n")
+	puts "\t}"
+end
+
+$foundop = false
+def parse_page(lines)
+	# all instr defining pages include this
+	return unless lines.find { |l| l.str =~ /Special Registers Altered|Memory Barrier Instructions|Data Cache Instructions/  }	# sync L/dcbt
+
+	ilist = [] # line buffer
+	extended = false
+
+	# concat lines with same y
+	lines = lines.sort_by { |l| [-l.y, l.x] }
+	lastline = nil
+	lines.delete_if { |l|
+		if lastline and lastline.y == l.y and ([lastline.fontx, lastline.fonty] == [l.fontx, l.fonty] or l.str =~ /^\s*$/)
+			lastline.str << ' ' << l.str
+			true
+		else
+			lastline = l
+			false
+		end
+	}
+
+	lines.each { |l|
+		# search for the bit indices list
+		if l.fonty < 7 and l.str =~ /^0 [\d ]+ 31\s*$/ and (ilist.last.str.split.length == l.str.split.length or ilist.last.str.split.length == l.str.split.length-1)
+			$foundop = true
+			bitindices = l.str.split.map { |i| i.to_i }
+			# previous line is the binary encoding
+			encoding = ilist.pop.str.split
+			bitindices.pop if encoding.length < bitindices.length
+			# previous line is the instruction text format
+			ilist.pop if ilist.last.str =~ /\[POWER2? mnemonics?: (.*)\]/
+			text = []
+			text.unshift l while l = ilist.pop and l = l.str and (l =~ /,|\)$/ or text.empty?)
+			ilist = []
+			make_instr(encoding, bitindices, text)
+		elsif l.str.include? 'Special Registers Altered'
+			if not $foundop
+				puts ilist.map { |l_| "(#{l_.y}) #{l_.str}" }
+				puts lines.map { |l_| "(#{l_.y}) #{l_.str}" } if ilist.empty?
+				raise 'nofoundop'
+			else
+				$foundop = false
+			end
+		elsif l.str =~ /Extended:\s+Equivalent to:/
+			extended = true
+		elsif extended
+			if l.str.include? ',' and l.str =~ /^(\S+)\s+(\S+)\s+(\S+)\s+(.*)/ and $opcodes.find { |op| op[0] == $3 }
+				newop, newargs, exop, exargs = $1, $2, $3, $4
+				make_alias(newop, newargs.split(','), exop, exargs.split(','))
+			else extended = false
+			end
+		else ilist << l
+		end
+	}
+end
+
+# PowerPC Architecture v2.02:
+#  1 - User Instruction Set
+#  2 - Virtual Environment
+#  3 - Operating Environment
+Dir['PPC_Vers202_Book?_public.pdf'].sort.each { |book|
+	$stderr.puts book if $stderr.tty?
+	pdf = PDF.read book
+	pagecount = pdf.trailer['Root']['Pages']['Count'] || 0
+	curpage = 0
+	pdf.each_page { |p|
+		$stderr.print "#{curpage+=1}/#{pagecount} \r" if $stderr.tty?
+		p.clip_lines(50, 740)
+		list = p.lines.flatten
+
+		# split columns
+		sp1, sp2 = list.partition { |l| l.x < 288 }
+
+		parse_page(sp1)
+		parse_page(sp2)
+	}
+	$stderr.print "           \r" if $stderr.tty?
+}
+
+epilog()

data/misc/tcp_proxy_hex.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/ruby
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+
+# this is a TCP proxy which dumps the transmitted data in hex on stdout
+# usage: tcp_proxy <lhost> <lport> <rhost> <rport> [<timeout_s>]
+
+require 'socket'
+require File.join(File.dirname(__FILE__), 'hexdump')
+
+def bouncepkt(clt, srv, timeout=nil)
+	s2c = ''
+	c2s = ''
+	loop do
+		break if not IO.select([clt, srv], nil, nil, timeout)
+
+		while srv and s2c.length < 1024*16 and IO.select([srv], nil, nil, 0)
+			str = (srv.read(1) rescue nil)
+			if not str or str.empty?
+				srv = false
+			else
+				s2c << str
+			end
+		end
+
+		while clt and c2s.length < 1024*16 and IO.select([clt], nil, nil, 0)
+			str = (clt.read(1) rescue nil)
+			if not str or str.empty?
+				clt = false
+			else
+				c2s << str
+			end
+		end
+
+		if clt and s2c.length > 0 and IO.select(nil, [clt], nil, 0)
+			puts Time.now.strftime('s -> c  %H:%M:%S')
+			s2c.hexdump(:fmt => ['c', 'a'])
+			clt.write s2c
+			s2c.replace ''
+		end
+
+		if srv and c2s.length > 0 and IO.select(nil, [srv], nil, 0)
+			puts Time.now.strftime('c -> s  %H:%M:%S')
+			c2s.hexdump(:fmt => ['c', 'a'])
+			srv.write c2s
+			c2s.replace ''
+		end
+		break if not clt or not srv
+	end
+end
+
+
+if $0 == __FILE__
+if ARGV.length < 4
+	abort "usage: bnc <lhost> <lport> <rhost> <rport> [<timeout_s>]"
+end
+
+lhost = ARGV.shift
+lport = Integer(ARGV.shift)
+rhost = ARGV.shift
+rport = Integer(ARGV.shift)
+timeout = Float(ARGV.shift) if not ARGV.empty?
+
+s = TCPServer.new(lhost, lport)
+
+loop do
+	puts "waiting..."
+	a = s.accept
+	puts "incoming connection"
+	c = TCPSocket.new(rhost, rport)
+	
+	begin
+		bouncepkt(a, c, timeout)
+	rescue SystemCallError
+	end
+
+	puts "connection closed"
+	a.close
+	c.close
+end
+end

data/misc/txt2html.rb

@@ -0,0 +1,440 @@
+#!/usr/bin/env ruby
+#    This file is part of Metasm, the Ruby assembly manipulation suite
+#    Copyright (C) 2006-2009 Yoann GUILLOT
+#
+#    Licence is LGPL, see LICENCE in the top-level directory
+
+# This scripts is used to compile the Metasm documentation into html files
+# Losely inspired from the rst syntax
+
+# stuff to generate html code
+module Html
+class Elem
+	attr_reader :name, :attrs, :content, :style
+
+	IndentAdd = '  '
+	LineLenMax = 80
+	
+	def initialize(name, attrs=nil, content=nil)
+		@name = name
+		@attrs = Hash.new
+		@style = Hash.new
+		attrs.each { |k, v| set_attr(k, v) } if attrs
+		if content == false
+			@content = Array.new
+			@uniq = true
+		else
+			@content = content ? content : Array.new
+			@uniq = false
+		end
+		self
+	end
+	
+	@@quotechars = {
+		'è' => 'è',
+		'ë' => 'ë',
+		'>' => '>',
+		'<' => '&lt;',
+		'"' => '&quot;',
+		'&' => '&amp;',
+	}
+	
+	def add(*content)
+		content.each { |e|
+			if (e.class == Array)
+				add(*e)
+				next
+			end
+			if e.class.ancestors.include? Elem
+				@content << e
+			else
+				@content << e.to_s.gsub(Regexp.new("(#{@@quotechars.keys.join('|')})")) { |x| @@quotechars[x] }
+			end
+		}
+		self
+	end
+	alias << add
+
+	def add_style(k, v)
+		@style[k] = v
+		self
+	end
+
+	def set_attr(k, v)
+		if k == 'style'
+			v.split(/\s*;\s*/).each { |s|
+				add_style($1, $2) if s =~ /^\s*(\S+)\s*:\s*(.*?)\s*$/
+			}
+		else
+			@attrs[k]=v
+		end
+		self
+	end
+
+	def bg(c)
+		@style['background'] = c
+		self
+	end
+	
+	def hclass(c)
+		@attrs['class'] = c
+		self
+	end
+
+	def length(start=nil)
+		# text length on one line w/o indent
+		if start
+			l = start.length
+		else
+			# '<name>'
+			l = @name.length + 2
+			@attrs.each{ |k, v|
+				l += " #{k}=\"#{v}\"".length
+			}
+			# ' style=""' - last '; '
+			l += 9-2 unless @style.empty?
+			# 'k: v; '
+			@style.each{ |k, v|
+				l += "#{k}: #{v}; ".length
+			}
+			# ' /'
+			l += 2 if @uniq
+		end
+		@content.each{ |c|
+			l += c.length
+		}
+		# '</name>'
+		l += 3+@name.length unless @uniq
+		return l
+	end
+	
+	def to_s(indent = '')
+		attrs = @attrs.map { |k, v| " #{k}=\"#{v}\"" }.join
+		attrs += ' style="' + @style.map{ |k, v| "#{k}: #{v}" }.join('; ') + '"' unless @style.empty?
+		s = '' << indent << '<' << @name << attrs << (@uniq ? ' />' : '>')
+		if @uniq
+			s
+		elsif @name == 'pre'
+			s << @content.map { |c| c.to_s }.join.chomp << '</pre>'
+		else
+			if length(s) > LineLenMax
+				sindent = indent + IndentAdd
+				sep = "\n"
+				@content.each { |c|
+					case c
+					when Elem
+						if sep == ''
+							s << c.to_s(sindent).sub(/^\s+/, '')
+						else
+							news = c.to_s(sindent)
+							plen = s.length - (s.rindex("\n") || -1) - 1
+							plen -= 1 if s[-1, 1] == ' '
+							newss = news.sub(/^\s+/, '')
+							if not news.include?("\n") and s[-1] != ?> and
+									plen + 1 + newss.length <= LineLenMax
+								# concat inline tag to previous String
+								s << ' ' if s[-1, 1] != ' '
+								s << newss
+							else
+								s << sep if c.name =~ /^h\d$/ and c != @content.first
+								s << sep << news
+							end
+						end
+					when String
+						cw = c.split(/\s+/)
+						if @name == 'p' and c.object_id == @content.first.object_id
+							cw.shift if cw[0] == ''
+							s << "\n" << sindent
+						else
+							s << cw.shift.to_s
+						end
+						plen = s.length - (s.rindex("\n") || -1) - 1
+						while w = cw.shift
+							plen -= 1 if s[-1, 1] == ' '
+							if plen + 1 + w.length > LineLenMax
+								s << "\n" << sindent
+								plen = sindent.length
+							end
+							s << ' ' if s[-1, 1] != ' '
+							s << w
+							plen += w.length+1
+						end
+						if c !~ /\s+$/
+							sep = ''
+							next
+						end
+					else
+						s << sep << sindent << c.to_s
+					end
+					sep = "\n"
+				}
+				sep = "\n" if @name == 'p'
+				sep << indent if sep != ''
+				s << sep << "</#@name>"
+			else
+				s << @content.map { |c| c.to_s }.join << "</#@name>"
+			end
+		end
+	end
+
+	def inspect
+		"<#{@name}"+@content.map{|c|"\n"+c.inspect}.join+"\n/#{@name}>"
+	end
+end	
+
+class Page < Elem
+	attr_reader :body, :head
+	def initialize
+		@body = Elem.new('body')
+		@head = Elem.new('head')
+		super('html', {'xmlns'=>'http://www.w3.org/1999/xhtml', 'xml:lang'=>'fr'})
+		add(@head)
+		add(@body)
+	end
+
+	def to_s
+		'<?xml version="1.0" encoding="us-ascii" ?>'+"\n"+
+		'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"'+"\n"+
+		IndentAdd*2+'"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">'+"\n"+
+		super.to_s
+	end
+end
+class Img < Elem
+	def initialize(src, alt=nil)
+		super('img', {'src'=>src}, false)
+		set_attr('alt', alt) if alt
+		self
+	end
+end
+class A < Elem
+	def initialize(href, text)
+		super('a', {'href'=>href}, [text])
+	end
+end
+class P < Elem
+	def initialize(content = nil)
+		super('p')
+		add(content) if content
+		self
+	end
+end
+class Div < Elem
+	def initialize(hclass = nil)
+		super('div')
+		hclass(hclass) if hclass
+		self
+	end
+end
+class Span < Elem
+	def initialize(hclass = nil)
+		super('span')
+		hclass(hclass) if hclass
+		self
+	end
+end
+class Stylesheet < Elem
+	def initialize(href)
+		super('link', {'rel'=>'stylesheet', 'type'=>'text/css', 'href'=>href}, false)
+	end
+end
+class Br < Elem
+	def initialize
+		super('br', nil, false)
+	end
+end
+class Hr < Elem
+	def initialize
+		super('hr', nil, false)
+	end
+end
+
+class List < Elem
+	def initialize(*elems)
+		super('ul')
+		elems.each { |e| add_line(e) }
+	end
+
+	def add_line(line)
+		add(Elem.new('li').add(line))
+		self
+	end
+end
+end
+
+class Txt2Html
+	def initialize(f)
+		@@done ||= []
+		return if @@done.include? f
+		@@done << f
+
+		raise 'bad path' if (f.split('/') & ['.', '..']).first
+
+		outf = outfilename(f)
+		puts "compiling #{outf}..." if $VERBOSE
+
+		@pathfix = outf.split('/')[0...-1].map { '../' }.join
+		out = compile(File.read(f).gsub("\r", '') + "\n\n")
+		File.open(outf, 'wb') { |fd| fd.write out.to_s.gsub("\r", '').gsub("\n", "\r\n") }
+	end
+
+	def outfilename(f)
+		f.sub(/\.txt$/, '') + '.html'
+	end
+
+	def compile(raw)
+		prev = ''
+		state = {}
+		anchors = {}
+		out = Html::Page.new
+		out.head << Html::Stylesheet.new(@pathfix + 'style.css')
+		flush = lambda {
+			out.body << Html::P.new(compile_string(prev)) if prev.length > 0
+			[:pre, :list, :par].each { |f| state.delete f }
+			prev = ''
+		}
+		raw.each_line { |l|
+			case l = l.chomp
+			when /^([=#*-])\1{3,}$/
+				if prev.length > 0
+					# title
+					if    not state[:h1] or state[:h1] == $1
+						state[:h1] = $1
+						e = 'h1'
+					elsif not state[:h2] or state[:h2] == $1
+						state[:h2] = $1
+						e = 'h2'
+					elsif not state[:h3] or state[:h3] == $1
+						state[:h3] = $1
+						e = 'h3'
+					else raise "unknown title level after #{prev.inspect}"
+					end
+					str = compile_string(prev)
+					state[:title] ||= str if e == 'h1'
+					if id = prev[/[a-z]\w+/i]
+						id = id.downcase
+						id += '_' while anchors[id]
+						anchors[id] = true
+						attr = { 'id' => id }
+					end
+					out.body << Html::Elem.new(e, attr).add(str)
+					prev = ''
+					flush[]
+				else
+					# horizontal rule
+					out.body << Html::Hr.new
+					flush[]
+				end
+			when /^([*-]+)\s+(.*)/
+				# list
+				bullet = $1
+				text = $2
+				if lst = state[:list] && state[:list][bullet]
+					state[:list].delete_if { |k, v| k.length > bullet.length }
+				else
+					flush[] if not state[:list]
+					state[:list] ||= {}
+					state[:list].delete_if { |k, v| k.length > bullet.length }
+					lst = state[:list][bullet] = Html::List.new
+					if pl = state[:list][bullet.chop]
+						pl.content.last.content << lst
+					else
+						out.body << lst
+					end
+				end
+				lst.add_line compile_string(text)
+
+			when /^\s+(\S.*)$/
+				# preformatted text
+				if not pre = state[:pre]
+					flush[]
+					pre = state[:pre] = Html::Elem.new('pre')
+					out.body << pre
+				end
+				pre.add compile_string(l) + ["\n"]
+			when /^\s*$/
+				flush[]
+			else
+				if state[:list]
+					lst = state[:list].sort.last[1]
+					lst.content.last.content << ' ' << compile_string(l)
+				else
+					prev << ' ' if prev.length > 0
+					prev << l
+				end
+			end
+		}
+		flush[]
+		out.head << Html::Elem.new('title').add(state[:title]) if state[:title]
+		out
+	end
+
+	# handle **bold_words** *italic* `fixed` <links>
+	def compile_string(str)
+		o = [str]
+		on = []
+		o.each { |s|
+			while s.kind_of? String and o1 = s.index('**') and o2 = s.index('**', o1+2) and not s[o1..o2].index(' ')
+				on << s[0...o1] << Html::Elem.new('b').add(s[o1+2...o2].tr('_', ' '))
+				s = s[o2+2..-1]
+			end
+			on << s
+		}
+		o = on
+		on = []
+		o.each { |s|
+			while s.kind_of? String and o1 = s.index('*') and o2 = s.index('*', o1+1) and not s[o1..o2].index(' ')
+				on << s[0...o1] << Html::Elem.new('i').add(s[o1+1...o2].tr('_', ' '))
+				s = s[o2+1..-1]
+			end
+			on << s
+		}
+		o = on
+		on = []
+		o.each { |s|
+			while s.kind_of? String and o1 = s.index('`') and o2 = s.index('`', o1+1)
+				on << s[0...o1] << Html::Span.new('quote').add(s[o1+1...o2])
+				s = s[o2+1..-1]
+			end
+			on << s
+		}
+		o = on
+		on = []
+		o.each { |s|
+			while s.kind_of? String and o1 = s.index('<') and o2 = s.index('>', o1+1) and not s[o1..o2].index(' ')
+				on << s[0...o1]
+				lnk = s[o1+1...o2]
+				s = s[o2+1..-1]
+				if File.exist? lnk
+					case lnk[/\.(\w+)$/, 1]
+					when 'txt'
+						tg = outfilename(lnk)
+						Txt2Html.new(lnk)
+						on << Html::A.new(@pathfix + tg, File.basename(lnk, '.txt').tr('_', ' '))
+					when 'jpg', 'png'
+						on << Html::Img.new(lnk)
+					end
+				else
+					if lnk =~ /\.txt$/
+						@@seen_nofile ||= []
+						if not @@seen_nofile.include? lnk
+							@@seen_nofile << lnk
+							puts "reference to missing #{lnk.inspect}"
+						end
+					end
+					on << Html::A.new(lnk, lnk)
+				end
+			end
+			on << s
+		}
+		o = on
+	end
+end
+
+if __FILE__ == $0
+	$VERBOSE = true if ARGV.delete '-v'
+	if ARGV.empty?
+		Dir.chdir(File.expand_path(File.join(File.dirname(__FILE__), '../doc')))
+		ARGV.concat Dir['**/index.txt']
+	end
+	ARGV.each { |a| Txt2Html.new(a) }
+end