fluent-plugin-perf-tools 0.1.0

data/perf-tools/examples/uprobe_example.txt

@@ -0,0 +1,321 @@
+Demonstrations of uprobe, the Linux ftrace version.
+
+Trace the readline() function from all processes named "bash":
+
+# ./uprobe p:bash:readline
+Tracing uprobe readline (p:readline /bin/bash:0x8db60). Ctrl-C to end.
+            bash-11886 [003] d... 19601233.618462: readline: (0x48db60)
+            bash-11886 [003] d... 19601235.152067: readline: (0x48db60)
+            bash-11915 [003] d... 19601238.976244: readline: (0x48db60)
+^C
+Ending tracing...
+
+readline() is the bash shell's function for reading interactive input, and
+a line is printed each time I entered commands in separate bash shells.
+The line contains default ftrace columns: the process name, "-", and PID;
+the CPU, flags, a timestamp (in units of seconds), the probe name, then
+other arguments. These columns are documented in the kernel source, under
+Documentation/trace/ftrace.txt.
+
+The first line of output is informational, and shows what uprobe is really
+doing: it turned "bash" into "/bin/bash", using a $PATH lookup (via which(1)).
+It then turned the "readline" symbol into 0x8db60, using objdump(1) for
+symbol lookups.
+
+Note that this traces _all_ bash processes simultaneously.
+
+
+Tracing PID 11886 only:
+
+# ./uprobe -p 11886 p:bash:readline
+Tracing uprobe readline (p:readline /bin/bash:0x8db60). Ctrl-C to end.
+            bash-11886 [002] d... 19601657.753893: readline: (0x48db60)
+            bash-11886 [002] d... 19601658.246613: readline: (0x48db60)
+            bash-11886 [002] d... 19601658.386666: readline: (0x48db60)
+            bash-11886 [002] d... 19601661.415952: readline: (0x48db60)
+^C
+Ending tracing...
+
+This may be important if you are tracing shared library functions, and only care
+about one target process.
+
+
+You can specify the full path to a binary to trace:
+
+# ./uprobe p:/bin/bash:readline
+Tracing uprobe readline (p:readline /bin/bash:0x8db60). Ctrl-C to end.
+            bash-11886 [002] d... 19601746.902461: readline: (0x48db60)
+            bash-11886 [002] d... 19601749.543485: readline: (0x48db60)
+            bash-11886 [001] d... 19601749.702369: readline: (0x48db60)
+^C
+Ending tracing...
+
+This might be useful if uprobe picked the wrong binary to trace, as shown by
+the informational line, and you wanted to specify it directly. It is also useful
+for tracing binaries not in the $PATH, which uprobe can't otherwise find.
+
+
+Use -l to list symbols available to trace; eg, searching for functions
+containing "readline" in bash:
+
+# ./uprobe -l bash | grep readline
+initialize_readline
+pcomp_set_readline_variables
+posix_readline_initialize
+readline
+readline_internal_char
+readline_internal_setup
+readline_internal_teardown
+
+
+Tracing the return of readline() with return value as a string:
+
+# ./uprobe 'r:bash:readline +0($retval):string'
+Tracing uprobe readline (r:readline /bin/bash:0x8db60 +0($retval):string). Ctrl-C to end.
+            bash-11886 [003] d... 19601837.001935: readline: (0x41e876 <- 0x48db60) arg1="ls -l"
+            bash-11886 [002] d... 19601851.008409: readline: (0x41e876 <- 0x48db60) arg1="echo "hello world""
+            bash-11886 [002] d... 19601854.099730: readline: (0x41e876 <- 0x48db60) arg1="df -h"
+            bash-11886 [002] d... 19601858.805740: readline: (0x41e876 <- 0x48db60) arg1="cd .."
+            bash-11886 [003] d... 19601898.378753: readline: (0x41e876 <- 0x48db60) arg1="foo bar"
+^C
+Ending tracing...
+
+Now I can see the commands entered. Note that this traces what bash reads in,
+even if the command eventually fails. Eg, the last command "foo bar" didn't
+work (No command 'foo' found).
+
+Note that this invocation now uses "r:" at the start of the probe description,
+instead of "p:". r is for return probes, p for entry probes.
+
+
+Tracing sleep() calls in all running libc shared libraries:
+
+# ./uprobe p:libc:sleep
+Tracing uprobe sleep (p:sleep /lib/x86_64-linux-gnu/libc-2.15.so:0xbf130). Ctrl-C to end.
+          svscan-2134  [000] d... 19602402.959904: sleep: (0x7f2dba562130)
+            cron-923   [000] d... 19602404.640507: sleep: (0x7f3e26d9e130)
+            cron-923   [002] d... 19602404.655232: sleep: (0x7f3e26d9e130)
+            cron-923   [002] d... 19602405.189271: sleep: (0x7f3e26d9e130)
+          svscan-2134  [000] d... 19602407.959947: sleep: (0x7f2dba562130)
+[...]
+
+This shows different programs calling sleep -- likely threads waiting for work.
+
+I ran a "sleep 1" command in a bash shell, which wasn't seen above: probably
+using a different sleep library call, which I'd need to trace separately.
+
+
+Including headers (-H):
+
+# ./uprobe -H p:libc:sleep
+Tracing uprobe sleep (p:sleep /lib/x86_64-linux-gnu/libc-2.15.so:0xbf130). Ctrl-C to end.
+# tracer: nop
+#
+# entries-in-buffer/entries-written: 0/0   #P:4
+#
+#                              _-----=> irqs-off
+#                             / _----=> need-resched
+#                            | / _---=> hardirq/softirq
+#                            || / _--=> preempt-depth
+#                            ||| /     delay
+#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
+#              | |       |   ||||       |         |
+          svscan-2134  [000] d... 19603052.976770: sleep: (0x7f2dba562130)
+          svscan-2134  [002] d... 19603057.976927: sleep: (0x7f2dba562130)
+[...]
+
+These are documented in Documentation/trace/ftrace.txt.
+
+
+Tracing sleep() with its argument (seconds):
+
+# ./uprobe 'p:libc:sleep %di'
+Tracing uprobe sleep (p:sleep /lib/x86_64-linux-gnu/libc-2.15.so:0xbf130 %di). Ctrl-C to end.
+          svscan-2134  [002] d... 19602517.962925: sleep: (0x7f2dba562130) arg1=0x5
+          svscan-2134  [002] d... 19602522.963082: sleep: (0x7f2dba562130) arg1=0x5
+            cron-923   [002] d... 19602524.187733: sleep: (0x7f3e26d9e130) arg1=0x3c
+          svscan-2134  [002] d... 19602527.963267: sleep: (0x7f2dba562130) arg1=0x5
+[...]
+
+So svcan was sleeping for 5 seconds, and cron for 60 seconds (0x3c = 60).
+
+The argument is specified by its register, %di. This is platform dependent: %di
+may only be meaningful on x86. If you're on a different architecture (eg, ARM),
+you will probably need to use something else.
+
+If working with registers is not for you, then consider tracing this using
+perf_events with debuginfo installed: in which case you can use the variable
+names. Or consider a different tracer.
+
+
+Here is an example of the optional filter expression, to only trace the return
+of fopen() when it failed and returned NULL (0):
+
+# ./uprobe 'r:libc:fopen file=$retval' 'file == 0'
+Tracing uprobe fopen (r:fopen /lib/x86_64-linux-gnu/libc-2.15.so:0x6e540 file=$retval). Ctrl-C to end.
+           prog1-23982 [000] d... 19602894.346872: fopen: (0x40051e <- 0x7f637867f540) file=0x0
+^C
+Ending tracing...
+
+The argument $retval was given a vanity name "file", which was then tested in
+the filter expression "file == 0".
+
+
+Here's an example of tracing the MySQL server dispatch_command() function, along
+with the query string (note: the %dx register is only valid for this
+architecture and this software build):
+
+# ./uprobe 'p:dispatch_command /opt/mysql/bin/mysqld:_Z16dispatch_command19enum_server_commandP3THDPcj +0(%dx):string'
+Tracing uprobe dispatch_command (p:dispatch_command /opt/mysql/bin/mysqld:0x2dbd40 +0(%dx):string). Ctrl-C to end.
+          mysqld-2855  [001] d... 19956674.509085: dispatch_command: (0x6dbd40) arg1="show tables"
+          mysqld-2855  [001] d... 19956675.541155: dispatch_command: (0x6dbd40) arg1="SELECT * FROM numbers where number > 32000"
+^C
+Ending tracing...
+
+The function name, "_Z16dispatch_command19enum_server_commandP3THDPcj", is the
+C++ mangled symbol.
+
+I can name the query string argument "cmd" then test it in a filter; eg, to only
+match queries that begin with "SELECT":
+
+# ./uprobe 'p:dispatch_command /opt/mysql/bin/mysqld:_Z16dispatch_command19enum_server_commandP3THDPcj cmd=+0(%dx):string' 'cmd ~ "SELECT*"'
+Tracing uprobe dispatch_command (p:dispatch_command /opt/mysql/bin/mysqld:0x2dbd40 cmd=+0(%dx):string). Ctrl-C to end.
+          mysqld-2855  [001] d... 19956754.619958: dispatch_command: (0x6dbd40) cmd="SELECT * FROM numbers where number > 32000"
+          mysqld-2855  [001] d... 19956755.060125: dispatch_command: (0x6dbd40) cmd="SELECT * FROM numbers where number > 32000"
+^C
+Ending tracing...
+
+
+Overhead is relative to the rate of events: a higher rate of traced events,
+means uprobe costs higher overhead. If you are unsure of the rate of events,
+you can capture a set number only, or trace for a limited duration only (covered
+in the next example). To trace a set number only, you can pipe into head, eg:
+
+# ./uprobe -p 11982 p:bash:sh_malloc | head -15
+Tracing uprobe sh_malloc (p:sh_malloc /bin/bash:0xaafa0). Ctrl-C to end.
+            bash-11982 [001] d... 19643121.529484: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529493: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529506: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529510: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529519: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529521: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529523: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529525: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529531: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529533: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529536: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529541: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529546: sh_malloc: (0x4aafa0)
+            bash-11982 [001] d... 19643121.529549: sh_malloc: (0x4aafa0)
+
+uprobe traps SIGPIPE, so that it properly exits and cleans up probes when used
+in this fashion.
+
+Note the timestamps: by examining the rate they are increasing, you can have
+some estimation for the rate of events. In this case, the 15 events all
+happened within the same millisecond (the timestamp column is in units of
+seconds), which suggests these are frequent events.
+
+
+The -d option can be used to specify a duration for tracing, which also causes
+uprobe to perform in-kernel buffering, which reduces the overhead of tracing:
+
+# ./uprobe -d 5 p:libc:gettimeofday
+Tracing uprobe gettimeofday for 5 seconds (buffered)...
+           sleep-12743 [001] d... 19642858.943440: gettimeofday: (0x7f400138ac10)
+       rotatelog-12744 [000] d... 19642858.955665: gettimeofday: (0x7f0ba34ebc10)
+       rotatelog-12745 [003] d... 19642858.956425: gettimeofday: (0x7f1e6db20c10)
+       rotatelog-12744 [000] d... 19642858.956924: gettimeofday: (0x7f0ba34ebc10)
+       rotatelog-12745 [003] d... 19642858.957608: gettimeofday: (0x7f1e6db20c10)
+       rotatelog-12744 [001] d... 19642858.958005: gettimeofday: (0x7fd8a1d64c10)
+       rotatelog-12744 [003] d... 19642858.959496: gettimeofday: (0x7f9531acdc10)
+           mkdir-12746 [002] d... 19642858.959542: gettimeofday: (0x7fd539474c10)
+           chown-12747 [001] d... 19642858.961455: gettimeofday: (0x7ff5646afc10)
+       rotatelog-12745 [000] d... 19642858.963065: gettimeofday: (0x7f406aca7c10)
+       rotatelog-12745 [001] d... 19642858.964280: gettimeofday: (0x7f6548debc10)
+       rotatelog-12749 [000] d... 19642859.977462: gettimeofday: (0x7fecaf7e1c10)
+       rotatelog-12750 [003] d... 19642859.977697: gettimeofday: (0x7f821eb3cc10)
+       rotatelog-12749 [000] d... 19642859.978707: gettimeofday: (0x7fecaf7e1c10)
+[...]
+
+You will not see live output during the -d mode, as it is being buffered
+in-kernel.
+
+
+Tracing func_abc() in my test program, and including user-level stacks:
+
+# ./uprobe -s p:/root/func_abc:func_c
+Tracing uprobe func_c (p:func_c /root/func_abc:0x4f4). Ctrl-C to end.
+        func_abc-25394 [000] d... 19603250.054040: func_c: (0x4004f4)
+        func_abc-25394 [000] d... 19603250.054056: <user stack trace>
+ =>  <00000000004004f4>
+ =>  <0000000000400527>
+ =>  <0000000000400537>
+ =>  <00007fca9f0e376d>
+        func_abc-25394 [000] d... 19603251.054250: func_c: (0x4004f4)
+        func_abc-25394 [000] d... 19603251.054266: <user stack trace>
+ =>  <00000000004004f4>
+ =>  <0000000000400527>
+ =>  <0000000000400537>
+ =>  <00007fca9f0e376d>
+^C
+Ending tracing...
+
+The output has the raw hex addresses. If this is too much of a nuisance, then
+try tracing this using perf_events which should automate the translation.
+
+It can get worse, eg:
+
+l# ./uprobe -s p:bash:readline
+Tracing uprobe readline (p:readline /bin/bash:0x8db60). Ctrl-C to end.
+            bash-11886 [002] d... 19603434.397818: readline: (0x48db60)
+            bash-11886 [002] d... 19603434.397832: <user stack trace>
+ =>  <000000000048db60>
+            bash-11886 [002] d... 19603434.592500: readline: (0x48db60)
+            bash-11886 [002] d... 19603434.592510: <user stack trace>
+ =>  <000000000048db60>
+^C
+Ending tracing...
+
+Here the stack trace is missing (0x48db60 is the traced function, transposed
+from the base load address). This is due to compiler optimizations. It can be
+fixed by recompiling with -fno-omit-frame-pointer, or, using perf_events and
+a different method of stack walking.
+
+
+Use -h to print the USAGE message:
+
+# ./uprobe -h
+USAGE: uprobe [-FhHsv] [-d secs] [-p PID] [-L TID] {-l target |
+              uprobe_definition [filter]}
+                 -F              # force. trace despite warnings.
+                 -d seconds      # trace duration, and use buffers
+                 -l target       # list functions from this executable
+                 -p PID          # PID to match on events
+                 -L TID          # thread id to match on events
+                 -v              # view format file (don't trace)
+                 -H              # include column headers
+                 -s              # show user stack traces
+                 -h              # this usage message
+
+Note that these examples may need modification to match your kernel
+version's function names and platform's register usage.
+   eg,
+       # trace readline() calls in all running "bash" executables:
+           uprobe p:bash:readline
+       # trace readline() with explicit executable path:
+           uprobe p:/bin/bash:readline
+       # trace the return of readline() with return value as a string:
+           uprobe 'r:bash:readline +0($retval):string'
+       # trace sleep() calls in all running libc shared libraries:
+           uprobe p:libc:sleep
+       # trace sleep() with register %di (x86):
+           uprobe 'p:libc:sleep %di'
+       # trace this address (use caution: must be instruction aligned):
+           uprobe p:libc:0xbf130
+       # trace gettimeofday() for PID 1182 only:
+           uprobe -p 1182 p:libc:gettimeofday
+       # trace the return of fopen() only when it returns NULL:
+           uprobe 'r:libc:fopen file=$retval' 'file == 0'
+
+See the man page and example file for more info.

data/perf-tools/execsnoop

@@ -0,0 +1,292 @@
+#!/bin/bash
+#
+# execsnoop - trace process exec() with arguments.
+#             Written using Linux ftrace.
+#
+# This shows the execution of new processes, especially short-lived ones that
+# can be missed by sampling tools such as top(1).
+#
+# USAGE: ./execsnoop [-hrt] [-n name]
+#
+# REQUIREMENTS: FTRACE and KPROBE CONFIG, sched:sched_process_fork tracepoint,
+# and either the sys_execve, stub_execve or do_execve kernel function. You may
+# already have these on recent kernels. And awk.
+#
+# This traces exec() from the fork()->exec() sequence, which means it won't
+# catch new processes that only fork(). With the -r option, it will also catch
+# processes that re-exec. It makes a best-effort attempt to retrieve the program
+# arguments and PPID; if these are unavailable, 0 and "[?]" are printed
+# respectively. There is also a limit to the number of arguments printed (by
+# default, 8), which can be increased using -a.
+#
+# This implementation is designed to work on older kernel versions, and without
+# kernel debuginfo. It works by dynamic tracing an execve kernel function to
+# read the arguments from the %si register. The sys_execve function is tried
+# first, then stub_execve and do_execve. The sched:sched_process_fork
+# tracepoint is used to get the PPID. This program is a workaround that should be
+# improved in the future when other kernel capabilities are made available. If
+# you need a more reliable tool now, then consider other tracing alternatives
+# (eg, SystemTap). This tool is really a proof of concept to see what ftrace can
+# currently do.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the execsnoop(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 07-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock; wroteflock=0
+opt_duration=0; duration=; opt_name=0; name=; opt_time=0; opt_reexec=0
+opt_argc=0; argc=8; max_argc=16; ftext=
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: execsnoop [-hrt] [-a argc] [-d secs] [name]
+	                 -d seconds      # trace duration, and use buffers
+	                 -a argc         # max args to show (default 8)
+	                 -r              # include re-execs
+	                 -t              # include time (seconds)
+	                 -h              # this usage message
+	                 name            # process name to match (REs allowed)
+	  eg,
+	       execsnoop                 # watch exec()s live (unbuffered)
+	       execsnoop -d 1            # trace 1 sec (buffered)
+	       execsnoop grep            # trace process names containing grep
+	       execsnoop 'udevd$'        # process names ending in "udevd"
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+	warn "echo 0 > events/kprobes/$kname/enable"
+	warn "echo 0 > events/sched/sched_process_fork/enable"
+	warn "echo -:$kname >> kprobe_events"
+	warn "echo > trace"
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts a:d:hrt opt
+do
+	case $opt in
+	a)	opt_argc=1; argc=$OPTARG ;;
+	d)	opt_duration=1; duration=$OPTARG ;;
+	r)	opt_reexec=1 ;;
+	t)	opt_time=1 ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+if (( $# )); then
+	opt_name=1
+	name=$1
+	shift
+fi
+(( $# )) && usage
+
+### option logic
+(( opt_pid && opt_name )) && die "ERROR: use either -p or -n."
+(( opt_pid )) && ftext=" issued by PID $pid"
+(( opt_name )) && ftext=" issued by process name \"$name\""
+(( opt_file )) && ftext="$ftext for filenames containing \"$file\""
+(( opt_argc && argc > max_argc )) && die "ERROR: max -a argc is $max_argc."
+if (( opt_duration )); then
+	echo "Tracing exec()s$ftext for $duration seconds (buffered)..."
+else
+	echo "Tracing exec()s$ftext. Ctrl-C to end."
+fi
+
+### select awk
+if (( opt_duration )); then
+	[[ -x /usr/bin/mawk ]] && awk=mawk || awk=awk
+else
+	# workarounds for mawk/gawk fflush behavior
+	if [[ -x /usr/bin/gawk ]]; then
+		awk=gawk
+	elif [[ -x /usr/bin/mawk ]]; then
+		awk="mawk -W interactive"
+	else
+		awk=awk
+	fi
+fi
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### build probe
+if [[ -x /usr/bin/getconf ]]; then
+	bits=$(getconf LONG_BIT)
+else
+	bits=64
+	[[ $(uname -m) == i* ]] && bits=32
+fi
+(( offset = bits / 8 ))
+function makeprobe {
+	func=$1
+	kname=execsnoop_$func
+	kprobe="p:$kname $func"
+	i=0
+	while (( i < argc + 1 )); do
+		# p:kname do_execve +0(+0(%si)):string +0(+8(%si)):string ...
+		kprobe="$kprobe +0(+$(( i * offset ))(%si)):string"
+		(( i++ ))
+	done
+}
+# try in this order: sys_execve, stub_execve, do_execve
+makeprobe sys_execve
+
+### setup and begin tracing
+echo nop > current_tracer
+if ! echo $kprobe >> kprobe_events 2>/dev/null; then
+	makeprobe stub_execve
+	if ! echo $kprobe >> kprobe_events 2>/dev/null; then
+	    makeprobe do_execve
+	    if ! echo $kprobe >> kprobe_events 2>/dev/null; then
+		    edie "ERROR: adding a kprobe for execve. Exiting."
+        fi
+	fi
+fi
+if ! echo 1 > events/kprobes/$kname/enable; then
+	edie "ERROR: enabling kprobe for execve. Exiting."
+fi
+if ! echo 1 > events/sched/sched_process_fork/enable; then
+	edie "ERROR: enabling sched:sched_process_fork tracepoint. Exiting."
+fi
+echo "Instrumenting $func"
+(( opt_time )) && printf "%-16s " "TIMEs"
+printf "%6s %6s %s\n" "PID" "PPID" "ARGS"
+
+#
+# Determine output format. It may be one of the following (newest first):
+#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
+#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
+# To differentiate between them, the number of header fields is counted,
+# and an offset set, to skip the extra column when needed.
+#
+offset=$($awk 'BEGIN { o = 0; }
+	$1 == "#" && $2 ~ /TASK/ && NF == 6 { o = 1; }
+	$2 ~ /TASK/ { print o; exit }' trace)
+
+### print trace buffer
+warn "echo > trace"
+( if (( opt_duration )); then
+	# wait then dump buffer
+	sleep $duration
+	cat -v trace
+else
+	# print buffer live
+	cat -v trace_pipe
+fi ) | $awk -v o=$offset -v opt_name=$opt_name -v name=$name \
+    -v opt_duration=$opt_duration -v opt_time=$opt_time -v kname=$kname \
+    -v opt_reexec=$opt_reexec '
+	# common fields
+	$1 != "#" {
+		# task name can contain dashes
+		comm = pid = $1
+		sub(/-[0-9][0-9]*/, "", comm)
+		sub(/.*-/, "", pid)
+	}
+
+	$1 != "#" && $(4+o) ~ /sched_process_fork/ {
+		cpid=$0
+		sub(/.* child_pid=/, "", cpid)
+		sub(/ .*/, "", cpid)
+		getppid[cpid] = pid
+		delete seen[pid]
+	}
+
+	$1 != "#" && $(4+o) ~ kname {
+		if (seen[pid])
+			next
+		if (opt_name && comm !~ name)
+			next
+
+		#
+		# examples:
+		# ... arg1="/bin/echo" arg2="1" arg3="2" arg4="3" ...
+		# ... arg1="sleep" arg2="2" arg3=(fault) arg4="" ...
+		# ... arg1="" arg2=(fault) arg3="" arg4="" ...
+		# the last example is uncommon, and may be a race.
+		#
+		if ($0 ~ /arg1=""/) {
+			args = comm " [?]"
+		} else {
+			args=$0
+			sub(/ arg[0-9]*=\(fault\).*/, "", args)
+			sub(/.*arg1="/, "", args)
+			gsub(/" arg[0-9]*="/, " ", args)
+			sub(/"$/, "", args)
+			if ($0 !~ /\(fault\)/)
+				args = args " [...]"
+		}
+
+		if (opt_time) {
+			time = $(3+o); sub(":", "", time)
+			printf "%-16s ", time
+		}
+		printf "%6s %6d %s\n", pid, getppid[pid], args
+		if (!opt_duration)
+			fflush()
+		if (!opt_reexec) {
+			seen[pid] = 1
+			delete getppid[pid]
+		}
+	}
+
+	$0 ~ /LOST.*EVENT[S]/ { print "WARNING: " $0 > "/dev/stderr" }
+'
+
+### end tracing
+end