fluent-plugin-perf-tools 0.1.0

data/perf-tools/iosnoop

@@ -0,0 +1,296 @@
+#!/bin/bash
+#
+# iosnoop - trace block device I/O.
+#           Written using Linux ftrace.
+#
+# This traces disk I/O at the block device interface, using the block:
+# tracepoints. This can help characterize the I/O requested for the storage
+# devices and their resulting performance. I/O completions can also be studied
+# event-by-event for debugging disk and controller I/O scheduling issues.
+#
+# USAGE: ./iosnoop [-hQst] [-d device] [-i iotype] [-p pid] [-n name] [duration]
+#
+# Run "iosnoop -h" for full usage.
+#
+# REQUIREMENTS: FTRACE CONFIG, block:block_rq_* tracepoints (you may
+# already have these on recent kernels).
+#
+# OVERHEAD: By default, iosnoop works without buffering, printing I/O events
+# as they happen (uses trace_pipe), context switching and consuming CPU to do
+# so. This has a limit of about 10,000 IOPS (depending on your platform), at
+# which point iosnoop will be consuming 1 CPU. The duration mode uses buffering,
+# and can handle much higher IOPS rates, however, the buffer has a limit of
+# about 50,000 I/O, after which events will be dropped. You can tune this with
+# bufsize_kb, which is per-CPU. Also note that the "-n" option is currently
+# post-filtered, so all events are traced.
+#
+# This was written as a proof of concept for ftrace. It would be better written
+# using perf_events (after some capabilities are added), which has a better
+# buffering policy, or a tracer such as SystemTap or ktap.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the iosnoop(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 12-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock
+bufsize_kb=4096
+opt_duration=0; duration=; opt_name=0; name=; opt_pid=0; pid=; ftext=
+opt_start=0; opt_end=0; opt_device=0; device=; opt_iotype=0; iotype=
+opt_queue=0
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: iosnoop [-hQst] [-d device] [-i iotype] [-p PID] [-n name]
+	               [duration]
+	                 -d device       # device string (eg, "202,1)
+	                 -i iotype       # match type (eg, '*R*' for all reads)
+	                 -n name         # process name to match on I/O issue
+	                 -p PID          # PID to match on I/O issue
+	                 -Q              # use queue insert as start time
+	                 -s              # include start time of I/O (s)
+	                 -t              # include completion time of I/O (s)
+	                 -h              # this usage message
+	                 duration        # duration seconds, and use buffers
+	  eg,
+	       iosnoop                   # watch block I/O live (unbuffered)
+	       iosnoop 1                 # trace 1 sec (buffered)
+	       iosnoop -Q                # include queueing time in LATms
+	       iosnoop -ts               # include start and end timestamps
+	       iosnoop -i '*R*'          # trace reads
+	       iosnoop -p 91             # show I/O issued when PID 91 is on-CPU
+	       iosnoop -Qp 91            # show I/O queued by PID 91, queue time
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+	warn "echo 0 > events/block/$b_start/enable"
+	warn "echo 0 > events/block/block_rq_complete/enable"
+	if (( opt_device || opt_iotype || opt_pid )); then
+		warn "echo 0 > events/block/$b_start/filter"
+		warn "echo 0 > events/block/block_rq_complete/filter"
+	fi
+	warn "echo > trace"
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts d:hi:n:p:Qst opt
+do
+	case $opt in
+	d)	opt_device=1; device=$OPTARG ;;
+	i)	opt_iotype=1; iotype=$OPTARG ;;
+	n)	opt_name=1; name=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+	Q)	opt_queue=1 ;;
+	s)	opt_start=1 ;;
+	t)	opt_end=1 ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+if (( $# )); then
+	opt_duration=1
+	duration=$1
+	shift
+fi
+if (( opt_device )); then
+	major=${device%,*}
+	minor=${device#*,}
+	dev=$(( (major << 20) + minor ))
+fi
+
+### option logic
+(( opt_pid && opt_name )) && die "ERROR: use either -p or -n."
+(( opt_pid )) && ftext=" issued by PID $pid"
+(( opt_name )) && ftext=" issued by process name \"$name\""
+if (( opt_duration )); then
+	echo "Tracing block I/O$ftext for $duration seconds (buffered)..."
+else
+	echo "Tracing block I/O$ftext. Ctrl-C to end."
+fi
+if (( opt_queue )); then
+	b_start=block_rq_insert
+else
+	b_start=block_rq_issue
+fi
+
+### select awk
+(( opt_duration )) && use=mawk || use=gawk	# workaround for mawk fflush()
+[[ -x /usr/bin/$use ]] && awk=$use || awk=awk
+wroteflock=1
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+
+### setup and begin tracing
+echo nop > current_tracer
+warn "echo $bufsize_kb > buffer_size_kb"
+filter=
+if (( opt_iotype )); then
+	filter="rwbs ~ \"$iotype\""
+fi
+if (( opt_device )); then
+	[[ "$filter" != "" ]] && filter="$filter && "
+	filter="${filter}dev == $dev"
+fi
+filter_i=$filter
+if (( opt_pid )); then
+	[[ "$filter_i" != "" ]] && filter_i="$filter_i && "
+	filter_i="${filter_i}common_pid == $pid"
+	[[ "$filter" == "" ]] && filter=0
+fi
+if (( opt_iotype || opt_device || opt_pid )); then
+	if ! echo "$filter_i" > events/block/$b_start/filter || \
+	    ! echo "$filter" > events/block/block_rq_complete/filter
+	then
+		edie "ERROR: setting -d or -t filter. Exiting."
+	fi
+fi
+if ! echo 1 > events/block/$b_start/enable || \
+    ! echo 1 > events/block/block_rq_complete/enable; then
+	edie "ERROR: enabling block I/O tracepoints. Exiting."
+fi
+(( opt_start )) && printf "%-15s " "STARTs"
+(( opt_end )) && printf "%-15s " "ENDs"
+printf "%-12.12s %-6s %-4s %-8s %-12s %-6s %8s\n" \
+    "COMM" "PID" "TYPE" "DEV" "BLOCK" "BYTES" "LATms"
+
+#
+# Determine output format. It may be one of the following (newest first):
+#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
+#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
+# To differentiate between them, the number of header fields is counted,
+# and an offset set, to skip the extra column when needed.
+#
+offset=$($awk 'BEGIN { o = 0; }
+	$1 == "#" && $2 ~ /TASK/ && NF == 6 { o = 1; }
+	$2 ~ /TASK/ { print o; exit }' trace)
+
+### print trace buffer
+warn "echo > trace"
+( if (( opt_duration )); then
+	# wait then dump buffer
+	sleep $duration
+	cat trace
+else
+	# print buffer live
+	cat trace_pipe
+fi ) | $awk -v o=$offset -v opt_name=$opt_name -v name=$name \
+    -v opt_duration=$opt_duration -v opt_start=$opt_start -v opt_end=$opt_end \
+    -v b_start=$b_start '
+	# common fields
+	$1 != "#" {
+		# task name can contain dashes
+		comm = pid = $1
+		sub(/-[0-9][0-9]*/, "", comm)
+		sub(/.*-/, "", pid)
+		time = $(3+o); sub(":", "", time)
+		dev = $(5+o)
+	}
+
+	# block I/O request
+	$1 != "#" && $0 ~ b_start {
+		if (opt_name && match(comm, name) == 0)
+			next
+		#
+		# example: (fields1..4+o) 202,1 W 0 () 12862264 + 8 [tar]
+		# The cmd field "()" might contain multiple words (hex),
+		# hence stepping from the right (NF-3).
+		#
+		loc = $(NF-3)
+		starts[dev, loc] = time
+		comms[dev, loc] = comm
+		pids[dev, loc] = pid
+		next
+	}
+
+	# block I/O completion
+	$1 != "#" && $0 ~ /rq_complete/ {
+		#
+		# example: (fields1..4+o) 202,1 W () 12862256 + 8 [0]
+		#
+		dir = $(6+o)
+		loc = $(NF-3)
+		nsec = $(NF-1)
+
+		if (starts[dev, loc] > 0) {
+			latency = sprintf("%.2f",
+			    1000 * (time - starts[dev, loc]))
+			comm = comms[dev, loc]
+			pid = pids[dev, loc]
+
+			if (opt_start)
+				printf "%-15s ", starts[dev, loc]
+			if (opt_end)
+				printf "%-15s ", time
+			printf "%-12.12s %-6s %-4s %-8s %-12s %-6s %8s\n",
+			    comm, pid, dir, dev, loc, nsec * 512, latency
+			if (!opt_duration)
+				fflush()
+
+			delete starts[dev, loc]
+			delete comms[dev, loc]
+			delete pids[dev, loc]
+		}
+		next
+	}
+
+	$0 ~ /LOST.*EVENTS/ { print "WARNING: " $0 > "/dev/stderr" }
+'
+
+### end tracing
+end

data/perf-tools/kernel/funccount

@@ -0,0 +1,146 @@
+#!/bin/bash
+#
+# funccount - count kernel function calls matching specified wildcards.
+#             Uses Linux ftrace.
+#
+# This is a proof of concept using Linux ftrace capabilities on older kernels,
+# and works by using function profiling: in-kernel counters.
+#
+# USAGE: funccount [-hT] [-i secs] [-d secs] [-t top] funcstring
+#    eg,
+#        funccount 'ext3*'	# count all ext3* kernel function calls
+#
+# Run "funccount -h" for full usage.
+#
+# WARNING: This uses dynamic tracing of kernel functions, and could cause
+# kernel panics or freezes. Test, and know what you are doing, before use.
+#
+# REQUIREMENTS: CONFIG_FUNCTION_PROFILER, awk.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 12-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+opt_duration=0; duration=; opt_interval=0; interval=999999; opt_timestamp=0
+opt_tail=0; tcmd=cat; ttext=
+trap 'quit=1' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: funccount [-hT] [-i secs] [-d secs] [-t top] funcstring
+	                 -d seconds      # total duration of trace
+	                 -h              # this usage message
+	                 -i seconds      # interval summary
+	                 -t top          # show top num entries only
+	                 -T              # include timestamp (for -i)
+	  eg,
+	       funccount 'vfs*'          # trace all funcs that match "vfs*"
+	       funccount -d 5 'tcp*'     # trace "tcp*" funcs for 5 seconds
+	       funccount -t 10 'ext3*'   # show top 10 "ext3*" funcs
+	       funccount -i 1 'ext3*'    # summary every 1 second
+	       funccount -i 1 -d 5 'ext3*' # 5 x 1 second summaries
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+### process options
+while getopts d:hi:t:T opt
+do
+	case $opt in
+	d)	opt_duration=1; duration=$OPTARG ;;
+	i)	opt_interval=1; interval=$OPTARG ;;
+	t)	opt_tail=1; tnum=$OPTARG ;;
+	T)	opt_timestamp=1 ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+
+### option logic
+(( $# == 0 )) && usage
+funcs="$1"
+if (( opt_tail )); then
+	tcmd="tail -$tnum"
+	ttext=" Top $tnum only."
+fi
+if (( opt_duration )); then
+	echo "Tracing \"$funcs\" for $duration seconds.$ttext.."
+else
+	echo "Tracing \"$funcs\".$ttext.. Ctrl-C to end."
+fi
+(( opt_duration && !opt_interval )) && interval=$duration
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### enable tracing
+sysctl -q kernel.ftrace_enabled=1	# doesn't set exit status
+echo "$funcs" > set_ftrace_filter || die "ERROR: enabling \"$funcs\". Exiting."
+warn "echo nop > current_tracer"
+if ! echo 1 > function_profile_enabled; then
+	echo > set_ftrace_filter
+	die "ERROR: enabling function profiling."\
+	    "Have CONFIG_FUNCTION_PROFILER? Exiting."
+fi
+
+### summarize
+quit=0; secs=0
+while (( !quit && (!opt_duration || secs < duration) )); do
+	(( secs += interval ))
+	echo 0 > function_profile_enabled
+	echo 1 > function_profile_enabled
+	sleep $interval
+
+	echo
+	(( opt_timestamp )) && date
+	printf "%-30s %8s\n" "FUNC" "COUNT"
+
+	cat trace_stat/function* | awk '
+	# skip headers by matching on the numeric hit column
+	$2 ~ /[0-9]/ { a[$1] += $2 }
+	END {
+		for (k in a) {
+			printf "%-30s %8d\n", k,  a[k]
+		}
+	}' | sort -n -k2 | $tcmd
+done
+
+### end tracing
+echo 2>/dev/null
+echo "Ending tracing..." 2>/dev/null
+warn "echo 0 > function_profile_enabled"
+warn "echo > set_ftrace_filter"

data/perf-tools/kernel/funcgraph

@@ -0,0 +1,259 @@
+#!/bin/bash
+#
+# funcgraph - trace kernel function graph, showing child function calls.
+#             Uses Linux ftrace.
+#
+# This is an exploratory tool that shows the graph of child function calls
+# for a given kernel function. This can cost moderate overhead to execute, and
+# should only be used to understand kernel behavior for a given function before
+# using other, lower overhead tools. This is a proof of concept using Linux
+# ftrace capabilities on older kernels.
+#
+# USAGE: funcgraph [-aCDhHPtT] [-m maxdepth] [-p PID] [-L TID] [-d secs] funcstring
+#
+# Run "funcgraph -h" for full usage.
+#
+# The output format is the same as the ftrace function graph trace format,
+# described in the kernel source under Documentation/trace/ftrace.txt.
+# Note that the output may be shuffled when different CPU buffers are read;
+# check the CPU column for changes, or include timestamps (-t) and post sort.
+#
+# The "-d duration" mode leaves the trace data in the kernel buffer, and
+# only reads it at the end. If the trace data is large, beware of exhausting
+# buffer space (/sys/kernel/debug/tracing/buffer_size_kb) and losing data.
+#
+# Also beware of feedback loops: tracing tcp* functions over an ssh session,
+# or writing ext4* functions to an ext4 file system. For the former, tcp
+# trace data could be redirected to a file (as in the usage message). For
+# the latter, trace to the screen or a different file system.
+#
+# WARNING: This uses dynamic tracing of kernel functions, and could cause
+# kernel panics or freezes. Test, and know what you are doing, before use.
+#
+# OVERHEADS: This tool causes moderate to high overheads. Use with caution for
+# exploratory purposes, then switch to lower overhead techniques based on
+# findings. It's expected that the kernel will run at least 50% slower while
+# this tool is running -- even while no output is being generated. This is
+# because ALL kernel functions are traced, and filtered based on the function
+# of interest. When output is generated, it can generate many lines quickly
+# depending on the traced event. Such data will cause performance overheads.
+# This also works without buffering by default, printing function events
+# as they happen (uses trace_pipe), context switching and consuming CPU to do
+# so. If needed, you can try the "-d secs" option, which buffers events
+# instead, reducing overhead. If you think the buffer option is losing events,
+# try increasing the buffer size (buffer_size_kb).
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 12-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock
+opt_duration=0; duration=; opt_pid=0; pid=; opt_tid=0; tid=; pidtext=
+opt_headers=0; opt_proc=0; opt_time=0; opt_tail=0; opt_nodur=0; opt_cpu=0
+opt_max=0; max=0
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: funcgraph [-aCDhHPtT] [-m maxdepth] [-p PID] [-L TID] [-d secs] funcstring
+	                 -a              # all info (same as -HPt)
+	                 -C              # measure on-CPU time only
+	                 -d seconds      # trace duration, and use buffers
+	                 -D              # do not show function duration
+	                 -h              # this usage message
+	                 -H              # include column headers
+	                 -m maxdepth     # max stack depth to show
+	                 -p PID          # trace when this pid is on-CPU
+	                 -L TID          # trace when this thread is on-CPU
+	                 -P              # show process names & PIDs
+	                 -t              # show timestamps
+	                 -T              # comment function tails
+	  eg,
+	       funcgraph do_nanosleep    # trace do_nanosleep() and children
+	       funcgraph -m 3 do_sys_open # trace do_sys_open() to 3 levels only
+	       funcgraph -a do_sys_open    # include timestamps and process name
+	       funcgraph -p 198 do_sys_open # trace vfs_read() for PID 198 only
+	       funcgraph -d 1 do_sys_open >out # trace 1 sec, then write to file
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+
+	(( opt_time )) && warn "echo nofuncgraph-abstime > trace_options"
+	(( opt_proc )) && warn "echo nofuncgraph-proc > trace_options"
+	(( opt_tail )) && warn "echo nofuncgraph-tail > trace_options"
+	(( opt_nodur )) && warn "echo funcgraph-duration > trace_options"
+	(( opt_cpu )) && warn "echo sleep-time > trace_options"
+
+	warn "echo nop > current_tracer"
+	(( opt_pid || opt_tid )) && warn "echo > set_ftrace_pid"
+	(( opt_max )) && warn "echo 0 > max_graph_depth"
+	warn "echo > set_graph_function"
+	warn "echo > trace"
+
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts aCd:DhHm:p:L:PtT opt
+do
+	case $opt in
+	a)	opt_headers=1; opt_proc=1; opt_time=1 ;;
+	C)	opt_cpu=1; ;;
+	d)	opt_duration=1; duration=$OPTARG ;;
+	D)	opt_nodur=1; ;;
+	m)	opt_max=1; max=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+	L)	opt_tid=1; tid=$OPTARG ;;
+	H)	opt_headers=1; ;;
+	P)	opt_proc=1; ;;
+	t)	opt_time=1; ;;
+	T)	opt_tail=1; ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+
+### option logic
+(( $# == 0 )) && usage
+(( opt_pid && opt_tid )) && edie "ERROR: You can use -p or -L but not both."
+funcs="$1"
+(( opt_pid )) && pidtext=" for PID $pid"
+(( opt_tid )) && pidtext=" for TID $tid"
+if (( opt_duration )); then
+	echo "Tracing \"$funcs\"$pidtext for $duration seconds..."
+else
+	echo "Tracing \"$funcs\"$pidtext... Ctrl-C to end."
+fi
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### setup and commence tracing
+sysctl -q kernel.ftrace_enabled=1	# doesn't set exit status
+read mode < current_tracer
+[[ "$mode" != "nop" ]] && edie "ERROR: ftrace active (current_tracer=$mode)"
+if (( opt_max )); then
+	if ! echo $max > max_graph_depth; then
+		edie "ERROR: setting -m $max. Older kernel version? Exiting."
+	fi
+fi
+if (( opt_pid )); then
+    echo > set_ftrace_pid
+    # ftrace expects kernel pids, which are thread ids
+    for tid in /proc/$pid/task/*; do
+        if ! echo ${tid##*/} >> set_ftrace_pid; then
+            edie "ERROR: setting -p $pid (PID exist?). Exiting."
+        fi
+    done
+fi
+if (( opt_tid )); then
+    if ! echo $tid > set_ftrace_pid; then
+        edie "ERROR: setting -L $tid (TID exist?). Exiting."
+    fi
+fi
+if ! echo > set_ftrace_filter; then
+	edie "ERROR: writing to set_ftrace_filter. Exiting."
+fi
+if ! echo "$funcs" > set_graph_function; then
+	edie "ERROR: enabling \"$funcs\". Exiting."
+fi
+if ! echo function_graph > current_tracer; then
+	edie "ERROR: setting current_tracer to \"function\". Exiting."
+fi
+if (( opt_cpu )); then
+	if ! echo nosleep-time > trace_options; then
+		edie "ERROR: setting -C (nosleep-time). Exiting."
+	fi
+fi
+# the following must be done after setting current_tracer
+if (( opt_time )); then
+	if ! echo funcgraph-abstime > trace_options; then
+		edie "ERROR: setting -t (funcgraph-abstime). Exiting."
+	fi
+fi
+if (( opt_proc )); then
+	if ! echo funcgraph-proc > trace_options; then
+		edie "ERROR: setting -P (funcgraph-proc). Exiting."
+	fi
+fi
+if (( opt_tail )); then
+	if ! echo funcgraph-tail > trace_options; then
+		edie "ERROR: setting -T (funcgraph-tail). Old kernel? Exiting."
+	fi
+fi
+if (( opt_nodur )); then
+	if ! echo nofuncgraph-duration > trace_options; then
+		edie "ERROR: setting -D (nofuncgraph-duration). Exiting."
+	fi
+fi
+
+### print trace buffer
+warn "echo > trace"
+if (( opt_duration )); then
+	sleep $duration
+	if (( opt_headers )); then
+		cat trace
+	else
+		grep -v '^#' trace
+	fi
+else
+	# trace_pipe lack headers, so fetch them from trace
+	(( opt_headers )) && cat trace
+	cat trace_pipe
+fi
+
+### end tracing
+end