fluent-plugin-perf-tools 0.1.0

data/perf-tools/kernel/funcslower

@@ -0,0 +1,248 @@
+#!/bin/bash
+#
+# funcslower - trace kernel functions slower than a threshold (microseconds).
+#              Uses Linux ftrace.
+#
+# This uses the Linux ftrace function graph profiler to time kernel functions
+# and filter them based on a latency threshold. This is a proof of concept using
+# Linux ftrace capabilities on older kernels.
+#
+# USAGE: funcslower [-aChHPt] [-p PID] [-d secs] funcstring latency_us
+#
+# Run "funcslower -h" for full usage.
+#
+# REQUIREMENTS: FTRACE function graph, which you may already have available
+# and enabled in recent kernels. And awk.
+#
+# The output format is the same as the ftrace function graph trace format,
+# described in the kernel source under Documentation/trace/ftrace.txt.
+# Note that the output may be shuffled when different CPU buffers are read;
+# check the CPU column for changes, or include timestamps (-t) and post sort.
+#
+# WARNING: This uses dynamic tracing of kernel functions, and could cause
+# kernel panics or freezes. Test, and know what you are doing, before use.
+#
+# OVERHEADS: Timing and filtering is performed in-kernel context, costing
+# lower overheads than post-processing in user space. If you trace frequent
+# events (eg, pick a common function and a low threshold), you might want to
+# try the "-d secs" option, which buffers events in-kernel instead of printing
+# them live.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 12-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock
+opt_duration=0; duration=; opt_pid=0; pid=; opt_tid=0; tid=
+pidtext=; opt_headers=0; opt_proc=0; opt_time=0; opt_cpu=0
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: funcslower [-aChHPt] [-p PID] [-L TID] [-d secs] funcstring latency_us
+	                 -a              # all info (same as -HPt)
+	                 -C              # measure on-CPU time only
+	                 -d seconds      # trace duration, and use buffers
+	                 -h              # this usage message
+	                 -H              # include column headers
+	                 -p PID          # trace when this pid is on-CPU
+	                 -L TID          # trace when this thread is on-CPU
+	                 -P              # show process names & PIDs
+	                 -t              # show timestamps
+	  eg,
+	       funcslower vfs_read 10000 # trace vfs_read() slower than 10 ms
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+
+	(( opt_time )) && warn "echo nofuncgraph-abstime > trace_options"
+	(( opt_proc )) && warn "echo nofuncgraph-proc > trace_options"
+	(( opt_cpu )) && warn "echo sleep-time > trace_options"
+
+	warn "echo nop > current_tracer"
+	(( opt_pid )) && warn "echo > set_ftrace_pid"
+	warn "echo > set_ftrace_filter"
+	warn "echo > set_graph_function"
+	warn "echo 0 > tracing_thresh"
+	warn "echo > trace"
+
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts aCd:hHp:L:Pt opt
+do
+	case $opt in
+	a)	opt_headers=1; opt_proc=1; opt_time=1 ;;
+	C)	opt_cpu=1; ;;
+	d)	opt_duration=1; duration=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+    L)  opt_tid=1; tid=$OPTARG ;;
+	H)	opt_headers=1; ;;
+	P)	opt_proc=1; ;;
+	t)	opt_time=1; ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+
+### option logic
+(( $# < 2 )) && usage
+(( opt_pid && opt_tid )) && edie "ERROR: You can use -p or -L but not both."
+funcs="$1"
+shift
+thresh=$1
+(( opt_pid )) && pidtext=" for PID $pid"
+(( opt_tid )) && pidtext=" for TID $tid"
+printf "Tracing \"$funcs\"$pidtext slower than $thresh us"
+if (( opt_duration )); then
+	echo " for $duration seconds..."
+else
+	echo "... Ctrl-C to end."
+fi
+
+## select awk
+if (( opt_duration )); then
+	[[ -x /usr/bin/mawk ]] && awk=mawk || awk=awk
+else
+	# workarounds for mawk/gawk fflush behavior
+	if [[ -x /usr/bin/gawk ]]; then
+		awk=gawk
+	elif [[ -x /usr/bin/mawk ]]; then
+		awk="mawk -W interactive"
+	else
+		awk=awk
+	fi
+fi
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### setup and commence tracing
+sysctl -q kernel.ftrace_enabled=1	# doesn't set exit status
+read mode < current_tracer
+[[ "$mode" != "nop" ]] && edie "ERROR: ftrace active (current_tracer=$mode)"
+if ! echo $thresh > tracing_thresh; then
+	edie "ERROR: setting tracing_thresh to $thresh. Exiting."
+fi
+if (( opt_pid )); then
+    echo '' > set_ftrace_pid
+    # ftrace expects kernel pids, which are thread ids
+    for tid in /proc/$pid/task/*; do
+        if ! echo ${tid##*/} >> set_ftrace_pid; then
+            edie "ERROR: setting -p $pid (PID exist?). Exiting."
+        fi
+    done
+fi
+if (( opt_tid )); then
+    if ! echo $tid > set_ftrace_pid; then
+        edie "ERROR: setting -L $tid (TID exist?). Exiting."
+    fi
+fi
+if ! echo "$funcs" > set_ftrace_filter; then
+	edie "ERROR: enabling \"$funcs\" filter. Function exist? Exiting."
+fi
+if ! echo "$funcs" > set_graph_function; then
+	edie "ERROR: enabling \"$funcs\" graph. Exiting."
+fi
+if ! echo function_graph > current_tracer; then
+	edie "ERROR: setting current_tracer to \"function_graph\". Exiting."
+fi
+if (( opt_cpu )); then
+	if ! echo nosleep-time > trace_options; then
+		edie "ERROR: setting -C (nosleep-time). Exiting."
+	fi
+fi
+# the following must be done after setting current_tracer
+if (( opt_time )); then
+	if ! echo funcgraph-abstime > trace_options; then
+		edie "ERROR: setting -t (funcgraph-abstime). Exiting."
+	fi
+fi
+if (( opt_proc )); then
+	if ! echo funcgraph-proc > trace_options; then
+		edie "ERROR: setting -P (funcgraph-proc). Exiting."
+	fi
+fi
+
+### setup output filter
+cat=cat
+if (( opt_proc )); then
+	# remove proc change entries, since PID is included. example:
+	#  ------------------------------------------
+	#  0)  supervi-1699  =>  supervi-1693 
+	#  ------------------------------------------
+	#
+	cat=$awk' "/(^ ---|^$)/ || \$3 == \"=>\" { next } { print \$0 }"'
+fi
+
+### print trace buffer
+warn "echo > trace"
+if (( opt_duration )); then
+	sleep $duration
+	if (( opt_headers )); then
+		$cat trace
+	else
+		$cat trace | grep -v '^#'
+	fi
+else
+	# trace_pipe lack headers, so fetch them from trace
+	(( opt_headers )) && cat trace
+	eval $cat trace_pipe
+fi
+
+### end tracing
+end

data/perf-tools/kernel/functrace

@@ -0,0 +1,192 @@
+#!/bin/bash
+#
+# functrace - trace kernel function calls matching specified wildcards.
+#             Uses Linux ftrace.
+#
+# This is a proof of concept using Linux ftrace capabilities on older kernels.
+#
+# USAGE: functrace [-hH] [-p PID] [-L TID] [-d secs] funcstring
+#    eg,
+#        functrace '*sleep'	# trace all functions ending in "sleep"
+#
+# Run "functrace -h" for full usage.
+#
+# The output format is the same as the ftrace function trace format, described
+# in the kernel source under Documentation/trace/ftrace.txt.
+#
+# The "-d duration" mode leaves the trace data in the kernel buffer, and
+# only reads it at the end. If the trace data is large, beware of exhausting
+# buffer space (/sys/kernel/debug/tracing/buffer_size_kb) and losing data.
+#
+# Also beware of feedback loops: tracing tcp* functions over an ssh session,
+# or writing ext4* functions to an ext4 file system. For the former, tcp
+# trace data could be redirected to a file (as in the usage message). For
+# the latter, trace to the screen or a different file system.
+#
+# WARNING: This uses dynamic tracing of kernel functions, and could cause
+# kernel panics or freezes. Test, and know what you are doing, before use.
+#
+# OVERHEADS: This can generate a lot of trace data quickly, depending on the
+# frequency of the traced events. Such data will cause performance overheads.
+# This also works without buffering by default, printing function events
+# as they happen (uses trace_pipe), context switching and consuming CPU to do
+# so. If needed, you can try the "-d secs" option, which buffers events
+# instead, reducing overhead. If you think the buffer option is losing events,
+# try increasing the buffer size (buffer_size_kb).
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 12-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock
+opt_duration=0; duration=; opt_pid=0; pid=; opt_tid=0; tid=; pidtext=
+opt_headers=0
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: functrace [-hH] [-p PID] [-L TID] [-d secs] funcstring
+	                 -d seconds      # trace duration, and use buffers
+	                 -h              # this usage message
+	                 -H              # include column headers
+	                 -p PID          # trace when this pid is on-CPU
+	                 -L TID          # trace when this thread is on-CPU
+	  eg,
+	       functrace do_nanosleep    # trace the do_nanosleep() function
+	       functrace '*sleep'        # trace functions ending in "sleep"
+	       functrace -p 198 'vfs*'   # trace "vfs*" funcs for PID 198
+	       functrace 'tcp*' > out    # trace all "tcp*" funcs to out file
+	       functrace -d 1 'tcp*' > out  # trace 1 sec, then write out file
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+	warn "echo nop > current_tracer"
+	(( opt_pid || opt_tid )) && warn "echo > set_ftrace_pid"
+	warn "echo > set_ftrace_filter"
+	warn "echo > trace"
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts d:hHp:L: opt
+do
+	case $opt in
+	d)	opt_duration=1; duration=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+	L)	opt_tid=1; tid=$OPTARG ;;
+	H)	opt_headers=1; ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+
+### option logic
+(( $# == 0 )) && usage
+(( opt_pid && opt_tid )) && edie "ERROR: You can use -p or -L but not both."
+funcs="$1"
+(( opt_pid )) && pidtext=" for PID $pid"
+(( opt_tid )) && pidtext=" for TID $pid"
+if (( opt_duration )); then
+	echo "Tracing \"$funcs\"$pidtext for $duration seconds..."
+else
+	echo "Tracing \"$funcs\"$pidtext... Ctrl-C to end."
+fi
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### setup and commence tracing
+sysctl -q kernel.ftrace_enabled=1	# doesn't set exit status
+read mode < current_tracer
+[[ "$mode" != "nop" ]] && edie "ERROR: ftrace active (current_tracer=$mode)"
+if (( opt_pid )); then
+    echo > set_ftrace_pid
+    # ftrace expects kernel pids, which are thread ids
+    for tid in /proc/$pid/task/*; do
+        if ! echo ${tid##*/} >> set_ftrace_pid; then
+            edie "ERROR: setting -p $pid (PID exist?). Exiting."
+        fi
+    done
+fi
+if (( opt_tid )); then
+    if ! echo $tid > set_ftrace_pid; then
+        edie "ERROR: setting -L $tid (TID exist?). Exiting."
+    fi
+fi
+if ! echo "$funcs" > set_ftrace_filter; then
+	edie "ERROR: enabling \"$funcs\". Exiting."
+fi
+if ! echo function > current_tracer; then
+	edie "ERROR: setting current_tracer to \"function\". Exiting."
+fi
+
+### print trace buffer
+warn "echo > trace"
+if (( opt_duration )); then
+	sleep $duration
+	if (( opt_headers )); then
+		cat trace
+	else
+		grep -v '^#' trace
+	fi
+else
+	# trace_pipe lack headers, so fetch them from trace
+	(( opt_headers )) && cat trace
+	cat trace_pipe
+fi
+
+### end tracing
+end

data/perf-tools/kernel/kprobe

@@ -0,0 +1,270 @@
+#!/bin/bash
+#
+# kprobe - trace a given kprobe definition. Kernel dynamic tracing.
+#          Written using Linux ftrace.
+#
+# This will create, trace, then destroy a given kprobe definition. See
+# Documentation/trace/kprobetrace.txt in the Linux kernel source for the
+# syntax of a kprobe definition, and "kprobe -h" for examples. With this tool,
+# the probe alias is optional (it will become to kprobe:<funcname> if not
+# specified).
+#
+# USAGE: ./kprobe [-FhHsv] [-d secs] [-p pid] [-L tid] kprobe_definition [filter]
+#
+# Run "kprobe -h" for full usage.
+#
+# I wrote this because I kept testing different custom kprobes at the command
+# line, and wanted a way to automate the steps.
+#
+# WARNING: This uses dynamic tracing of kernel functions, and could cause
+# kernel panics or freezes, depending on the function traced. Test in a lab
+# environment, and know what you are doing, before use.
+#
+# REQUIREMENTS: FTRACE and KPROBE CONFIG, which you may already have on recent
+# kernel versions.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the kprobe(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 22-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock; wroteflock=0
+opt_duration=0; duration=; opt_pid=0; pid=; opt_tid=0; tid=
+opt_filter=0; filter=; opt_view=0; opt_headers=0; opt_stack=0; dmesg=2
+debug=0; opt_force=0
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: kprobe [-FhHsv] [-d secs] [-p PID] [-L TID] kprobe_definition [filter]
+	                 -F              # force. trace despite warnings.
+	                 -d seconds      # trace duration, and use buffers
+	                 -p PID          # PID to match on events
+	                 -L TID          # thread id to match on events
+	                 -v              # view format file (don't trace)
+	                 -H              # include column headers
+	                 -s              # show kernel stack traces
+	                 -h              # this usage message
+	
+	Note that these examples may need modification to match your kernel
+	version's function names and platform's register usage.
+	   eg,
+	       kprobe p:do_sys_open
+	                                 # trace open() entry
+	       kprobe r:do_sys_open
+	                                 # trace open() return
+	       kprobe 'r:do_sys_open \$retval'
+	                                 # trace open() return value
+	       kprobe 'r:myopen do_sys_open \$retval'
+	                                 # use a custom probe name
+	       kprobe 'p:myopen do_sys_open mode=%cx:u16'
+	                                 # trace open() file mode
+	       kprobe 'p:myopen do_sys_open filename=+0(%si):string'
+	                                 # trace open() with filename
+	       kprobe -s 'p:myprobe tcp_retransmit_skb'
+	                                 # show kernel stacks
+	       kprobe 'p:do_sys_open file=+0(%si):string' 'file ~ "*stat"'
+	                                 # opened files ending in "stat"
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+	warn "echo 0 > events/kprobes/$kname/enable"
+	if (( opt_filter )); then
+		warn "echo 0 > events/kprobes/$kname/filter"
+	fi
+	warn "echo -:$kname >> kprobe_events"
+	(( opt_stack )) && warn "echo 0 > options/stacktrace"
+	warn "echo > trace"
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts Fd:hHp:L:sv opt
+do
+	case $opt in
+	F)	opt_force=1 ;;
+	d)	opt_duration=1; duration=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+	L)	opt_tid=1; tid=$OPTARG ;;
+	H)	opt_headers=1 ;;
+	s)	opt_stack=1 ;;
+	v)	opt_view=1 ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+(( $# )) || usage
+kprobe=$1
+shift
+if (( $# )); then
+	opt_filter=1
+	filter=$1
+fi
+
+### option logic
+(( opt_pid + opt_filter + opt_tid > 1 )) && \
+	die "ERROR: use at most one of -p, -L, or filter."
+(( opt_duration && opt_view )) && die "ERROR: use either -d or -v."
+if (( opt_pid )); then
+	# convert to filter
+	opt_filter=1
+	# ftrace common_pid is thread id from user's perspective
+	for tid in /proc/$pid/task/*; do
+		filter="$filter || common_pid == ${tid##*/}"
+	done
+	filter=${filter:3}  # trim leading ' || ' (four characters)
+fi
+if (( opt_tid )); then
+	opt_filter=1
+	filter="common_pid == $tid"
+fi
+if [[ "$kprobe" != p:* && "$kprobe" != r:* ]]; then
+	echo >&2 "ERROR: invalid kprobe definition (should start with p: or r:)"
+	usage
+fi
+#
+# parse the following:
+# r:do_sys_open
+# r:my_sys_open do_sys_open
+# r:do_sys_open %ax
+# r:do_sys_open $retval %ax
+# r:my_sys_open do_sys_open $retval %ax
+# r:do_sys_open rval=$retval
+# r:my_sys_open do_sys_open rval=$retval
+# r:my_sys_open do_sys_open rval=$retval %ax
+# ... and examples from USAGE message
+#
+krest=${kprobe#*:}
+kname=${krest%% *}
+set -- $krest
+if [[ $2 == "" || $2 == *[=%\$]* ]]; then
+	# if probe name unspecified, default to function name
+	ktype=${kprobe%%:*}
+	kprobe="$ktype:$kname $krest"
+fi
+if (( debug )); then
+	echo "kname: $kname, kprobe: $kprobe"
+fi
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+## check function
+set -- $kprobe
+fname=$2
+if (( !opt_force )) && ! grep -w $fname available_filter_functions >/dev/null \
+    2>&1
+then
+	echo >&2 "ERROR: func $fname not in $PWD/available_filter_functions."
+	printf >&2 "Either it doesn't exist, or, it might be unsafe to kprobe. "
+	echo >&2 "Exiting. Use -F to override."
+	exit 1
+fi
+
+if (( !opt_view )); then
+	if (( opt_duration )); then
+		echo "Tracing kprobe $kname for $duration seconds (buffered)..."
+	else
+		echo "Tracing kprobe $kname. Ctrl-C to end."
+	fi
+fi
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### setup and begin tracing
+echo nop > current_tracer
+if ! echo "$kprobe" >> kprobe_events; then
+	echo >&2 "ERROR: adding kprobe \"$kprobe\"."
+	if (( dmesg )); then
+		echo >&2 "Last $dmesg dmesg entries (might contain reason):"
+		dmesg | tail -$dmesg | sed 's/^/    /'
+	fi
+	edie "Exiting."
+fi
+if (( opt_view )); then
+	cat events/kprobes/$kname/format
+	edie ""
+fi
+if (( opt_filter )); then
+	if ! echo "$filter" > events/kprobes/$kname/filter; then
+		edie "ERROR: setting filter or -p. Exiting."
+	fi
+fi
+if (( opt_stack )); then
+	if ! echo 1 > options/stacktrace; then
+		edie "ERROR: enabling stack traces (-s). Exiting"
+	fi
+fi
+if ! echo 1 > events/kprobes/$kname/enable; then
+	edie "ERROR: enabling kprobe $kname. Exiting."
+fi
+
+### print trace buffer
+warn "echo > trace"
+if (( opt_duration )); then
+	sleep $duration
+	if (( opt_headers )); then
+		cat trace
+	else
+		grep -v '^#' trace
+	fi
+else
+	# trace_pipe lack headers, so fetch them from trace
+	(( opt_headers )) && cat trace
+	cat trace_pipe
+fi
+
+### end tracing
+end