fluent-plugin-perf-tools 0.1.0

data/perf-tools/syscount

@@ -0,0 +1,192 @@
+#!/bin/bash
+#
+# syscount - count system calls.
+#            Written using Linux perf_events (aka "perf").
+#
+# This is a proof-of-concept using perf_events capabilities for older kernel
+# versions, that lack custom in-kernel aggregations. Once they exist, this
+# script can be substantially rewritten and improved (lower overhead).
+#
+# USAGE: syscount [-chv] [-t top] {-p PID|-d seconds|command}
+#
+# Run "syscount -h" for full usage.
+#
+# REQUIREMENTS: Linux perf_events: add linux-tools-common, run "perf", then
+# add any additional packages it requests. Also needs awk.
+#
+# OVERHEADS: Modes that report syscall names only (-c, -cp PID, -cd secs) have
+# lower overhead, since they use in-kernel counts. Other modes which report
+# process IDs (-cv) or process names (default) create a perf.data file for
+# post processing, and you will see messages about it doing this. Beware of
+# the file size (test for short durations, or use -c to see counts based on
+# in-kernel counters), and gauge overheads based on the perf.data size.
+#
+# Note that this script delibrately does not pipe perf record into
+# perf script, which would avoid perf.data, because it can create a feedback
+# loop where the perf script syscalls are recorded. Hopefully there will be a
+# fix for this in a later perf version, so perf.data can be skipped, or other
+# kernel features to aggregate by process name in-kernel directly (eg, via
+# eBPF, ktap, or SystemTap).
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the syscount(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 07-Jul-2014	Brendan Gregg	Created this.
+
+# default variables
+opt_count=0; opt_pid=0; opt_verbose=0; opt_cmd=0; opt_duration=0; opt_tail=0
+tnum=; pid=; duration=; cmd=; cpus=-a; opts=; tcmd=cat; ttext=
+trap '' INT QUIT TERM PIPE HUP
+
+stdout_workaround=1	# needed for older perf versions
+write_workaround=1	# needed for perf versions that trace their own writes
+
+### parse options
+while getopts cd:hp:t:v opt
+do
+	case $opt in
+	c)	opt_count=1 ;;
+	d)	opt_duration=1; duration=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+	t)	opt_tail=1; tnum=$OPTARG ;;
+	v)	opt_verbose=1 ;;
+	h|?)	cat <<-END >&2
+		USAGE: syscount [-chv] [-t top] {-p PID|-d seconds|command}
+		       syscount                  # count by process name
+		                -c               # show counts by syscall name
+		                -h               # this usage message
+		                -v               # verbose: shows PID
+		                -p PID           # trace this PID only
+		                -d seconds       # duration of trace
+		                -t num           # show top number only
+		                command          # run and trace this command
+		  eg,
+		        syscount                 # syscalls by process name
+		        syscount -c              # syscalls by syscall name
+		        syscount -d 5            # trace for 5 seconds
+		        syscount -cp 923         # syscall names for PID 923
+		        syscount -c ls           # syscall names for "ls"
+
+		See the man page and example file for more info.
+		END
+		exit 1
+	esac
+done
+shift $(( $OPTIND - 1 ))
+
+### option logic
+if (( $# > 0 )); then
+	opt_cmd=1
+	cmd="$@"
+	cpus=
+fi
+if (( opt_pid + opt_duration + opt_cmd > 1 )); then
+	echo >&2 "ERROR: Pick one of {-p PID|-n name|-d seconds|command}"
+	exit 1
+fi
+if (( opt_tail )); then
+	tcmd="tail -$tnum"
+	ttext=" Top $tnum only."
+fi
+if (( opt_duration )); then
+	cmd="sleep $duration"
+	echo "Tracing for $duration seconds.$ttext.."
+fi
+if (( opt_pid )); then
+	cpus=
+	cmd="-p $pid"
+	echo "Tracing PID $pid.$ttext.. Ctrl-C to end."
+fi
+(( opt_cmd )) && echo "Tracing while running: \"$cmd\".$ttext.."
+(( opt_pid + opt_duration + opt_cmd == 0 )) && \
+    echo "Tracing.$ttext.. Ctrl-C to end."
+(( stdout_workaround )) && opts="-o /dev/stdout"
+
+ulimit -n 32768		# often needed
+
+### execute syscall name mode
+if (( opt_count && ! opt_verbose )); then
+	: ${cmd:=sleep 999999}
+	out=$(perf stat $opts -e 'syscalls:sys_enter_*' $cpus $cmd)
+	printf "%-17s %8s\n" "SYSCALL" "COUNT"
+	echo "$out" | awk '
+	$1 && $2 ~ /syscalls:/ {
+		sub("syscalls:sys_enter_", ""); sub(":", "")
+		gsub(",", "")
+		printf "%-17s %8s\n", $2, $1
+	}' | sort -n -k2 | $tcmd
+	exit
+fi
+
+### execute syscall name with pid mode
+if (( opt_count && opt_verbose )); then
+	if (( write_workaround )); then
+		# this list must end in write to associate the filter
+		tp=$(perf list syscalls:sys_enter_* | awk '
+		    $1 != "syscalls:sys_enter_write" &&  $1 ~ /syscalls:/ { printf "-e %s ", $1 }')
+		tp="$tp -e syscalls:sys_enter_write"
+		sh -c "perf record $tp --filter 'common_pid != '\$\$ $cpus $cmd"
+	else
+		perf record 'syscalls:sys_enter_*' $cpus $cmd
+		# could also pipe direct to perf script
+	fi
+
+	printf "%-6s %-16s %-17s %8s\n" "PID" "COMM" "SYSCALL" "COUNT"
+	perf script --fields pid,comm,event | awk '$1 != "#" {
+		sub("syscalls:sys_enter_", ""); sub(":", "")
+		a[$1 ";" $2 ";" $3]++
+	}
+	END {
+		for (k in a) {
+			split(k, b, ";");
+			printf "%-6s %-16s %-17s %8d\n", b[2], b[1], b[3], a[k]
+		}
+	}' | sort -n -k4 | $tcmd
+	exit
+fi
+
+### execute process name mode
+tp="-e raw_syscalls:sys_enter"
+if (( write_workaround )); then
+	sh -c "perf record $tp --filter 'common_pid != '\$\$ $cpus $cmd"
+else
+	perf record $tp $cpus $cmd
+fi
+
+if (( opt_verbose )); then
+	printf "%-6s %-16s %8s\n" "PID" "COMM" "COUNT"
+	perf script --fields pid,comm | awk '$1 != "#" { a[$1 ";" $2]++ }
+	END {
+		for (k in a) {
+			split(k, b, ";");
+			printf "%-6s %-16s %8d\n", b[2], b[1],  a[k]
+		}
+	}' | sort -n -k3 | $tcmd
+else
+	printf "%-16s %8s\n" "COMM" "COUNT"
+	perf script --fields comm | awk '$1 != "#" { a[$1]++ }
+	END {
+		for (k in a) {
+			printf "%-16s %8d\n", k,  a[k]
+		}
+	}' | sort -n -k2 | $tcmd
+fi

data/perf-tools/system/tpoint

@@ -0,0 +1,232 @@
+#!/bin/bash
+#
+# tpoint - trace a given tracepoint. Static tracing.
+#          Written using Linux ftrace.
+#
+# This will enable a given tracepoint, print events, then disable the tracepoint
+# when the program ends. This is like a simple version of the "perf" command for
+# printing live tracepoint events only. Wildcards are currently not supported.
+# If this is insufficient for any reason, use the perf command instead.
+#
+# USAGE: ./tpoint [-hHsv] [-d secs] [-p pid] [-L tid] tracepoint [filter]
+#        ./tpoint -l
+#
+# Run "tpoint -h" for full usage.
+#
+# I wrote this because I often needed a quick way to dump stack traces for a
+# given tracepoint.
+#
+# OVERHEADS: Relative to the frequency of traced events. You might want to
+# check their frequency beforehand, using perf_events. Eg:
+#
+#	perf stat -e block:block_rq_issue -a sleep 5
+#
+# To count occurrences of that tracepoint for 5 seconds.
+#
+# REQUIREMENTS: FTRACE and tracepoints, which you may already have on recent
+# kernel versions.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the tpoint(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 22-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock; wroteflock=0
+opt_duration=0; duration=; opt_pid=0; pid=; opt_tid=0; tid=
+opt_filter=0; filter=; opt_view=0; opt_headers=0; opt_stack=0; dmesg=2
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: tpoint [-hHsv] [-d secs] [-p PID] [-L TID] tracepoint [filter]
+	       tpoint -l
+	                 -d seconds      # trace duration, and use buffers
+	                 -p PID          # PID to match on event
+	                 -L TID          # thread id to match on event
+	                 -v              # view format file (don't trace)
+	                 -H              # include column headers
+	                 -l              # list all tracepoints
+	                 -s              # show kernel stack traces
+	                 -h              # this usage message
+	   eg,
+	       tpoint -l | grep open
+	                                 # find tracepoints containing "open"
+	       tpoint syscalls:sys_enter_open
+	                                 # trace open() syscall entry
+	       tpoint block:block_rq_issue
+	                                 # trace block I/O issue
+	       tpoint -s block:block_rq_issue
+	                                 # show kernel stacks
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+	warn "echo 0 > $tdir/enable"
+	if (( opt_filter )); then
+		warn "echo 0 > $tdir/filter"
+	fi
+	(( opt_stack )) && warn "echo 0 > options/stacktrace"
+	warn "echo > trace"
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts d:hHlp:L:sv opt
+do
+	case $opt in
+	d)	opt_duration=1; duration=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+	L)	opt_tid=1; tid=$OPTARG ;;
+	H)	opt_headers=1 ;;
+	l)	opt_list=1 ;;
+	s)	opt_stack=1 ;;
+	v)	opt_view=1 ;;
+	h|?)	usage ;;
+	esac
+done
+if (( !opt_list )); then
+	shift $(( $OPTIND - 1 ))
+	(( $# )) || usage
+	tpoint=$1
+	shift
+	if (( $# )); then
+		opt_filter=1
+		filter=$1
+	fi
+fi
+
+### option logic
+(( opt_pid + opt_filter + opt_tid > 1 )) && \
+	die "ERROR: use at most one of -p, -L, or filter."
+(( opt_duration && opt_view )) && die "ERROR: use either -d or -v."
+if (( opt_pid )); then
+	# convert to filter
+	opt_filter=1
+	# ftrace common_pid is thread id from user's perspective
+	for tid in /proc/$pid/task/*; do
+		filter="$filter || common_pid == ${tid##*/}"
+	done
+	filter=${filter:3}  # trim leading ' || ' (four characters)
+fi
+if (( opt_tid )); then
+	opt_filter=1
+	filter="common_pid == $tid"
+fi
+if (( !opt_view && !opt_list )); then
+	if (( opt_duration )); then
+		echo "Tracing $tpoint for $duration seconds (buffered)..."
+	else
+		echo "Tracing $tpoint. Ctrl-C to end."
+	fi
+fi
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### do list tracepoints
+if (( opt_list )); then
+	cd events
+	for tp in */*; do
+		# skip filter/enable files
+		[[ -f $tp ]] && continue
+		echo ${tp/\//:}
+	done
+	exit
+fi
+
+### check tracepoints
+tdir=events/${tpoint/:/\/}
+[[ -e $tdir ]] || die "ERROR: tracepoint $tpoint not found. Exiting"
+
+### view
+if (( opt_view )); then
+	cat $tdir/format
+	exit
+fi
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### setup and begin tracing
+echo nop > current_tracer
+if (( opt_filter )); then
+	if ! echo "$filter" > $tdir/filter; then
+		edie "ERROR: setting filter or -p. Exiting."
+	fi
+fi
+if (( opt_stack )); then
+	if ! echo 1 > options/stacktrace; then
+		edie "ERROR: enabling stack traces (-s). Exiting"
+	fi
+fi
+if ! echo 1 >> $tdir/enable; then
+	edie "ERROR: enabling tracepoint $tprobe. Exiting."
+fi
+
+### print trace buffer
+warn "echo > trace"
+if (( opt_duration )); then
+	sleep $duration
+	if (( opt_headers )); then
+		cat trace
+	else
+		grep -v '^#' trace
+	fi
+else
+	# trace_pipe lack headers, so fetch them from trace
+	(( opt_headers )) && cat trace
+	cat trace_pipe
+fi
+
+### end tracing
+end

data/perf-tools/tools/reset-ftrace

@@ -0,0 +1,123 @@
+#!/bin/bash
+#
+# reset-ftrace - reset state of ftrace, disabling all tracing.
+#                Written for Linux ftrace.
+#
+# This may only be of use to ftrace hackers who, in the process of developing
+# ftrace software, often get the subsystem into a partially active state, and
+# would like a quick way to reset state. Check the end of this script for the
+# actually files reset, and add more if you need.
+#
+# USAGE: ./reset-ftrace [-fhq]
+#
+# REQUIREMENTS: FTRACE CONFIG.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the reset-ftrace(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 20-Jul-2014	Brendan Gregg	Created this.
+
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock
+opt_force=0; opt_quiet=0
+
+function usage {
+	cat <<-END >&2
+	USAGE: reset-ftrace [-fhq]
+	                 -f              # force: delete ftrace lock file
+	                 -q              # quiet: reset, but say nothing
+	                 -h              # this usage message
+	  eg,
+	       reset-ftrace              # disable active ftrace session
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function vecho {
+	(( opt_quiet )) && return
+	echo "$@"
+}
+
+# write to file
+function writefile {
+	file=$1
+	string=$2	# optional
+	if [[ ! -w $file ]]; then
+		echo >&2 "WARNING: file $file not writable/exists. Skipping."
+		return
+	fi
+	vecho "$file, before:"
+	(( ! opt_quiet )) && cat -n $file
+	warn "echo $string > $file"
+	vecho "$file, after:"
+	(( ! opt_quiet )) && cat -n $file
+	vecho
+}
+
+### process options
+while getopts fhq opt
+do
+	case $opt in
+	f)	opt_force=1 ;;
+	q)	opt_quiet=1 ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+
+### ftrace lock
+if [[ -e $flock ]]; then
+	if (( opt_force )); then
+		warn rm $flock
+	else
+		echo -e >&2 "ERROR: ftrace lock ($flock) exists. It shows" \
+		    "ftrace may be in use by PID $(cat $flock).\nDouble check" \
+		    "to see if that PID is still active. If not, consider" \
+		    "using -f to force a reset. Exiting."
+		exit 1
+	fi
+fi
+
+### reset ftrace state
+vecho "Reseting ftrace state..."
+vecho
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?"
+writefile current_tracer nop
+writefile set_ftrace_filter
+writefile set_graph_function
+writefile set_ftrace_pid
+writefile events/enable 0
+writefile tracing_thresh 0
+writefile kprobe_events
+writefile tracing_on 1
+vecho "Done."