fluent-plugin-perf-tools 0.1.0

data/perf-tools/net/tcpretrans

@@ -0,0 +1,311 @@
+#!/usr/bin/perl
+#
+# tcpretrans - show TCP retransmts, with address and other details.
+#              Written using Linux ftrace.
+#
+# This traces TCP retransmits, showing address, port, and TCP state information,
+# and sometimes the PID (although usually not, since retransmits are usually
+# sent by the kernel on timeouts). To keep overhead low, only
+# tcp_retransmit_skb() calls are traced (this does not trace every packet).
+#
+# USAGE: ./tcpretrans [-hls]
+#
+# REQUIREMENTS: FTRACE and KPROBE CONFIG, tcp_retransmit_skb() kernel function,
+# and tcp_send_loss_probe() when -l is used. You may have these already have
+# these on recent kernels. And Perl.
+#
+# This was written as a proof of concept for ftrace, for older Linux systems,
+# and without kernel debuginfo. It uses dynamic tracing of tcp_retransmit_skb(),
+# and reads /proc/net/tcp for socket details. Its use of dynamic tracing and
+# CPU registers is an unstable platform-specific workaround, and may require
+# modifications to work on different kernels and platforms. This would be better
+# written using a tracer such as SystemTap, and will likely be rewritten in the
+# future when certain tracing features are added to the Linux kernel.
+#
+# When -l is used, this also uses dynamic tracing of tcp_send_loss_probe() and
+# a register.
+#
+# Currently only IPv4 is supported, on x86_64. If you try this on a different
+# architecture, you'll likely need to adjust the register locations (search
+# for %di).
+#
+# OVERHEAD: The CPU overhead is relative to the rate of TCP retransmits, and is
+# designed to be low as this does not examine every packet. Once per second the
+# /proc/net/tcp file is read, and a buffer of retransmit trace events is
+# retrieved from the kernel and processed.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the tcpretrans(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 28-Jul-2014	Brendan Gregg	Created this.
+
+use strict;
+use warnings;
+use POSIX qw(strftime);
+use Getopt::Long;
+my $tracing = "/sys/kernel/debug/tracing";
+my $flock = "/var/tmp/.ftrace-lock";
+my $interval = 1;
+local $SIG{INT} = \&cleanup;
+local $SIG{QUIT} = \&cleanup;
+local $SIG{TERM} = \&cleanup;
+local $SIG{PIPE} = \&cleanup;
+local $SIG{HUP} = \&cleanup;
+$| = 1;
+
+### options
+my ($help, $stacks, $tlp);
+GetOptions("help|h"   => \$help,
+	   "stacks|s" => \$stacks,
+	   "tlp|l" => \$tlp)
+or usage();
+usage() if $help;
+
+sub usage {
+	print STDERR "USAGE: tcpretrans [-hls]\n";
+	print STDERR "                  -h      # help message\n";
+	print STDERR "                  -l      # trace TCP tail loss probes\n";
+	print STDERR "                  -s      # print stack traces\n";
+	print STDERR "   eg,\n";
+	print STDERR "       tcpretrans         # trace TCP retransmits\n";
+	exit;
+}
+
+# delete lock and die
+sub ldie {
+	unlink $flock;
+	die @_;
+}
+
+# end tracing (silently) and die
+sub edie {
+	print STDERR "@_\n";
+	close STDOUT;
+	close STDERR;
+	cleanup();
+}
+
+sub writeto {
+	my ($string, $file) = @_;
+	open FILE, ">$file" or return 0;
+	print FILE $string or return 0;
+	close FILE or return 0;
+}
+
+sub appendto {
+	my ($string, $file) = @_;
+	open FILE, ">>$file" or return 0;
+	print FILE $string or return 0;
+	close FILE or return 0;
+}
+
+# kprobe functions
+sub create_kprobe {
+	my ($kname, $kval) = @_;
+	appendto "p:$kname $kval", "kprobe_events" or return 0;
+}
+
+sub enable_kprobe {
+	my ($kname) = @_;
+	writeto "1", "events/kprobes/$kname/enable" or return 0;
+}
+
+sub remove_kprobe {
+	my ($kname) = @_;
+	writeto "0", "events/kprobes/$kname/enable" or return 0;
+	appendto "-:$kname", "kprobe_events" or return 0;
+}
+
+# tcp socket cache
+my %tcp;
+sub cache_tcp {
+	undef %tcp;
+	open(TCP, "/proc/net/tcp") or ldie "ERROR: reading /proc/net/tcp.";
+	while (<TCP>) {
+		next if /^ *sl/;
+		my ($sl, $local_address, $rem_address, $st, $tx_rx, $tr_tm,
+		    $retrnsmt, $uid, $timeout, $inode, $jf, $sk) = split;
+		$sk =~ s/^0x//;
+		$tcp{$sk}{laddr} = $local_address;
+		$tcp{$sk}{raddr} = $rem_address;
+		$tcp{$sk}{state} = $st;
+	}
+	close TCP;
+}
+
+my @tcpstate;
+sub map_tcp_states {
+	push @tcpstate, "NULL";
+	for (<DATA>) {
+		chomp;
+		s/.*TCP_//;
+		s/[, ].*$//;
+		push @tcpstate, $_;
+	}
+}
+
+# /proc/net/tcp hex addr to dotted quad decimal
+sub inet_h2a {
+	my ($haddr) = @_;
+
+	my @addr = ();
+	for my $num ($haddr =~ /(..)(..)(..)(..)/) {
+		unshift @addr, hex($num);
+	}
+	return join(".", @addr);
+}
+
+### check permissions
+chdir "$tracing" or die "ERROR: accessing tracing. Root? Kernel has FTRACE?" .
+    "\ndebugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)";
+
+### ftrace lock
+if (-e $flock) {
+	open FLOCK, $flock; my $fpid = <FLOCK>; chomp $fpid; close FLOCK;
+	die "ERROR: ftrace may be in use by PID $fpid ($flock)";
+}
+writeto "$$", $flock or die "ERROR: unable to write $flock.";
+
+#
+# Setup and begin tracing.
+# Use of ldie() and edie() ensures that if an error is encountered, the
+# kernel is not left in a partially configured state.
+#
+writeto "nop", "current_tracer" or ldie "ERROR: disabling current_tracer.";
+my $kname_rtr = "tcpretrans_tcp_retransmit_skb";
+my $kname_tlp = "tcpretrans_tcp_send_loss_probe";
+create_kprobe $kname_rtr, "tcp_retransmit_skb sk=%di" or
+    ldie "ERROR: creating kprobe for tcp_retransmit_skb().";;
+if ($tlp) {
+	create_kprobe $kname_tlp, "tcp_send_loss_probe sk=%di" or
+	    edie "ERROR: creating kprobe for tcp_send_loss_probe(). " .
+	         "Older kernel version?";
+}
+if ($stacks) {
+	writeto "1", "options/stacktrace" or print STDERR "WARNING: " .
+	    "unable to enable stacktraces.\n";
+}
+enable_kprobe $kname_rtr or edie "ERROR: enabling $kname_rtr probe.";
+if ($tlp) {
+	enable_kprobe $kname_tlp or edie "ERROR: enabling $kname_tlp probe.";
+}
+map_tcp_states();
+printf "%-8s %-6s %-20s -- %-20s %-12s\n", "TIME", "PID", "LADDR:LPORT",
+    "RADDR:RPORT", "STATE";
+
+#
+# Read and print event data. This loop waits one second then reads the buffered
+# trace data, then caches /proc/net/tcp, then iterates over the buffered trace
+# data using the cached state. While this minimizes CPU overheads, it only
+# works because sockets that are retransmitting are usually long lived, and
+# remain in /proc/net/tcp for at least our sleep interval.
+#
+while (1) {
+	sleep $interval;
+
+	# buffer trace data
+	open TPIPE, "trace" or edie "ERROR: opening trace_pipe.";
+	my @trace = ();
+	while (<TPIPE>) {
+		next if /^#/;
+		push @trace, $_;
+	}
+	close TPIPE;
+	writeto "0", "trace" or edie "ERROR: clearing trace";
+
+	# cache /proc/net/tcp state
+	if (scalar @trace) {
+		cache_tcp();
+	}
+
+	# process and print events
+	for (@trace) {
+		if ($stacks && /^ *=>/) {
+			print $_;
+			next;
+		}
+
+		my ($taskpid, $rest) = split ' ', $_, 2;
+		my ($task, $pid) = $taskpid =~ /(.*)-(\d+)/;
+
+		my ($skp) = $rest =~ /sk=([0-9a-fx]*)/;
+		next unless defined $skp and $skp ne "";
+		$skp =~ s/^0x//;
+
+		my ($laddr, $lport, $raddr, $rport, $state);
+		if (defined $tcp{$skp}) {
+			# convert /proc/net/tcp hex to dotted quads
+			my ($hladdr, $hlport) = split /:/, $tcp{$skp}{laddr};
+			my ($hraddr, $hrport) = split /:/, $tcp{$skp}{raddr};
+			$laddr = inet_h2a($hladdr);
+			$raddr = inet_h2a($hraddr);
+			$lport = hex($hlport);
+			$rport = hex($hrport);
+			$state = $tcpstate[hex($tcp{$skp}{state})];
+		} else {
+			# socket closed too quickly
+			($laddr, $raddr) = ("-", "-");
+			($lport, $rport) = ("-", "-");
+			$state = "-";
+		}
+
+		my $now = strftime "%H:%M:%S", localtime;
+		printf "%-8s %-6s %-20s %s> %-20s %-12s\n", $now, $pid,
+		    "$laddr:$lport", $rest =~ /$kname_tlp/ ? "L" : "R",
+		    "$raddr:$rport", $state,
+	}
+}
+
+### end tracing
+cleanup();
+
+sub cleanup {
+	print "\nEnding tracing...\n";
+	close TPIPE;
+	if ($stacks) {
+		writeto "0", "options/stacktrace" or print STDERR "WARNING: " .
+		    "unable to disable stacktraces.\n";
+	}
+	remove_kprobe $kname_rtr
+	    or print STDERR "ERROR: removing kprobe $kname_rtr\n";
+	if ($tlp) {
+		remove_kprobe $kname_tlp
+		    or print STDERR "ERROR: removing kprobe $kname_tlp\n";
+	}
+	writeto "", "trace";
+	unlink $flock;
+	exit;
+}
+
+# from /usr/include/netinet/tcp.h:
+__DATA__
+  TCP_ESTABLISHED = 1,
+  TCP_SYN_SENT,
+  TCP_SYN_RECV,
+  TCP_FIN_WAIT1,
+  TCP_FIN_WAIT2,
+  TCP_TIME_WAIT,
+  TCP_CLOSE,
+  TCP_CLOSE_WAIT,
+  TCP_LAST_ACK,
+  TCP_LISTEN,
+  TCP_CLOSING   /* now a valid state */

data/perf-tools/opensnoop

@@ -0,0 +1,280 @@
+#!/bin/bash
+#
+# opensnoop - trace open() syscalls with file details.
+#             Written using Linux ftrace.
+#
+# This traces open() syscalls, showing the file name and returned file
+# descriptor number (or -1, for error).
+#
+# This implementation is designed to work on older kernel versions, and without
+# kernel debuginfo. It works by dynamic tracing of the return value of getname()
+# as a string, and associating it with the following open() syscall return.
+# This approach is kernel version specific, and may not work on your version.
+# It is a workaround, and proof of concept for ftrace, until more kernel tracing
+# functionality is available.
+#
+# USAGE: ./opensnoop [-htx] [-d secs] [-p pid] [-L tid] [-n name] [filename]
+#
+# Run "opensnoop -h" for full usage.
+#
+# REQUIREMENTS: FTRACE and KPROBE CONFIG, syscalls:sys_exit_open tracepoint,
+# getname() kernel function (you may already have these on recent kernels),
+# and awk.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the opensnoop(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 20-Jul-2014	Brendan Gregg	Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock; wroteflock=0
+opt_duration=0; duration=; opt_name=0; name=; opt_pid=0; pid=; opt_tid=0; tid=
+ftext=; opt_time=0; opt_fail=0; opt_file=0; file=
+trap ':' INT QUIT TERM PIPE HUP	# sends execution to end tracing section
+
+function usage {
+	cat <<-END >&2
+	USAGE: opensnoop [-htx] [-d secs] [-p PID] [-L TID] [-n name] [filename]
+	                 -d seconds      # trace duration, and use buffers
+	                 -n name         # process name to match on open
+	                 -p PID          # PID to match on open
+	                 -L TID          # PID to match on open
+	                 -t              # include time (seconds)
+	                 -x              # only show failed opens
+	                 -h              # this usage message
+	                 filename        # match filename (partials, REs, ok)
+	  eg,
+	       opensnoop                 # watch open()s live (unbuffered)
+	       opensnoop -d 1            # trace 1 sec (buffered)
+	       opensnoop -p 181          # trace I/O issued by PID 181 only
+	       opensnoop conf            # trace filenames containing "conf"
+	       opensnoop 'log$'          # filenames ending in "log"
+
+	See the man page and example file for more info.
+END
+	exit
+}
+
+function warn {
+	if ! eval "$@"; then
+		echo >&2 "WARNING: command failed \"$@\""
+	fi
+}
+
+function end {
+	# disable tracing
+	echo 2>/dev/null
+	echo "Ending tracing..." 2>/dev/null
+	cd $tracing
+	warn "echo 0 > events/kprobes/getnameprobe/enable"
+	warn "echo 0 > events/syscalls/sys_exit_open/enable"
+	warn "echo 0 > events/syscalls/sys_exit_openat/enable"
+	if (( opt_pid || opt_tid )); then
+		warn "echo 0 > events/kprobes/getnameprobe/filter"
+		warn "echo 0 > events/syscalls/sys_exit_open/filter"
+		warn "echo 0 > events/syscalls/sys_exit_openat/filter"
+	fi
+	warn "echo -:getnameprobe >> kprobe_events"
+	warn "echo > trace"
+	(( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+function edie {
+	# die with a quiet end()
+	echo >&2 "$@"
+	exec >/dev/null 2>&1
+	end
+	exit 1
+}
+
+### process options
+while getopts d:hn:p:L:tx opt
+do
+	case $opt in
+	d)	opt_duration=1; duration=$OPTARG ;;
+	n)	opt_name=1; name=$OPTARG ;;
+	p)	opt_pid=1; pid=$OPTARG ;;
+	L)	opt_tid=1; tid=$OPTARG ;;
+	t)	opt_time=1 ;;
+	x)	opt_fail=1 ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+if (( $# )); then
+	opt_file=1
+	file=$1
+	shift
+fi
+(( $# )) && usage
+
+### option logic
+(( opt_pid + opt_name + opt_tid > 1 )) && \
+	die "ERROR: use at most one of -p, -n, -L."
+(( opt_pid )) && ftext=" issued by PID $pid"
+(( opt_tid )) && ftext=" issued by TID $tid"
+(( opt_name )) && ftext=" issued by process name \"$name\""
+(( opt_file )) && ftext="$ftext for filenames containing \"$file\""
+if (( opt_duration )); then
+	echo "Tracing open()s$ftext for $duration seconds (buffered)..."
+else
+	echo "Tracing open()s$ftext. Ctrl-C to end."
+fi
+
+### select awk
+(( opt_duration )) && use=mawk || use=gawk	# workaround for mawk fflush()
+[[ -x /usr/bin/$use ]] && awk=$use || awk=awk
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### setup and begin tracing
+echo nop > current_tracer
+ver=$(uname -r)
+if [[ "$ver" == 2.* || "$ver" == 3.[1-6].* ]]; then
+	# rval is char *
+	kprobe='r:getnameprobe getname +0($retval):string'
+else
+	# rval is struct filename *
+	kprobe='r:getnameprobe getname +0(+0($retval)):string'
+fi
+if ! echo $kprobe >> kprobe_events; then
+	edie "ERROR: adding a kprobe for getname(). Exiting."
+fi
+if (( opt_pid )); then
+	filter=
+	for tid in /proc/$pid/task/*; do
+		filter="$filter || common_pid == ${tid##*/}"
+	done
+	filter=${filter:3}  # trim leading ' || ' (four characters)
+	if ! echo $filter > events/kprobes/getnameprobe/filter || \
+	    ! echo $filter > events/syscalls/sys_exit_open/filter || \
+	    ! echo $filter > events/syscalls/sys_exit_openat/filter
+	then
+	    edie "ERROR: setting -p $pid. Exiting."
+	fi
+fi
+if (( opt_tid )); then
+	if ! echo "common_pid == $tid" > events/kprobes/getnameprobe/filter || \
+	    ! echo "common_pid == $tid" > events/syscalls/sys_exit_open/filter || \
+	    ! echo "common_pid == $tid" > events/syscalls/sys_exit_openat/filter
+	then
+	    edie "ERROR: setting -L $tid. Exiting."
+	fi
+fi
+if ! echo 1 > events/kprobes/getnameprobe/enable; then
+	edie "ERROR: enabling kprobe for getname(). Exiting."
+fi
+if ! echo 1 > events/syscalls/sys_exit_open/enable; then
+	edie "ERROR: enabling open() exit tracepoint. Exiting."
+fi
+if ! echo 1 > events/syscalls/sys_exit_openat/enable; then
+	edie "ERROR: enabling openat() exit tracepoint. Exiting."
+fi
+(( opt_time )) && printf "%-16s " "TIMEs"
+printf "%-16.16s %-6s %4s %s\n" "COMM" "PID" "FD" "FILE"
+
+#
+# Determine output format. It may be one of the following (newest first):
+#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
+#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
+# To differentiate between them, the number of header fields is counted,
+# and an offset set, to skip the extra column when needed.
+#
+offset=$($awk 'BEGIN { o = 0; }
+	$1 == "#" && $2 ~ /TASK/ && NF == 6 { o = 1; }
+	$2 ~ /TASK/ { print o; exit }' trace)
+
+### print trace buffer
+warn "echo > trace"
+( if (( opt_duration )); then
+	# wait then dump buffer
+	sleep $duration
+	cat trace
+else
+	# print buffer live
+	cat trace_pipe
+fi ) | $awk -v o=$offset -v opt_name=$opt_name -v name=$name \
+    -v opt_duration=$opt_duration -v opt_time=$opt_time -v opt_fail=$opt_fail \
+    -v opt_file=$opt_file -v file=$file '
+	# common fields
+	$1 != "#" {
+		# task name can contain dashes and space
+		split($0, line, "-")
+		sub(/^[ \t\r\n]+/, "", line[1])
+		comm = line[1]
+		if (opt_name && match(comm, name) == 0)
+			next
+		sub(/ .*$/, "", line[2])
+		pid = line[2]
+	}
+
+	# do_sys_open()
+	$1 != "#" && $(5+o) ~ /do_sys_open/ {
+		#
+		# eg: ... (do_sys_open+0xc3/0x220 <- getname) arg1="file1"
+		#
+		match($0, /arg1=\"(.+)\"/, m)
+		lastfile[pid] = m[1]
+	}
+
+	# sys_open() / sys_openat()
+	$1 != "#" && ($(4+o) == "sys_open" || $(4+o) == "sys_openat") {
+		filename = lastfile[pid]
+		if (!filename)
+			next
+		delete lastfile[pid]
+		if (opt_file && filename !~ file)
+			next
+		rval = $NF
+		# matched failed as beginning with 0xfffff
+		if (opt_fail && rval !~ /0xfffff/)
+			next
+		if (rval ~ /0xfffff/)
+			rval = -1
+
+		if (opt_time) {
+			time = $(3+o); sub(":", "", time)
+			printf "%-16s ", time
+		}
+		printf "%-16.16s %-6s %4s %s\n", comm, pid, rval, filename
+	}
+
+	$0 ~ /LOST.*EVENTS/ {
+		delete lastfile
+		print "WARNING: " $0 > "/dev/stderr" }
+'
+
+### end tracing
+end