fluent-plugin-perf-tools 0.1.0

data/perf-tools/deprecated/execsnoop-proc

@@ -0,0 +1,150 @@
+#!/usr/bin/perl
+#
+# execsnoop - trace process exec() with arguments. /proc version.
+#             Written using Linux ftrace.
+#
+# This shows the execution of new processes, especially short-lived ones that
+# can be missed by sampling tools such as top(1).
+#
+# USAGE: ./execsnoop [-h] [-n name]
+#
+# REQUIREMENTS: FTRACE CONFIG, sched:sched_process_exec tracepoint (you may
+# already have these on recent kernels), and Perl.
+#
+# This traces exec() from the fork()->exec() sequence, which means it won't
+# catch new processes that only fork(), and, it will catch processes that
+# re-exec. This instruments sched:sched_process_exec without buffering, and then
+# in user-space (this program) reads PPID and process arguments asynchronously
+# from /proc.
+#
+# If the process traced is very short-lived, this program may miss reading
+# arguments and PPID details. In that case, "<?>" and "?" will be printed
+# respectively. This program is best-effort, and should be improved in the
+# future when other kernel capabilities are made available. If you need a
+# more reliable tool now, then consider other tracing alternatives (eg,
+# SystemTap). This tool is really a proof of concept to see what ftrace can
+# currently do.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the execsnoop(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 07-Jul-2014	Brendan Gregg	Created this.
+
+use strict;
+use warnings;
+use POSIX qw(strftime);
+use Getopt::Long;
+my $tracing = "/sys/kernel/debug/tracing";
+my $flock = "/var/tmp/.ftrace-lock";
+my $tpdir = "sched/sched_process_exec";
+my $tptext = $tpdir; $tptext =~ s/\//:/;
+local $SIG{INT} = \&cleanup;
+local $SIG{QUIT} = \&cleanup;
+local $SIG{TERM} = \&cleanup;
+local $SIG{PIPE} = \&cleanup;
+local $SIG{HUP} = \&cleanup;
+$| = 1;
+
+### options
+my ($name, $help);
+GetOptions("name=s" => \$name,
+           "help"   => \$help)
+or usage();
+usage() if $help;
+sub usage {
+	print STDERR "USAGE: execsnoop [-h] [-n name]\n";
+	print STDERR "   eg,\n";
+	print STDERR "       execsnoop -n ls	# show \"ls\" cmds only.\n";
+	exit;
+}
+
+sub ldie {
+	unlink $flock;
+	die @_;
+}
+
+sub writeto {
+	my ($string, $file) = @_;
+	open FILE, ">$file" or return 0;
+	print FILE $string or return 0;
+	close FILE or return 0;
+}
+
+### check permissions
+chdir "$tracing" or ldie "ERROR: accessing tracing. Root? Kernel has FTRACE?" .
+    "\ndebugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)";
+
+### ftrace lock
+if (-e $flock) {
+	open FLOCK, $flock; my $fpid = <FLOCK>; chomp $fpid; close FLOCK;
+	die "ERROR: ftrace may be in use by PID $fpid ($flock)";
+}
+writeto "$$", $flock or die "ERROR: unable to write $flock.";
+
+### setup and begin tracing
+writeto "nop", "current_tracer" or ldie "ERROR: disabling current_tracer.";
+writeto "1", "events/$tpdir/enable" or ldie "ERROR: enabling tracepoint " .
+    "\"$tptext\" (tracepoint missing in this kernel version?)";
+open TPIPE, "trace_pipe"  or warn "ERROR: opening trace_pipe.";
+printf "%-8s %6s %6s %s\n",  "TIME", "PID", "PPID", "ARGS";
+
+while (<TPIPE>) {
+	my ($taskpid, $rest) = split;
+	my ($task, $pid) = $taskpid =~ /(.*)-(\d+)/;
+
+	next if (defined $name and $name ne $task);
+
+	my $args = "$task <?>";
+	if (open CMDLINE, "/proc/$pid/cmdline") {
+		my $arglist = <CMDLINE>;
+		if (defined $arglist) {
+			$arglist =~ s/\000/ /g;
+			$args = $arglist;
+		}
+		close CMDLINE;
+	}
+
+	my $ppid = "?";
+	if (open STAT, "/proc/$pid/stat") {
+		my $fields = <STAT>;
+		if (defined $fields) {
+			$ppid = (split ' ', $fields)[3];
+		}
+		close STAT;
+	}
+
+	my $now = strftime "%H:%M:%S", localtime;
+	printf "%-8s %6s %6s %s\n", $now, $pid, $ppid, $args;
+}
+
+### end tracing
+cleanup();
+
+sub cleanup {
+	print "\nEnding tracing...\n";
+	close TPIPE;
+	writeto "0", "events/$tpdir/enable" or
+	    ldie "ERROR: disabling \"$tptext\"";
+	writeto "", "trace";
+	unlink $flock;
+	exit;
+}

data/perf-tools/deprecated/execsnoop-proc.8

@@ -0,0 +1,80 @@
+.TH execsnoop\-proc 8  "2014-07-07" "USER COMMANDS"
+.SH NAME
+execsnoop\-proc \- trace process exec() with arguments. Uses Linux ftrace. /proc version.
+.SH SYNOPSIS
+.B execsnoop\-proc
+[\-h] [\-n name]
+.SH DESCRIPTION
+execsnoop\-proc traces process execution, showing PID, PPID, and argument details
+if possible.
+
+This traces exec() from the fork()->exec() sequence, which means it won't
+catch new processes that only fork(), and, it will catch processes that
+re-exec. This instruments sched:sched_process_exec without buffering, and then
+in user-space (this program) reads PPID and process arguments asynchronously
+from /proc.
+
+If the process traced is very short-lived, this program may miss reading
+arguments and PPID details. In that case, "<?>" and "?" will be printed
+respectively.
+
+This program is best-effort (a hack), and should be improved in the future when
+other kernel capabilities are made available. It may be useful in the meantime.
+If you need a more reliable tool now, consider other tracing alternates (eg,
+SystemTap). This tool is really a proof of concept to see what ftrace can
+currently do.
+
+See execsnoop(8) for another version that reads arguments from registers
+instead of /proc.
+
+Since this uses ftrace, only the root user can use this tool.
+.SH REQUIREMENTS
+FTRACE CONFIG and the sched:sched_process_exec tracepoint, which you may already
+have enabled and available on recent kernels, and Perl.
+.SH OPTIONS
+\-n name
+Only show processes that match this name. This is filtered in user space.
+.TP
+\-h
+Print usage message.
+.SH EXAMPLES
+.TP
+Trace all new processes and arguments (if possible):
+.B execsnoop\-proc
+.TP
+Trace all new processes with process name "sed":
+.B execsnoop\-proc -n sed
+.SH FIELDS
+.TP
+TIME
+Time of process exec(): HH:MM:SS.
+.TP
+PID
+Process ID.
+.TP
+PPID
+Parent process ID, if this was able to be read (may be missed for short-lived
+processes). If it is unable to be read, "?" is printed.
+.TP
+ARGS
+Command line arguments, if these were able to be read in time (may be missed
+for short-lived processes). If they are unable to be read, "<?>" is printed.
+.SH OVERHEAD
+This reads and processes exec() events in user space as they occur. Since the
+rate of exec() is expected to be low (< 500/s), the overhead is expected to
+be small or negligible.
+.SH SOURCE
+This is from the perf-tools collection.
+.IP
+https://github.com/brendangregg/perf-tools
+.PP
+Also look under the examples directory for a text file containing example
+usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+execsnoop(8), top(1)

data/perf-tools/deprecated/execsnoop-proc_example.txt

@@ -0,0 +1,46 @@
+Demonstrations of execsnoop-proc, the Linux ftrace version.
+
+Here's execsnoop showing what's really executed by "man ls":
+
+# ./execsnoop
+TIME        PID   PPID ARGS
+17:52:37  22406  25781 man ls 
+17:52:37  22413  22406 preconv -e UTF-8 
+17:52:37  22416  22406 pager -s 
+17:52:37  22415  22406 /bin/sh /usr/bin/nroff -mandoc -rLL=162n -rLT=162n -Tutf8 
+17:52:37  22414  22406 tbl 
+17:52:37  22419  22418 locale charmap 
+17:52:37  22420  22415 groff -mtty-char -Tutf8 -mandoc -rLL=162n -rLT=162n 
+17:52:37  22421  22420 troff -mtty-char -mandoc -rLL=162n -rLT=162n -Tutf8 
+17:52:37  22422  22420 grotty 
+
+
+These are short-lived processes, where the argument and PPID details are often
+missed by execsnoop:
+
+# ./execsnoop 
+TIME        PID   PPID ARGS
+18:00:33  26750   1961 multilog <?>
+18:00:33  26749   1972 multilog <?>
+18:00:33  26749   1972 multilog <?>
+18:00:33  26751      ? mkdir <?>
+18:00:33  26749   1972 multilog <?>
+18:00:33  26752      ? chown <?>
+18:00:33  26750   1961 multilog <?>
+18:00:33  26750   1961 multilog <?>
+18:00:34  26753   1961 multilog <?>
+18:00:34  26754   1972 multilog <?>
+[...]
+
+This will be fixed in a later version, but likely requires some kernel or
+tracer changes first (fetching cmdline as the probe fires).
+
+
+The previous examples were on Linux 3.14 and 3.16 kernels. Here's a 3.2 system
+I'm running:
+
+# ./execsnoop 
+ERROR: enabling tracepoint "sched:sched_process_exec" (tracepoint missing in this kernel version?) at ./execsnoop line 78.
+
+This kernel version is missing the sched_process_exec probe, which is pretty
+annoying.

data/perf-tools/disk/bitesize

@@ -0,0 +1,175 @@
+#!/bin/bash
+#
+# bitesize - show disk I/O size as a histogram.
+#            Written using Linux perf_events (aka "perf").
+#
+# This can be used to characterize the distribution of block device I/O
+# sizes. To study I/O in more detail, see iosnoop(8).
+#
+# USAGE: bitesize [-h] [-b buckets] [seconds]
+#    eg,
+#        ./bitesize 10
+#
+# Run "bitesize -h" for full usage.
+#
+# REQUIREMENTS: perf_events and block:block_rq_issue tracepoint, which you may
+# already have on recent kernels.
+#
+# This uses multiple counting tracepoints with different filters, one for each
+# histogram bucket. While this is summarized in-kernel, the use of multiple
+# tracepoints does add addiitonal overhead, which is more evident if you add
+# more buckets. In the future this functionality will be available in an
+# efficient way in the kernel, and this tool can be rewritten.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+# 
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 22-Jul-2014	Brendan Gregg	Created this.
+
+duration=0
+buckets=(1 8 64 128)
+secsz=512
+trap ':' INT QUIT TERM PIPE HUP
+
+function usage {
+	cat <<-END >&2
+	USAGE: bitesize [-h] [-b buckets] [seconds]
+	                 -b buckets      # specify histogram buckets (Kbytes)
+	                 -h              # this usage message
+	   eg,
+	       bitesize                  # trace I/O size until Ctrl-C
+	       bitesize 10               # trace I/O size for 10 seconds
+	       bitesize -b "8 16 32"     # specify custom bucket points
+END
+	exit
+}
+
+function die {
+	echo >&2 "$@"
+	exit 1
+}
+
+### process options
+while getopts b:h opt
+do
+	case $opt in
+	b)	buckets=($OPTARG) ;;
+	h|?)	usage ;;
+	esac
+done
+shift $(( $OPTIND - 1 ))
+tpoint=block:block_rq_issue
+var=nr_sector
+duration=$1
+
+### convert buckets (Kbytes) to disk sectors
+i=0
+sectors=(${buckets[*]})
+((max_i = ${#buckets[*]} - 1))
+while (( i <= max_i )); do
+	(( sectors[$i] = ${sectors[$i]} * 1024 / $secsz ))
+	# avoid negative array index errors for old version bash
+	if (( i > 0 ));then
+		if (( ${sectors[$i]} <= ${sectors[$i - 1]} )); then
+			die "ERROR: bucket list must increase in size."
+		fi
+	fi
+	(( i++ ))
+done
+
+### build list of tracepoints and filters for each histogram bucket
+max_b=${buckets[$max_i]}
+max_s=${sectors[$max_i]}
+tpoints="-e $tpoint --filter \"$var < ${sectors[0]}\""
+awkarray=
+i=0
+while (( i < max_i )); do
+	tpoints="$tpoints -e $tpoint --filter \"$var >= ${sectors[$i]} && "
+	tpoints="$tpoints $var < ${sectors[$i + 1]}\""
+	awkarray="$awkarray buckets[$i]=${buckets[$i]};"
+	(( i++ ))
+done
+awkarray="$awkarray buckets[$max_i]=${buckets[$max_i]};"
+tpoints="$tpoints -e $tpoint --filter \"$var >= ${sectors[$max_i]}\""
+
+### prepare to run
+if (( duration )); then
+	etext="for $duration seconds"
+	cmd="sleep $duration"
+else
+	etext="until Ctrl-C"
+	cmd="sleep 999999"
+fi
+echo "Tracing block I/O size (bytes), $etext..."
+
+### run perf
+out="-o /dev/stdout"	# a workaround needed in linux 3.2; not by 3.4.15
+stat=$(eval perf stat $tpoints -a $out $cmd 2>&1)
+if (( $? != 0 )); then
+	echo >&2 "ERROR running perf:"
+	echo >&2 "$stat"
+	exit
+fi
+
+### find max value for ASCII histogram
+most=$(echo "$stat" | awk -v tpoint=$tpoint '
+	$2 == tpoint { gsub(/,/, ""); if ($1 > m) { m = $1 } }
+	END { print m }'
+)
+
+### process output
+echo
+echo "$stat" | awk -v tpoint=$tpoint -v max_i=$max_i -v most=$most '
+	function star(sval, smax, swidth) {
+		stars = ""
+		# using int could avoid error on gawk
+		if (int(smax) == 0) return ""
+		for (si = 0; si < (swidth * sval / smax); si++) {
+			stars = stars "#"
+		}
+		return stars
+	}
+	BEGIN {
+		'"$awkarray"'
+		printf("            %-15s: %-8s %s\n", "Kbytes", "I/O",
+		    "Distribution")
+	}
+	/Performance counter stats/ { i = -1 }
+	# reverse order of rule set is important
+	{ ok = 0 }
+	$2 == tpoint { num = $1; gsub(/,/, "", num); ok = 1 }
+	ok && i >= max_i {
+		printf("   %10.1f -> %-10s: %-8s |%-38s|\n",
+		    buckets[i], "", num, star(num, most, 38))
+		next
+	}
+	ok && i >= 0 && i < max_i {
+		printf("   %10.1f -> %-10.1f: %-8s |%-38s|\n",
+		    buckets[i], buckets[i+1] - 0.1, num,
+		    star(num, most, 38))
+		i++
+		next
+	}
+	ok && i == -1 {
+		printf("   %10s -> %-10.1f: %-8s |%-38s|\n", "",
+		    buckets[0] - 0.1, num, star(num, most, 38))
+		i++
+	}
+'

data/perf-tools/examples/bitesize_example.txt

@@ -0,0 +1,63 @@
+Demonstrations of bitesize, the Linux perf_events version.
+
+
+bitesize traces block I/O issued, and reports a histogram of I/O size. By
+default five buckets are used to gather statistics on common I/O sizes:
+
+# ./bitesize 
+Tracing block I/O size (bytes), until Ctrl-C...
+^C
+            Kbytes         : I/O      Distribution
+              -> 0.9       : 0        |                                      |
+          1.0 -> 7.9       : 38       |#                                     |
+          8.0 -> 63.9      : 10108    |######################################|
+         64.0 -> 127.9     : 13       |#                                     |
+        128.0 ->           : 1        |#                                     |
+
+In this case, most of the I/O was between 8 and 63.9 Kbytes. The "63.9"
+really means "less than 64".
+
+
+Specifying custom buckets to examine the I/O size in more detail:
+
+# ./bitesize -b "8 16 24 32"
+Tracing block I/O size (bytes), until Ctrl-C...
+^C
+            Kbytes         : I/O      Distribution
+              -> 7.9       : 89       |#                                     |
+          8.0 -> 15.9      : 14665    |######################################|
+         16.0 -> 23.9      : 657      |##                                    |
+         24.0 -> 31.9      : 661      |##                                    |
+         32.0 ->           : 376      |#                                     |
+
+The I/O is mostly between 8 and 15.9 Kbytes
+
+It's probably 8 Kbytes. Checking:
+
+# ./bitesize -b "8 9"
+Tracing block I/O size (bytes), until Ctrl-C...
+^C
+            Kbytes         : I/O      Distribution
+              -> 7.9       : 62       |#                                     |
+          8.0 -> 8.9       : 11719    |######################################|
+          9.0 ->           : 1358     |#####                                 |
+
+It is.
+
+The overhead of this tool is relative to the number of buckets used, hence only
+using what is necessary.
+
+To study this I/O in more detail, I can use iosnoop(8) and capture it to a file
+for post-processing.
+
+
+Use -h to print the USAGE message:
+
+# ./bitesize -h
+USAGE: bitesize [-h] [-b buckets] [seconds]
+                 -b buckets      # specify histogram buckets (Kbytes)
+                 -h              # this usage message
+   eg,
+       bitesize                  # trace I/O size until Ctrl-C
+       bitesize 10               # trace I/O size for 10 seconds
+       bitesize -b "8 16 32"     # specify custom bucket points

data/perf-tools/examples/cachestat_example.txt

@@ -0,0 +1,58 @@
+Demonstrations of cachestat, the Linux ftrace version.
+
+
+Here is some sample output showing file system cache statistics, followed by
+the workload that caused it:
+
+# ./cachestat -t
+Counting cache functions... Output every 1 seconds.
+TIME         HITS   MISSES  DIRTIES    RATIO   BUFFERS_MB   CACHE_MB
+08:28:57      415        0        0   100.0%            1        191
+08:28:58      411        0        0   100.0%            1        191
+08:28:59      362       97        0    78.9%            0          8
+08:29:00      411        0        0   100.0%            0          9
+08:29:01      775    20489        0     3.6%            0         89
+08:29:02      411        0        0   100.0%            0         89
+08:29:03     6069        0        0   100.0%            0         89
+08:29:04    15249        0        0   100.0%            0         89
+08:29:05      411        0        0   100.0%            0         89
+08:29:06      411        0        0   100.0%            0         89
+08:29:07      411        0        3   100.0%            0         89
+[...]
+
+I used the -t option to include the TIME column, to make describing the output
+easier.
+
+The workload was:
+
+# echo 1 > /proc/sys/vm/drop_caches; sleep 2; cksum 80m; sleep 2; cksum 80m
+
+At 8:28:58, the page cache was dropped by the first command, which can be seen
+by the drop in size for "CACHE_MB" (page cache size) from 191 Mbytes to 8.
+After a 2 second sleep, a cksum command was issued at 8:29:01, for an 80 Mbyte
+file (called "80m"), which caused a total of ~20,400 misses ("MISSES" column),
+and the page cache size to grow by 80 Mbytes. The hit ratio during this dropped
+to 3.6%. Finally, after another 2 second sleep, at 8:29:03 the cksum command
+was run a second time, this time hitting entirely from cache.
+
+Instrumenting all file system cache accesses does cost some overhead, and this
+tool might slow your target system by 2% or so. Test before use if this is a
+concern.
+
+This tool also uses dynamic tracing, and is tied to Linux kernel implementation
+details. If it doesn't work for you, it probably needs fixing.
+
+
+Use -h to print the USAGE message:
+
+# ./cachestat -h
+USAGE: cachestat [-Dht] [interval]
+                 -D              # print debug counters
+                 -h              # this usage message
+                 -t              # include timestamp
+                 interval        # output interval in secs (default 1)
+  eg,
+       cachestat                 # show stats every second
+       cachestat 5               # show stats every 5 seconds
+
+See the man page and example file for more info.