fluent-plugin-perf-tools 0.1.0

data/perf-tools/killsnoop

@@ -0,0 +1,263 @@
+#!/bin/bash
+#
+# killsnoop - trace kill() syscalls with signal/process details.
+#             Written using Linux ftrace.
+#
+# This traces kill() syscalls, showing which process killed which pid and
+# returns the returncode (0 for success, -1 for error).
+#
+# This implementation is designed to work on older kernel versions, and without
+# kernel debuginfo. It works by dynamic tracing of the return value of kill()
+# and associating it with the previous kill() syscall return.
+# This approach is kernel version specific, and may not work on your version.
+# It is a workaround, and proof of concept for ftrace, until more kernel tracing
+# functionality is available.
+#
+# USAGE: ./killsnoop [-hst] [-d secs] [-p pid] [-n name]
+#
+# Run "killsnoop -h" for full usage.
+#
+# REQUIREMENTS: FTRACE and KPROBE CONFIG, syscalls:sys_enter_kill and
+# syscalls:sys_exit_kill kernel tracepoints (you may already have these
+# on recent kernels) and awk.
+#
+# From perf-tools: https://github.com/brendangregg/perf-tools
+#
+# See the killsnoop(8) man page (in perf-tools) for more info.
+#
+# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
+# COPYRIGHT: Copyright (c) 2014 Martin Probst.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  (http://www.gnu.org/copyleft/gpl.html)
+#
+# 20-Jul-2014   Brendan Gregg   Templated this.
+# 13-Sep-2014   Martin Probst   Created this.
+
+### default variables
+tracing=/sys/kernel/debug/tracing
+flock=/var/tmp/.ftrace-lock; wroteflock=0
+opt_duration=0; duration=; opt_name=0; name=; opt_pid=0; pid=; ftext=
+opt_time=0; opt_fail=0; opt_file=0; file=
+kevent_entry=events/syscalls/sys_enter_kill
+kevent_return=events/syscalls/sys_exit_kill
+trap ':' INT QUIT TERM PIPE HUP # sends execution to end tracing section
+
+function usage {
+    cat <<-END >&2
+USAGE: killsnoop [-hst] [-d secs] [-p PID] [-n name] [filename]
+                 -d seconds      # trace duration, and use buffers
+                 -n name         # process name to match 
+                 -p PID          # PID to match on kill issue
+                 -t              # include time (seconds)
+                 -s              # human readable signal names
+                 -h              # this usage message
+  eg,
+       killsnoop                 # watch kill()s live (unbuffered)
+       killsnoop -d 1            # trace 1 sec (buffered)
+       killsnoop -p 181          # trace kill()s issued to PID 181 only
+
+See the man page and example file for more info.
+END
+    exit
+}
+
+function warn {
+    if ! eval "$@"; then
+        echo >&2 "WARNING: command failed \"$@\""
+    fi
+}
+
+function end {
+    # disable tracing
+    echo 2>/dev/null
+    echo "Ending tracing..." 2>/dev/null
+    cd $tracing
+    warn "echo 0 > $kevent_entry/enable"
+    warn "echo 0 > $kevent_return/enable"
+    warn "echo > trace"
+    (( wroteflock )) && warn "rm $flock"
+}
+
+function die {
+    echo >&2 "$@"
+    exit 1
+}
+
+function edie {
+    # die with a quiet end()
+    echo >&2 "$@"
+    exec >/dev/null 2>&1
+    end
+    exit 1
+}
+
+### process options
+while getopts d:hn:p:st opt
+do
+    case $opt in
+    d)  opt_duration=1; duration=$OPTARG ;;
+    n)  opt_name=1; name=$OPTARG ;;
+    p)  opt_pid=1; pid=$OPTARG ;;
+    t)  opt_time=1 ;;
+    s)  opt_fancy=1 ;;
+    h|?)    usage ;;
+    esac
+done
+shift $(( $OPTIND - 1 ))
+(( $# )) && usage
+
+### option logic
+(( opt_pid && opt_name )) && die "ERROR: use either -p or -n."
+(( opt_pid )) && ftext=" issued to PID $pid"
+(( opt_name )) && ftext=" issued by process name \"$name\""
+if (( opt_duration )); then
+    echo "Tracing kill()s$ftext for $duration seconds (buffered)..."
+else
+    echo "Tracing kill()s$ftext. Ctrl-C to end."
+fi
+
+### select awk
+# workaround for mawk fflush()
+[[ -x /usr/bin/mawk ]] && awk="mawk" && mawk -W interactive && \
+[ $? -eq 0 ] && awk="mawk -W interactive"
+# workaround for gawk strtonum()
+[[ -x /usr/bin/gawk ]] && awk="gawk --non-decimal-data"
+
+### check permissions
+cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
+    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
+
+### ftrace lock
+[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
+echo $$ > $flock || die "ERROR: unable to write $flock."
+wroteflock=1
+
+### setup and begin tracing
+echo nop > current_tracer
+if ! echo 1 > $kevent_entry/enable; then
+    edie "ERROR: enabling kill() entry tracepoint Exiting."
+fi
+if ! echo 1 > $kevent_return/enable; then
+    edie "ERROR: enabling kill() return tracepoint. Exiting."
+fi
+(( opt_time )) && printf "%-16s " "TIMEs"
+printf "%-16.16s %-6s %-8s %-10s %4s\n" "COMM" "PID" "TPID" "SIGNAL" "RETURN"
+
+#
+# Determine output format. It may be one of the following (newest first):
+#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
+#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
+# To differentiate between them, the number of header fields is counted,
+# and an offset set, to skip the extra column when needed.
+#
+offset=$($awk 'BEGIN { o = 0; }
+    $1 == "#" && $2 ~ /TASK/ && NF == 6 { o = 1; }
+    $2 ~ /TASK/ { print o; exit }' trace)
+
+### print trace buffer
+warn "echo > trace"
+( if (( opt_duration )); then
+    # wait then dump buffer
+    sleep $duration
+    cat trace
+else
+    # print buffer live
+    cat trace_pipe
+fi ) | $awk -v o=$offset -v opt_name=$opt_name -v name=$name \
+    -v opt_duration=$opt_duration -v opt_time=$opt_time \
+    -v opt_pid=$pid -v opt_fancy=$opt_fancy '
+    # fancy signal names
+    BEGIN {
+        signals[1] = "SIGHUP"
+        signals[2] = "SIGINT"
+        signals[3] = "SIGQUIT"
+        signals[4] = "SIGILL"
+        signals[6] = "SIGABRT"
+        signals[8] = "SIGFPE"
+        signals[9] = "SIGKILL"
+        signals[11] = "SIGSEGV"
+        signals[13] = "SIGPIPE"
+        signals[14] = "SIGALRM"
+        signals[15] = "SIGTERM"
+        signals[10] = "SIGUSR1"
+        signals[12] = "SIGUSR2"
+        signals[17] = "SIGCHLD"
+        signals[18] = "SIGCONT"
+        signals[19] = "SIGSTOP"
+        signals[20] = "SIGTSTP"
+        signals[21] = "SIGTTIN"
+        signals[22] = "SIGTTOU"
+    }
+
+    # common fields
+    $1 != "#" {
+        # task name can contain dashes
+        comm = pid = $1
+        sub(/-[0-9][0-9]*/, "", comm)
+        if (opt_name && match(comm, name) == 0)
+            next
+        sub(/.*-/, "", pid)
+    }
+
+    # sys_kill() entry
+    $1 != "#" && $(4+o) ~ /sys_kill/ && $(5+o) !~ /->/ {
+        #
+        # eg: ... sys_kill(pid:...
+        #
+        kpid = $(5+o)
+        signal = $(7+o)
+        sub(/,$/, "", kpid)
+        sub(/\)$/, "", signal)
+        kpid = int("0x"kpid)
+        signal = int("0x"signal)
+        current[pid,"kpid"] = kpid
+        current[pid,"signal"] = signal
+    }
+
+    # sys_kill exit
+    $1 != "#" && $(5+o) ~ /->/ {
+        rv = int($NF)
+        killed_pid = current[pid,"kpid"]
+        signal = current[pid,"signal"]
+
+        delete current[pid,"kpid"]
+        delete current[pid,"signal"]
+
+        if(opt_pid && killed_pid != opt_pid) {
+            next
+        }
+
+        if (opt_time) {
+            time = $(3+o); sub(":", "", time)
+            printf "%-16s ", time
+        }
+
+        if (opt_fancy) {
+            if (signals[signal] != "") {
+                signal = signals[signal]
+            }
+        }
+
+        printf "%-16.16s %-6s %-8s %-10s %-4s\n", comm, pid, killed_pid, signal,
+            rv
+    }
+
+    $0 ~ /LOST.*EVENTS/ { print "WARNING: " $0 > "/dev/stderr" }
+'
+
+### end tracing
+end

data/perf-tools/man/man8/bitesize.8

@@ -0,0 +1,70 @@
+.TH bitesize 8  "2014-07-07" "USER COMMANDS"
+.SH NAME
+bitesize \- show disk I/O size as a histogram. Uses Linux perf_events.
+.SH SYNOPSIS
+.B bitesize
+[-h] [-b buckets] [seconds]
+.SH DESCRIPTION
+This can be used to characterize the distribution of block device (disk) I/O
+sizes. To study block device I/O in more detail, see iosnoop(8).
+
+This uses multiple counting tracepoints with different filters, one for each
+histogram bucket. While this is summarized in-kernel, the use of multiple
+tracepoints does add addiitonal overhead, which is more evident if you add
+more buckets. In the future this functionality will be available in an
+efficient way in the kernel, and this tool can be rewritten.
+.SH REQUIREMENTS
+Linux perf_events: add linux-tools-common, run "perf", then add any additional
+packages it requests. This also requires the block:block_rq_issue tracepoint,
+which should already be available in recent kernels.
+.SH OPTIONS
+.TP
+\-h
+Usage message.
+.TP
+\-b buckets
+Specify a list of bucket points for the histogram as a string (eg, "10 500
+1000"). The histogram will include buckets for less-than the minimum, and
+greater-than-or-equal-to the maximum.  If a single value is specified, two
+statistics only are gathered: for less-than and for greater-than-or-equal-to.
+The overhead is relative to the number of buckets, so only specifying a
+single value costs the lowest overhead.
+.TP
+seconds
+Number of seconds to trace. If not specified, this runs until Ctrl-C.
+.SH EXAMPLES
+.TP
+Trace read() syscalls until Ctrl-C, and show histogram of requested size:
+#
+.B bitesize syscalls:sys_enter_read count
+.SH FIELDS
+.TP
+Kbytes
+Kbyte range of the histogram bucket.
+.TP
+I/O
+Number of I/O that occurred in this range while tracing.
+.TP
+Distribution
+ASCII histogram representation of the I/O column.
+.SH OVERHEAD
+While the counts are performed in-kernel, there is one tracepoint used per
+histogram bucket, so the overheads are higher than usual (relative to the
+number of buckets) than function counting using perf stat. The lowest
+overhead is when \-b is used to specify one bucket only, bifurcating
+statistics.
+.SH SOURCE
+This is from the perf-tools collection.
+.IP
+https://github.com/brendangregg/perf-tools
+.PP
+Also look under the examples directory for a text file containing example
+usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+iosnoop(8), iolatency(8), iostat(1)

data/perf-tools/man/man8/cachestat.8

@@ -0,0 +1,111 @@
+.TH cachestat 8  "2014-12-28" "USER COMMANDS"
+.SH NAME
+cachestat \- Measure page cache hits/misses. Uses Linux ftrace.
+.SH SYNOPSIS
+.B cachestat
+[\-Dht] [interval]
+.SH DESCRIPTION
+This tool provides basic cache hit/miss statistics for the Linux page cache.
+
+Its current implementation uses Linux ftrace dynamic function profiling to
+create custom in-kernel counters, which is a workaround until such counters
+can be built-in to the kernel. Specifically, four kernel functions are counted:
+.IP
+mark_page_accessed() for measuring cache accesses
+.IP
+mark_buffer_dirty() for measuring cache writes
+.IP
+add_to_page_cache_lru() for measuring page additions
+.IP
+account_page_dirtied() for measuring page dirties
+.PP
+It is possible that these functions have been renamed (or are different
+logically) for your kernel version, and this script will not work as-is.
+This was written for a Linux 3.13 kernel, and tested on a few others versions.
+This script is a sandcastle: the kernel may wash some away, and you'll
+need to rebuild.
+
+This program's implementation can be improved in the future when other
+kernel capabilities are made available. If you need a more reliable tool now,
+then consider other tracing alternatives (eg, SystemTap). This tool is really
+a proof of concept to see what ftrace can currently do.
+
+WARNING: This uses dynamic tracing of kernel functions, and could cause
+kernel panics or freezes. Test, and know what you are doing, before use.
+It also traces cache activity, which can be frequent, and cost some overhead.
+The statistics should be treated as best-effort: there may be some error
+margin depending on unusual workload types.
+
+Since this uses ftrace, only the root user can use this tool.
+.SH REQUIREMENTS
+CONFIG_FUNCTION_PROFILER, which you may already have enabled and available on
+recent kernels, and awk.
+.SH OPTIONS
+.TP
+\-D
+Include extra fields for debug purposes (see script).
+.TP
+\-h
+Print usage message.
+.TP
+\-t
+Include timestamps in units of seconds.
+.TP
+interval
+Output interval in seconds. Default is 1.
+.SH EXAMPLES
+.TP
+Show per-second page cache statistics:
+#
+.B cachestat
+.SH FIELDS
+.TP
+TIME
+Time, in HH:MM:SS.
+.TP
+HITS
+Number of page cache hits (reads). Each hit is for one memory page (the size
+depends on your processor architecture; commonly 4 Kbytes). Since this tool
+outputs at a timed interval, this field indicates the cache hit rate.
+.TP
+MISSES
+Number of page cache misses (reads from storage I/O). Each miss is for one
+memory page. Cache misses should be causing disk I/O. Run iostat(1) for
+correlation (although the miss count and size by the time disk I/O is issued
+can differ due to I/O subsystem merging).
+.TP
+DIRTIES
+Number of times a page in the page cache was written to and thus "dirtied".
+The same page may be counted multiple times per interval, if it is written
+to multiple times. This field gives an indication of how much cache churn there
+is, caused by applications writing data.
+.TP
+RATIO
+The ratio of cache hits to total cache accesses (hits + misses), as a
+percentage.
+.TP
+BUFFERS_MB
+Size of the buffer cache, for disk I/O. From /proc/meminfo.
+.TP
+CACHED_MB
+Size of the page cache, for file system I/O. From /proc/meminfo.
+.SH OVERHEAD
+This tool currently uses ftrace function profiling, which provides efficient
+in-kernel counters. However, the functions profiled are executed frequently,
+so the overheads can add up. Test and measure before use. My own testing
+showed around a 2% loss in application performance while this tool was running.
+.SH SOURCE
+This is from the perf-tools collection.
+.IP
+https://github.com/brendangregg/perf-tools
+.PP
+Also look under the examples directory for a text file containing example
+usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+iostat(1), iosnoop(8)

data/perf-tools/man/man8/execsnoop.8

@@ -0,0 +1,104 @@
+.TH execsnoop 8  "2014-07-07" "USER COMMANDS"
+.SH NAME
+execsnoop \- trace process exec() with arguments. Uses Linux ftrace.
+.SH SYNOPSIS
+.B execsnoop
+[\-hrt] [\-a argc] [\-d secs] [name]
+.SH DESCRIPTION
+execsnoop traces process execution, showing PID, PPID, and argument details
+if possible.
+
+This traces exec() from the fork()->exec() sequence, which means it won't
+catch new processes that only fork(). With the -r option, it will also catch
+processes that re-exec. It makes a best-effort attempt to retrieve the program
+arguments and PPID; if these are unavailable, 0 and "[?]" are printed
+respectively. There is also a limit to the number of arguments printed (by
+default, 8), which can be increased using -a.
+
+This implementation is designed to work on older kernel versions, and without
+kernel debuginfo. It works by dynamic tracing an execve kernel function to
+read the arguments from the %si register. The stub_execve() function is tried
+first, and then the do_execve() function. The sched:sched_process_fork
+tracepoint, is used for the PPID. Tracing registers and kernel functions is
+an unstable technique, and this tool may not work for some kernels or platforms.
+
+This program is a workaround that should be
+improved in the future when other kernel capabilities are made available. If
+you need a more reliable tool now, then consider other tracing alternatives
+(eg, SystemTap). This tool is really a proof of concept to see what ftrace can
+currently do.
+
+Since this uses ftrace, only the root user can use this tool.
+.SH REQUIREMENTS
+FTRACE and KPROBE CONFIG, sched:sched_process_fork tracepoint,
+and either the stub_execve() or do_execve() kernel function. You may already
+have these on recent kernels. And awk.
+.SH OPTIONS
+.TP
+\-a argc
+Maximum number of arguments to show. The default is 8, and the maximum allowed
+is 16. If execsnoop thinks it has truncated the argument list, an ellipsis 
+"[...]" will be shown.
+.TP
+\-d seconds
+Duration to trace, in seconds. This also uses in-kernel buffering.
+.TP
+\-h
+Print usage message.
+.TP
+\-r
+Include re-exec()s.
+.TP
+\-t
+Include timestamps in units of seconds.
+.TP
+name
+Only show processes that match this name.
+Partials and regular expressions are allowed, as this is filtered in
+user space by awk.
+.SH EXAMPLES
+.TP
+Trace all new processes and arguments (if possible):
+#
+.B execsnoop
+.TP
+Trace all new process names containing the text "http":
+#
+.B execsnoop http
+.SH FIELDS
+.TP
+TIMEs
+Time of the exec(), in seconds.
+.TP
+PID
+Process ID.
+.TP
+PPID
+Parent process ID, if this was able to be read. If it wasn't, 0 is printed.
+.TP
+ARGS
+Command line arguments, if these were able to be read. If they aren't able to be
+read, "[?]" is printed (which would be due to a limitation in this tools
+implementation, since this is workaround for older kernels; if you need
+reliable argument tracing, use a different tracer). They will be truncated
+to the argc limit, and an ellipsis "[...]" may be printed if execsnoop is
+aware of the truncation.
+.SH OVERHEAD
+This reads and processes exec() events in user space as they occur. Since the
+rate of exec() is expected to be low (< 500/s), the overhead is expected to
+be small or negligible.
+.SH SOURCE
+This is from the perf-tools collection.
+.IP
+https://github.com/brendangregg/perf-tools
+.PP
+Also look under the examples directory for a text file containing example
+usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+top(1)

data/perf-tools/man/man8/funccount.8

@@ -0,0 +1,76 @@
+.TH funccount 8  "2014-07-19" "USER COMMANDS"
+.SH NAME
+funccount \- count kernel function calls matching specified wildcards. Uses Linux ftrace.
+.SH SYNOPSIS
+.B funccount
+[\-hT] [\-i secs] [\-d secs] [\-t top] funcstring
+.SH DESCRIPTION
+This tool is a quick way to determine which kernel functions are being called,
+and at what rate. It uses ftrace function profiling capabilities.
+
+WARNING: This uses dynamic tracing of (what can be many) kernel functions,
+and could cause kernel panics or freezes. Test, and know what you are doing,
+before use.
+
+Since this uses ftrace, only the root user can use this tool.
+.SH REQUIREMENTS
+CONFIG_FUNCTION_PROFILER, which you may already have enabled and available on
+recent kernels, and awk.
+.SH OPTIONS
+\-d seconds
+Total duration of the trace.
+.TP
+\-h
+Print usage message.
+.TP
+\-i seconds
+Print an interval summary every so many seconds.
+.TP
+\-t top
+Print top number of entries only.
+.TP
+\-T
+Include timestamp on each summary.
+.TP
+funcstring
+A function name to trace, which may include file glob style wildcards ("*") at
+the beginning or ending of a string only. Eg, "vfs*" means match "vfs" followed
+by anything.
+.SH EXAMPLES
+.TP
+Count every kernel function beginning with "bio_", until Ctrl-C is hit:
+#
+.B funccount 'bio_*'
+.TP
+Count every "tcp_*" kernel function, and print a summary every one second, five in total:
+#
+.B funccount \-i 1 \-d 5 'tcp_*'
+.TP
+Count every "ext4*" kernel function, and print the top 20 when Ctrl-C is hit:
+#
+.B funccount \-t 20 'ext4*'
+.SH FIELDS
+.TP
+FUNC
+Kernel function name.
+.TP
+COUNT
+Number of times this function was called during the tracing interval.
+.SH OVERHEAD
+This uses the ftrace profiling framework, which does in-kernel counts,
+lowering the overhead (compared to tracing each event).
+.SH SOURCE
+This is from the perf-tools collection:
+.IP
+https://github.com/brendangregg/perf-tools
+.PP
+Also look under the examples directory for a text file containing example
+usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+functrace(8)