perfmonger 0.12.1 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9610baac2556f61a8deb2cab5c7d4c10dac9c7da69c0c07a2f7d2b5f2414199e
-  data.tar.gz: 1e94e2c14421eabddd25f60abb8ef6e749aa57a557c4d661830b52fc5f91d7ca
+  metadata.gz: 9b1f5e883802377df1cc422ab0e100f2445bdaafb7d77da2ddcb279232cd63b4
+  data.tar.gz: b3cf4f9bd787f1b9b47dcff4da7480d8c30385c89a3785ad0793c5c40703dd06
 SHA512:
-  metadata.gz: 7a5f69061b79e2dcafe707bdb24b5a7be0443df2130bd928f33a7aba7a56623a946979ac22e0a82ce367f6d8d17d66facaf7286c6fe1c09016f8cedfc43dc5f6
-  data.tar.gz: 9754fb8acc4b1a521c2daf2a67017d95ec018b414f7f808989589efa498f3f66003f93d1d8d6aa7455a008259486f7ea71dae8c8d2934c52ed0b55e9dd0b0a1f
+  metadata.gz: cdca2aa8a3ad76a274cdc142f1738e12578dc06117de98a1e24e41aee5604452a113becdd506df19d49b5ccf7aa1e1d1eaa58ab8a52b1fd121181318f3fdad51
+  data.tar.gz: 76697490438d0b364be27504a7db0697d33b9f3bc5dc56f11db8dc89f5a1f682a270ad2d50bbedb40852ea759a0cf98708cc0fd159bc73568f17e11da2f43de7

data/NEWS

@@ -1,4 +1,28 @@
-## 20XX-XX-XX: PerfMonger 0.13.0
+## 2021-XX-XX: PerfMonger 0.15.0
+
+## 2021-12-01: PerfMonger 0.14.2
+  * Bugfix
+    * [player] subcommand:
+      * Read stdin if no argument is given (required behaviour for live subcommand)
+
+## 2021-12-01: PerfMonger 0.14.1
+  * Changes
+    * [plot] subcommand:
+      * Skip plotting memory usage if it is not recorded
+
+## 2021-07-27: PerfMonger 0.14.0
+  * New features
+    * [plot] subcommand:
+      * Added memory usage plot
+  * Changes
+    * [record] subcommand:
+      * Additionaly record 'Hugepagesize' in memory usage
+
+## 2021-07-26: PerfMonger 0.13.1
+  * New features
+    * [play] subcommand:
+      * Added --disk-only option to filter results of not interesting disks
+  * Note: v0.13.0 is yanked from rubygems.org
 
 ## 2021-07-08: PerfMonger 0.12.1
   * Bug fixes

data/core/cmd/perfmonger-player/perfmonger-player.go

@@ -9,16 +9,19 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
 
 	projson "github.com/hayamiz/go-projson"
+	core "github.com/hayamiz/perfmonger/core"
 	ss "github.com/hayamiz/perfmonger/core/subsystem"
-    core "github.com/hayamiz/perfmonger/core"
 )
 
 type PlayerOption struct {
-	logfile string
-	color   bool
-	pretty  bool
+	logfile         string
+	color           bool
+	pretty          bool
+	disk_only       string
+	disk_only_regex *regexp.Regexp
 }
 
 var option PlayerOption
@@ -50,10 +53,11 @@ func showInterruptStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cu
 	return nil
 }
 
-func showDiskStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
-	dusage, err := ss.GetDiskUsage(
+func showDiskStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord, disk_only_regex *regexp.Regexp) error {
+	dusage, err := ss.GetDiskUsage1(
 		prev_rec.Time, prev_rec.Disk,
-		cur_rec.Time, cur_rec.Disk)
+		cur_rec.Time, cur_rec.Disk,
+		option.disk_only_regex)
 	if err != nil {
 		return err
 	}
@@ -94,7 +98,9 @@ func showMemStat(printer *projson.JsonPrinter, cur_rec *ss.StatRecord) error {
 	return nil
 }
 
-func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
+func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord,
+	disk_only_regex *regexp.Regexp) error {
+
 	printer.Reset()
 	if option.pretty {
 		printer.SetStyle(projson.SmartStyle)
@@ -102,6 +108,7 @@ func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss
 	if option.color {
 		printer.SetColor(true)
 	}
+
 	printer.BeginObject()
 	printer.PutKey("time")
 	printer.PutFloatFmt(float64(cur_rec.Time.UnixNano())/1e9, "%.3f")
@@ -122,7 +129,7 @@ func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss
 		}
 	}
 	if cur_rec.Disk != nil {
-		err := showDiskStat(printer, prev_rec, cur_rec)
+		err := showDiskStat(printer, prev_rec, cur_rec, disk_only_regex)
 		if err != nil {
 			return err
 		}
@@ -148,10 +155,25 @@ func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss
 func parseArgs() {
 	flag.BoolVar(&option.color, "color", false, "Use colored JSON output")
 	flag.BoolVar(&option.pretty, "pretty", false, "Use human readable JSON output")
+	flag.StringVar(&option.disk_only, "disk-only", "", "Select disk devices by regex")
 
 	flag.Parse()
 
-	option.logfile = flag.Arg(0)
+	option.disk_only_regex = nil
+
+	if option.disk_only != "" {
+		var err error
+		option.disk_only_regex, err = regexp.Compile(option.disk_only)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	if len(flag.Args()) < 1 {
+		option.logfile = "-"
+	} else {
+		option.logfile = flag.Arg(0)
+	}
 }
 
 func main() {
@@ -160,7 +182,7 @@ func main() {
 
 	parseArgs()
 
-	if option.logfile == "" {
+	if option.logfile == "-" {
 		in = os.Stdin
 	} else {
 		f, err := os.Open(option.logfile)
@@ -219,7 +241,7 @@ func main() {
 			panic(err)
 		}
 
-		err = showStat(printer, prev_rec, cur_rec)
+		err = showStat(printer, prev_rec, cur_rec, option.disk_only_regex)
 		if err != nil {
 			printer.Reset()
 			fmt.Fprintln(os.Stderr, "skip by err")

data/core/cmd/perfmonger-plot-formatter/README.md

@@ -0,0 +1,24 @@
+# perfmonger-plot-formatter
+
+`perfmonger-plot-formatter` is a command to generate data files for plotting
+various metrices recorded by `perfmonger-recorder`.
+
+## Notes on memory usage records
+
+### `free(1)` compatible summarization
+
+- total: `MemStat.MemTotal`
+- used: `if (memUsed < 0) { MemStat.MemTotal - MemStat.MemFree } else { memUsed }`
+  - memUsed := `MemStat.MemTotal - MemStat.MemFree - mainCached - MemStat.Buffers`
+  - mainCached := `MemStat.Cached + MemStat.SReclaimable`
+- free: `MemStat.MemFree`
+- shared: `MemStat.Shmem`
+- buffers: `MemStat.Buffers`
+- cache: `mainCached`
+  - mainCached := `MemStat.Cached + MemStat.SReclaimable`
+- available: `MemStat.MemAvailable`
+
+### Additional info
+
+- hugeTotal: `MemStat.HugePages_Total`
+- hugeUsed: `MemStat.HugePages_Total - MemStat.HugePages_Free`

data/core/cmd/perfmonger-plot-formatter/perfmonger-plot-formatter.go

@@ -13,14 +13,16 @@ import (
 	"os"
 	"regexp"
 	"sort"
+	"strings"
 
-	ss "github.com/hayamiz/perfmonger/core/subsystem"
 	"github.com/hayamiz/perfmonger/core"
+	ss "github.com/hayamiz/perfmonger/core/subsystem"
 )
 
 type CmdOption struct {
 	DiskFile        string
 	CpuFile         string
+	MemFile         string
 	PerfmongerFile  string
 	disk_only       string
 	disk_only_regex *regexp.Regexp
@@ -31,8 +33,9 @@ func parseArgs() *CmdOption {
 
 	opt := new(CmdOption)
 
-	flag.StringVar(&opt.DiskFile, "diskfile", "./disk.dat", "Disk performance data file")
-	flag.StringVar(&opt.CpuFile, "cpufile", "./cpu.dat", "CPU performance data file")
+	flag.StringVar(&opt.DiskFile, "diskfile", "./disk.dat", "Disk usage data file for gnuplot")
+	flag.StringVar(&opt.CpuFile, "cpufile", "./cpu.dat", "CPU usage data file for gnuplot")
+	flag.StringVar(&opt.MemFile, "memfile", "./mem.dat", "Memory usage data file for gnuplot")
 	flag.StringVar(&opt.PerfmongerFile, "perfmonger", "", "Perfmonger log file")
 	flag.StringVar(&opt.disk_only, "disk-only",
 		"", "Select disk devices by regex")
@@ -80,6 +83,13 @@ type DiskDatTmpFile struct {
 	Idx    int
 }
 
+type MemDatTmpFile struct {
+	Name   string
+	Path   string
+	File   *os.File
+	Writer *bufio.Writer
+}
+
 type CpuDatTmpFile struct {
 	CoreId int
 	Path   string
@@ -118,6 +128,20 @@ func makeCpuDatTmpFile(coreid int) *CpuDatTmpFile {
 	return ret
 }
 
+func makeMemDatTmpFile() *MemDatTmpFile {
+	ret := new(MemDatTmpFile)
+
+	f, err := ioutil.TempFile("", "perfmonger-mem")
+	if err != nil {
+		panic(err)
+	}
+	ret.File = f
+	ret.Path = f.Name()
+	ret.Writer = bufio.NewWriter(f)
+
+	return ret
+}
+
 func printCoreUsage(writer *bufio.Writer, elapsed_time float64, coreusage *ss.CpuCoreUsage) {
 	writer.WriteString(
 		fmt.Sprintf("%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\n",
@@ -133,6 +157,80 @@ func printCoreUsage(writer *bufio.Writer, elapsed_time float64, coreusage *ss.Cp
 			coreusage.Idle))
 }
 
+func printMemUsage(writer *bufio.Writer, elapsed_time float64, mem *ss.MemStat) {
+	if mem == nil {
+		writer.WriteString("#")
+		writer.WriteString(
+			strings.Join([]string{
+				"elapsed_time",     // 1
+				"mem_total",        // 2
+				"mem_used",         // 3
+				"mem_free",         // 4
+				"buffers",          // 5
+				"cached",           // 6
+				"swap_cached",      // 7
+				"active",           // 8
+				"inactive",         // 9
+				"swap_total",       // 10
+				"swap_free",        // 11
+				"dirty",            // 12
+				"writeback",        // 13
+				"anon_pages",       // 14
+				"mapped",           // 15
+				"shmem",            // 16
+				"slab",             // 17
+				"s_reclaimable",    // 18
+				"s_unreclaim",      // 19
+				"kernel_stack",     // 20
+				"page_tables",      // 21
+				"nfs_unstable",     // 22
+				"bounce",           // 23
+				"commit_limit",     // 24
+				"committed_as",     // 25
+				"anon_huge_pages",  // 26
+				"huge_pages_total", // 27
+				"huge_pages_free",  // 28
+				"huge_pages_rsvd",  // 29
+				"huge_pages_surp",  // 30
+				"hugepagesize"},    // 31
+				"\t"))
+		writer.WriteString("\n")
+	} else {
+		writer.WriteString(fmt.Sprintf("%f\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
+			elapsed_time,
+			mem.MemTotal,
+			mem.MemTotal-mem.MemFree-mem.Buffers-mem.Cached-mem.SReclaimable,
+			mem.MemFree,
+			mem.Buffers,
+			mem.Cached,
+			mem.SwapCached,
+			mem.Active,
+			mem.Inactive,
+			mem.SwapTotal,
+			mem.SwapFree,
+			mem.Dirty,
+			mem.Writeback,
+			mem.AnonPages,
+			mem.Mapped,
+			mem.Shmem,
+			mem.Slab,
+			mem.SReclaimable,
+			mem.SUnreclaim,
+			mem.KernelStack,
+			mem.PageTables,
+			mem.NFS_Unstable,
+			mem.Bounce,
+			mem.CommitLimit,
+			mem.Committed_AS,
+			mem.AnonHugePages,
+			mem.HugePages_Total,
+			mem.HugePages_Free,
+			mem.HugePages_Rsvd,
+			mem.HugePages_Surp,
+			mem.Hugepagesize))
+	}
+}
+
 func main() {
 	opt := parseArgs()
 
@@ -193,6 +291,13 @@ func main() {
 	cpu_writer.WriteString("# All cpu usage\n")
 	cpu_writer.WriteString("# elapsed_time	%usr	%nice	%sys	%iowait	%hardirq	%softirq	%steal	%guest	%idle\n")
 
+	f, err = os.Create(opt.MemFile)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	mem_writer := bufio.NewWriter(f)
+
 	for {
 		prev_rec := &records[curr^1]
 		cur_rec := &records[curr]
@@ -284,6 +389,12 @@ func main() {
 		}
 		printCoreUsage(cpu_writer, prev_rec.Time.Sub(t0).Seconds(), cusage.All)
 
+		if !meta_set {
+			// print column labels
+			printMemUsage(mem_writer, prev_rec.Time.Sub(t0).Seconds(), nil)
+		}
+		printMemUsage(mem_writer, prev_rec.Time.Sub(t0).Seconds(), cur_rec.Mem)
+
 		curr ^= 1
 		meta_set = true
 	}
@@ -332,6 +443,7 @@ func main() {
 		os.Remove(cpu_dat.Path)
 	}
 	cpu_writer.Flush()
+	mem_writer.Flush()
 
 	json_enc := json.NewEncoder(os.Stdout)
 	json_enc.Encode(meta)

data/core/cmd/perfmonger-summarizer/perfmonger-summarizer.go

@@ -12,8 +12,8 @@ import (
 	"sort"
 
 	projson "github.com/hayamiz/go-projson"
-	ss "github.com/hayamiz/perfmonger/core/subsystem"
 	"github.com/hayamiz/perfmonger/core"
+	ss "github.com/hayamiz/perfmonger/core/subsystem"
 )
 
 type SummaryOption struct {
@@ -39,6 +39,7 @@ func parseArgs() {
 	flag.Parse()
 
 	if len(flag.Args()) < 1 {
+		fmt.Fprintln(os.Stderr, "Insufficient argument")
 		os.Exit(1)
 	}
 

data/core/subsystem/perfmonger_linux.go

@@ -469,6 +469,8 @@ func ReadMemStat(record *StatRecord) error {
 			mem_stat.HugePages_Free = val
 		case "HugePages_Total:":
 			mem_stat.HugePages_Total = val
+		case "Hugepagesize:":
+			mem_stat.Hugepagesize = val
 		case "AnonHugePages:":
 			mem_stat.AnonHugePages = val
 		case "Committed_AS:":

data/core/subsystem/stat.go

@@ -132,6 +132,7 @@ type MemStat struct {
 	HugePages_Free  int64
 	HugePages_Rsvd  int64
 	HugePages_Surp  int64
+	Hugepagesize    int64
 }
 
 type StatRecord struct {
@@ -286,6 +287,7 @@ func (entry *MemStat) Clear() {
 	entry.HugePages_Free = 0
 	entry.HugePages_Rsvd = 0
 	entry.HugePages_Surp = 0
+	entry.Hugepagesize = 0
 }
 
 func NewStatRecord() *StatRecord {

data/lib/perfmonger/command/play.rb

@@ -26,6 +26,10 @@ EOS
       @pretty = true
     end
 
+    @disk_only_regex = nil
+    @parser.on('--disk-only REGEX', "Select disk devices that matches REGEX (Ex. 'sd[b-d]')") do |regex|
+      @disk_only_regex = regex
+    end
   end
 
   def parse_args(argv)
@@ -62,6 +66,12 @@ EOS
     if @pretty
       cmd << "-pretty"
     end
+
+    if @disk_only_regex
+      cmd << "-disk-only"
+      cmd << @disk_only_regex
+    end
+
     cmd << @logfile
 
     Process.exec(*cmd)

data/lib/perfmonger/command/plot.rb

@@ -145,10 +145,13 @@ EOS
 
     @disk_dat = File.expand_path("disk.dat", @tmpdir)
     @cpu_dat = File.expand_path("cpu.dat", @tmpdir)
+    @mem_dat = File.expand_path("mem.dat", @tmpdir)
 
     meta_json = nil
-    cmd = [formatter_bin, "-perfmonger", @data_file, "-cpufile", @cpu_dat,
-      "-diskfile", @disk_dat]
+    cmd = [formatter_bin, "-perfmonger", @data_file,
+           "-cpufile", @cpu_dat,
+           "-diskfile", @disk_dat,
+           "-memfile", @mem_dat]
     if @disk_only_regex
       cmd << "-disk-only"
       cmd << @disk_only
@@ -164,6 +167,7 @@ EOS
 
     plot_disk(meta)
     plot_cpu(meta)
+    plot_mem(meta)
 
     FileUtils.rm_rf(@tmpdir)
 
@@ -508,6 +512,144 @@ EOS
     end
   end # def
 
+  def plot_mem(meta)
+    pdf_filename = @output_prefix + 'mem.pdf'
+    gp_filename  = @output_prefix + 'mem.gp'
+    dat_filename = @output_prefix + 'mem.dat'
+
+    unless File.exists?(dat_filename)
+      $stderr.puts("WARN: memory usage info is not available.")
+      return
+    end
+
+    if @output_type != 'pdf'
+      img_filename = @output_prefix + 'cpu.' + @output_type
+    else
+      img_filename = nil
+    end
+
+    start_time = meta["start_time"]
+    end_time = meta["end_time"]
+
+    mem_scaling = 2.0**20 # KB -> GB
+
+# "elapsed_time",     // 1
+# "mem_total",        // 2
+# "mem_used",         // 3
+# "mem_free",         // 4
+# "buffers",          // 5
+# "cached",           // 6
+# "swap_cached",      // 7
+# "active",           // 8
+# "inactive",         // 9
+# "swap_total",       // 10
+# "swap_free",        // 11
+# "dirty",            // 12
+# "writeback",        // 13
+# "anon_pages",       // 14
+# "mapped",           // 15
+# "shmem",            // 16
+# "slab",             // 17
+# "s_reclaimable",    // 18
+# "s_unreclaim",      // 19
+# "kernel_stack",     // 20
+# "page_tables",      // 21
+# "nfs_unstable",     // 22
+# "bounce",           // 23
+# "commit_limit",     // 24
+# "committed_as",     // 25
+# "anon_huge_pages",  // 26
+# "huge_pages_total", // 27
+# "huge_pages_free",  // 28
+# "huge_pages_rsvd",  // 29
+# "huge_pages_surp"}, // 30
+# "hugepagesize"},    // 31
+
+    Dir.chdir(@tmpdir) do
+      total = `tail -n+2 #{dat_filename}|head -n1`.split[1].to_f
+      if total == 0.0
+        raise RuntimeError.new("Failed to get MemTotal value from mem.dat file: #{dat_filename}")
+      end
+
+      gpfile = File.open(gp_filename, 'w')
+
+      pdf_file = File.join(@output_dir, "mem.pdf")
+      gpfile.puts <<EOS
+set term pdfcairo enhanced color size 6in,2.5in
+set title "Memory usage"
+set output "#{pdf_filename}"
+set key outside center bottom horizontal
+set size 1.0, 1.0
+
+set xlabel "elapsed time [sec]"
+set ylabel "memory usage [GB]"
+
+# scaling
+s = #{mem_scaling}
+
+set grid
+set xrange [#{@offset_time}:#{end_time - start_time}]
+set yrange [0:#{total * 1.2}/s]
+
+# line styles
+set style line 1 lt 1 lc rgb '#66C2A5' # teal
+set style line 2 lt 1 lc rgb '#FC8D62' # orange
+set style line 3 lt 1 lc rgb '#8DA0CB' # lilac
+set style line 4 lt 1 lc rgb '#E78AC3' # magentat
+set style line 5 lt 1 lc rgb '#A6D854' # lime green
+set style line 6 lt 1 lc rgb '#FFD92F' # banana
+set style line 7 lt 1 lc rgb '#E5C494' # tan
+set style line 8 lt 1 lc rgb '#B3B3B3' # grey
+
+# palette
+set palette maxcolors 8
+set palette defined ( 0 '#66C2A5',\
+    	    	      1 '#FC8D62',\
+		      2 '#8DA0CB',\
+		      3 '#E78AC3',\
+		      4 '#A6D854',\
+		      5 '#FFD92F',\
+		      6 '#E5C494',\
+		      7 '#B3B3B3' )
+
+used(total, free, cached, buffers, srecl) = \\
+  ( (total - free - cache(cached, srecl) - buffers < 0) ? \\
+    (total - free) : \\
+    (total - free - cache(cached, srecl) - buffers) )
+
+cache(cached, srecl) = cached + srecl
+
+plot "#{dat_filename}" usi 1:($4/s+$5/s+cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 8 ti "free", \\
+     "#{dat_filename}" usi 1:($5/s+cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 3 ti "buffers", \\
+     "#{dat_filename}" usi 1:(cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 5 ti "cached", \\
+     "#{dat_filename}" usi 1:($16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 6 ti "shared", \\
+     "#{dat_filename}" usi 1:(used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 2 ti "used", \\
+     "#{dat_filename}" usi 1:(($27-$28)*$31/s) wi lines dt (4,4) lc rgb 'black' lw 2.5 ti 'used (hugepage)'
+
+EOS
+
+      gpfile.close
+      system("gnuplot #{gpfile.path}")
+
+      if @output_type != 'pdf'
+        system("convert -density 150 -background white #{pdf_filename} #{img_filename}")
+      end
+    end # chdir
+
+    copy_targets = []
+    copy_targets << pdf_filename
+    copy_targets << img_filename if img_filename
+
+    if @save_gpfiles
+      copy_targets << gp_filename
+      copy_targets << dat_filename
+    end
+
+    copy_targets.each do |target|
+      FileUtils.copy(File.join(@tmpdir, target), @output_dir)
+    end
+  end # def
+
   private
   def factors(n)
     (2..([n, n / 2].max).to_i).select do |x|

data/lib/perfmonger/version.rb

@@ -1,3 +1,3 @@
 module PerfMonger
-  VERSION = "0.12.1"
+  VERSION = "0.14.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfmonger
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.14.2
 platform: ruby
 authors:
 - Yuto HAYAMIZU
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-12 00:00:00.000000000 Z
+date: 2021-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,6 +102,7 @@ files:
 - core/Makefile
 - core/build.sh
 - core/cmd/perfmonger-player/perfmonger-player.go
+- core/cmd/perfmonger-plot-formatter/README.md
 - core/cmd/perfmonger-plot-formatter/perfmonger-plot-formatter.go
 - core/cmd/perfmonger-recorder/perfmonger-recorder.go
 - core/cmd/perfmonger-summarizer/perfmonger-summarizer.go