perfmonger 0.13.1 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a2bc69e72d02745481dafb5bec13073d4289a3f38cdf8eb253416e649e56ff5
-  data.tar.gz: fd5db8b9b288b62060df2ab5881ec8c777c320479d53420ddef3e4420ade9c9a
+  metadata.gz: 7cafa0eb63dce619c0f5cf2f23042db020e435905eb7d00e2338d3c499495d02
+  data.tar.gz: ab6875fdfe16bd70461db8569b4c5572bbd261e931a6efa98c2bede6660e9d1b
 SHA512:
-  metadata.gz: c7184d7eeb69b60188a740031143eede5188b56dbf84cd6a4bdd28302439faba79f390f8baf27f1e33275d04355c1e99c42eb09b1eb663a243c0046e1cc5a29e
-  data.tar.gz: cda569ba86554c985e46249d77d1fdc606fbe126150591dcec55ccbe82eabd13b6081ed428c6b7301004c1955fdf00273549a82696f6ed07cd3c2bb55387dd45
+  metadata.gz: 17bb023ddebbde5fda03b2ed90411274a335e34ac5ddc83e4df7821454e40484957688ed7dfe72b9245190006fee0c2e1e39aa375fe17c12f95bba9838f2d767
+  data.tar.gz: 259636bb0f5049e404ec752edf0e659b23b8bc1adcaea69ef2f0db4cc3bfccac028793fd48920d9e4a54e5cda86a3a52581eb2dc250abd0b5f8476dec815772f

data/NEWS

@@ -1,4 +1,12 @@
-## 2021-XX-XX: PerfMonger 0.14.0
+## 2021-XX-XX: PerfMonger 0.15.0
+
+## 2021-07-27: PerfMonger 0.14.0
+  * New features
+    * [plot] subcommand:
+      * Added memory usage plot
+  * Changes
+    * [record] subcommand:
+      * Additionaly record 'Hugepagesize' in memory usage
 
 ## 2021-07-26: PerfMonger 0.13.1
   * New features

data/core/cmd/perfmonger-plot-formatter/README.md

@@ -0,0 +1,24 @@
+# perfmonger-plot-formatter
+
+`perfmonger-plot-formatter` is a command to generate data files for plotting
+various metrices recorded by `perfmonger-recorder`.
+
+## Notes on memory usage records
+
+### `free(1)` compatible summarization
+
+- total: `MemStat.MemTotal`
+- used: `if (memUsed < 0) { MemStat.MemTotal - MemStat.MemFree } else { memUsed }`
+  - memUsed := `MemStat.MemTotal - MemStat.MemFree - mainCached - MemStat.Buffers`
+  - mainCached := `MemStat.Cached + MemStat.SReclaimable`
+- free: `MemStat.MemFree`
+- shared: `MemStat.Shmem`
+- buffers: `MemStat.Buffers`
+- cache: `mainCached`
+  - mainCached := `MemStat.Cached + MemStat.SReclaimable`
+- available: `MemStat.MemAvailable`
+
+### Additional info
+
+- hugeTotal: `MemStat.HugePages_Total`
+- hugeUsed: `MemStat.HugePages_Total - MemStat.HugePages_Free`

data/core/cmd/perfmonger-plot-formatter/perfmonger-plot-formatter.go

@@ -13,14 +13,16 @@ import (
 	"os"
 	"regexp"
 	"sort"
+	"strings"
 
-	ss "github.com/hayamiz/perfmonger/core/subsystem"
 	"github.com/hayamiz/perfmonger/core"
+	ss "github.com/hayamiz/perfmonger/core/subsystem"
 )
 
 type CmdOption struct {
 	DiskFile        string
 	CpuFile         string
+	MemFile         string
 	PerfmongerFile  string
 	disk_only       string
 	disk_only_regex *regexp.Regexp
@@ -31,8 +33,9 @@ func parseArgs() *CmdOption {
 
 	opt := new(CmdOption)
 
-	flag.StringVar(&opt.DiskFile, "diskfile", "./disk.dat", "Disk performance data file")
-	flag.StringVar(&opt.CpuFile, "cpufile", "./cpu.dat", "CPU performance data file")
+	flag.StringVar(&opt.DiskFile, "diskfile", "./disk.dat", "Disk usage data file for gnuplot")
+	flag.StringVar(&opt.CpuFile, "cpufile", "./cpu.dat", "CPU usage data file for gnuplot")
+	flag.StringVar(&opt.MemFile, "memfile", "./mem.dat", "Memory usage data file for gnuplot")
 	flag.StringVar(&opt.PerfmongerFile, "perfmonger", "", "Perfmonger log file")
 	flag.StringVar(&opt.disk_only, "disk-only",
 		"", "Select disk devices by regex")
@@ -80,6 +83,13 @@ type DiskDatTmpFile struct {
 	Idx    int
 }
 
+type MemDatTmpFile struct {
+	Name   string
+	Path   string
+	File   *os.File
+	Writer *bufio.Writer
+}
+
 type CpuDatTmpFile struct {
 	CoreId int
 	Path   string
@@ -118,6 +128,20 @@ func makeCpuDatTmpFile(coreid int) *CpuDatTmpFile {
 	return ret
 }
 
+func makeMemDatTmpFile() *MemDatTmpFile {
+	ret := new(MemDatTmpFile)
+
+	f, err := ioutil.TempFile("", "perfmonger-mem")
+	if err != nil {
+		panic(err)
+	}
+	ret.File = f
+	ret.Path = f.Name()
+	ret.Writer = bufio.NewWriter(f)
+
+	return ret
+}
+
 func printCoreUsage(writer *bufio.Writer, elapsed_time float64, coreusage *ss.CpuCoreUsage) {
 	writer.WriteString(
 		fmt.Sprintf("%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\n",
@@ -133,6 +157,80 @@ func printCoreUsage(writer *bufio.Writer, elapsed_time float64, coreusage *ss.Cp
 			coreusage.Idle))
 }
 
+func printMemUsage(writer *bufio.Writer, elapsed_time float64, mem *ss.MemStat) {
+	if mem == nil {
+		writer.WriteString("#")
+		writer.WriteString(
+			strings.Join([]string{
+				"elapsed_time",     // 1
+				"mem_total",        // 2
+				"mem_used",         // 3
+				"mem_free",         // 4
+				"buffers",          // 5
+				"cached",           // 6
+				"swap_cached",      // 7
+				"active",           // 8
+				"inactive",         // 9
+				"swap_total",       // 10
+				"swap_free",        // 11
+				"dirty",            // 12
+				"writeback",        // 13
+				"anon_pages",       // 14
+				"mapped",           // 15
+				"shmem",            // 16
+				"slab",             // 17
+				"s_reclaimable",    // 18
+				"s_unreclaim",      // 19
+				"kernel_stack",     // 20
+				"page_tables",      // 21
+				"nfs_unstable",     // 22
+				"bounce",           // 23
+				"commit_limit",     // 24
+				"committed_as",     // 25
+				"anon_huge_pages",  // 26
+				"huge_pages_total", // 27
+				"huge_pages_free",  // 28
+				"huge_pages_rsvd",  // 29
+				"huge_pages_surp",  // 30
+				"hugepagesize"},    // 31
+				"\t"))
+		writer.WriteString("\n")
+	} else {
+		writer.WriteString(fmt.Sprintf("%f\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
+			elapsed_time,
+			mem.MemTotal,
+			mem.MemTotal-mem.MemFree-mem.Buffers-mem.Cached-mem.SReclaimable,
+			mem.MemFree,
+			mem.Buffers,
+			mem.Cached,
+			mem.SwapCached,
+			mem.Active,
+			mem.Inactive,
+			mem.SwapTotal,
+			mem.SwapFree,
+			mem.Dirty,
+			mem.Writeback,
+			mem.AnonPages,
+			mem.Mapped,
+			mem.Shmem,
+			mem.Slab,
+			mem.SReclaimable,
+			mem.SUnreclaim,
+			mem.KernelStack,
+			mem.PageTables,
+			mem.NFS_Unstable,
+			mem.Bounce,
+			mem.CommitLimit,
+			mem.Committed_AS,
+			mem.AnonHugePages,
+			mem.HugePages_Total,
+			mem.HugePages_Free,
+			mem.HugePages_Rsvd,
+			mem.HugePages_Surp,
+			mem.Hugepagesize))
+	}
+}
+
 func main() {
 	opt := parseArgs()
 
@@ -193,6 +291,13 @@ func main() {
 	cpu_writer.WriteString("# All cpu usage\n")
 	cpu_writer.WriteString("# elapsed_time	%usr	%nice	%sys	%iowait	%hardirq	%softirq	%steal	%guest	%idle\n")
 
+	f, err = os.Create(opt.MemFile)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	mem_writer := bufio.NewWriter(f)
+
 	for {
 		prev_rec := &records[curr^1]
 		cur_rec := &records[curr]
@@ -284,6 +389,12 @@ func main() {
 		}
 		printCoreUsage(cpu_writer, prev_rec.Time.Sub(t0).Seconds(), cusage.All)
 
+		if !meta_set {
+			// print column labels
+			printMemUsage(mem_writer, prev_rec.Time.Sub(t0).Seconds(), nil)
+		}
+		printMemUsage(mem_writer, prev_rec.Time.Sub(t0).Seconds(), cur_rec.Mem)
+
 		curr ^= 1
 		meta_set = true
 	}
@@ -332,6 +443,7 @@ func main() {
 		os.Remove(cpu_dat.Path)
 	}
 	cpu_writer.Flush()
+	mem_writer.Flush()
 
 	json_enc := json.NewEncoder(os.Stdout)
 	json_enc.Encode(meta)

data/core/subsystem/perfmonger_linux.go

@@ -469,6 +469,8 @@ func ReadMemStat(record *StatRecord) error {
 			mem_stat.HugePages_Free = val
 		case "HugePages_Total:":
 			mem_stat.HugePages_Total = val
+		case "Hugepagesize:":
+			mem_stat.Hugepagesize = val
 		case "AnonHugePages:":
 			mem_stat.AnonHugePages = val
 		case "Committed_AS:":

data/core/subsystem/stat.go

@@ -132,6 +132,7 @@ type MemStat struct {
 	HugePages_Free  int64
 	HugePages_Rsvd  int64
 	HugePages_Surp  int64
+	Hugepagesize    int64
 }
 
 type StatRecord struct {
@@ -286,6 +287,7 @@ func (entry *MemStat) Clear() {
 	entry.HugePages_Free = 0
 	entry.HugePages_Rsvd = 0
 	entry.HugePages_Surp = 0
+	entry.Hugepagesize = 0
 }
 
 func NewStatRecord() *StatRecord {

data/lib/perfmonger/command/plot.rb

@@ -145,10 +145,13 @@ EOS
 
     @disk_dat = File.expand_path("disk.dat", @tmpdir)
     @cpu_dat = File.expand_path("cpu.dat", @tmpdir)
+    @mem_dat = File.expand_path("mem.dat", @tmpdir)
 
     meta_json = nil
-    cmd = [formatter_bin, "-perfmonger", @data_file, "-cpufile", @cpu_dat,
-      "-diskfile", @disk_dat]
+    cmd = [formatter_bin, "-perfmonger", @data_file,
+           "-cpufile", @cpu_dat,
+           "-diskfile", @disk_dat,
+           "-memfile", @mem_dat]
     if @disk_only_regex
       cmd << "-disk-only"
       cmd << @disk_only
@@ -164,6 +167,7 @@ EOS
 
     plot_disk(meta)
     plot_cpu(meta)
+    plot_mem(meta)
 
     FileUtils.rm_rf(@tmpdir)
 
@@ -508,6 +512,139 @@ EOS
     end
   end # def
 
+  def plot_mem(meta)
+    pdf_filename = @output_prefix + 'mem.pdf'
+    gp_filename  = @output_prefix + 'mem.gp'
+    dat_filename = @output_prefix + 'mem.dat'
+
+    if @output_type != 'pdf'
+      img_filename = @output_prefix + 'cpu.' + @output_type
+    else
+      img_filename = nil
+    end
+
+    start_time = meta["start_time"]
+    end_time = meta["end_time"]
+
+    mem_scaling = 2.0**20 # KB -> GB
+
+# "elapsed_time",     // 1
+# "mem_total",        // 2
+# "mem_used",         // 3
+# "mem_free",         // 4
+# "buffers",          // 5
+# "cached",           // 6
+# "swap_cached",      // 7
+# "active",           // 8
+# "inactive",         // 9
+# "swap_total",       // 10
+# "swap_free",        // 11
+# "dirty",            // 12
+# "writeback",        // 13
+# "anon_pages",       // 14
+# "mapped",           // 15
+# "shmem",            // 16
+# "slab",             // 17
+# "s_reclaimable",    // 18
+# "s_unreclaim",      // 19
+# "kernel_stack",     // 20
+# "page_tables",      // 21
+# "nfs_unstable",     // 22
+# "bounce",           // 23
+# "commit_limit",     // 24
+# "committed_as",     // 25
+# "anon_huge_pages",  // 26
+# "huge_pages_total", // 27
+# "huge_pages_free",  // 28
+# "huge_pages_rsvd",  // 29
+# "huge_pages_surp"}, // 30
+# "hugepagesize"},    // 31
+
+    Dir.chdir(@tmpdir) do
+      total = `tail -n+2 #{dat_filename}|head -n1`.split[1].to_f
+      if total == 0.0
+        raise RuntimeError.new("Failed to get MemTotal value from mem.dat file: #{dat_filename}")
+      end
+
+      gpfile = File.open(gp_filename, 'w')
+
+      pdf_file = File.join(@output_dir, "mem.pdf")
+      gpfile.puts <<EOS
+set term pdfcairo enhanced color size 6in,2.5in
+set title "Memory usage"
+set output "#{pdf_filename}"
+set key outside center bottom horizontal
+set size 1.0, 1.0
+
+set xlabel "elapsed time [sec]"
+set ylabel "memory usage [GB]"
+
+# scaling
+s = #{mem_scaling}
+
+set grid
+set xrange [#{@offset_time}:#{end_time - start_time}]
+set yrange [0:#{total * 1.2}/s]
+
+# line styles
+set style line 1 lt 1 lc rgb '#66C2A5' # teal
+set style line 2 lt 1 lc rgb '#FC8D62' # orange
+set style line 3 lt 1 lc rgb '#8DA0CB' # lilac
+set style line 4 lt 1 lc rgb '#E78AC3' # magentat
+set style line 5 lt 1 lc rgb '#A6D854' # lime green
+set style line 6 lt 1 lc rgb '#FFD92F' # banana
+set style line 7 lt 1 lc rgb '#E5C494' # tan
+set style line 8 lt 1 lc rgb '#B3B3B3' # grey
+
+# palette
+set palette maxcolors 8
+set palette defined ( 0 '#66C2A5',\
+    	    	      1 '#FC8D62',\
+		      2 '#8DA0CB',\
+		      3 '#E78AC3',\
+		      4 '#A6D854',\
+		      5 '#FFD92F',\
+		      6 '#E5C494',\
+		      7 '#B3B3B3' )
+
+used(total, free, cached, buffers, srecl) = \\
+  ( (total - free - cache(cached, srecl) - buffers < 0) ? \\
+    (total - free) : \\
+    (total - free - cache(cached, srecl) - buffers) )
+
+cache(cached, srecl) = cached + srecl
+
+plot "#{dat_filename}" usi 1:($4/s+$5/s+cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 8 ti "free", \\
+     "#{dat_filename}" usi 1:($5/s+cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 3 ti "buffers", \\
+     "#{dat_filename}" usi 1:(cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 5 ti "cached", \\
+     "#{dat_filename}" usi 1:($16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 6 ti "shared", \\
+     "#{dat_filename}" usi 1:(used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 2 ti "used", \\
+     "#{dat_filename}" usi 1:(($27-$28)*$31/s) wi lines dt (4,4) lc rgb 'black' lw 2.5 ti 'used (hugepage)'
+
+EOS
+
+      gpfile.close
+      system("gnuplot #{gpfile.path}")
+
+      if @output_type != 'pdf'
+        system("convert -density 150 -background white #{pdf_filename} #{img_filename}")
+      end
+    end # chdir
+
+    copy_targets = []
+    copy_targets << pdf_filename
+    copy_targets << img_filename if img_filename
+
+    if @save_gpfiles
+      copy_targets << gp_filename
+      copy_targets << dat_filename
+    end
+
+    copy_targets.each do |target|
+      FileUtils.copy(File.join(@tmpdir, target), @output_dir)
+    end
+  end # def
+
   private
   def factors(n)
     (2..([n, n / 2].max).to_i).select do |x|

data/lib/perfmonger/version.rb

@@ -1,3 +1,3 @@
 module PerfMonger
-  VERSION = "0.13.1"
+  VERSION = "0.14.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfmonger
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.14.0
 platform: ruby
 authors:
 - Yuto HAYAMIZU
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-26 00:00:00.000000000 Z
+date: 2021-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,6 +102,7 @@ files:
 - core/Makefile
 - core/build.sh
 - core/cmd/perfmonger-player/perfmonger-player.go
+- core/cmd/perfmonger-plot-formatter/README.md
 - core/cmd/perfmonger-plot-formatter/perfmonger-plot-formatter.go
 - core/cmd/perfmonger-recorder/perfmonger-recorder.go
 - core/cmd/perfmonger-summarizer/perfmonger-summarizer.go