perfmonger 0.11.2 → 0.14.0

data/core/utils.go

@@ -1,4 +1,4 @@
-package main
+package core
 
 import (
 	"bufio"
@@ -6,7 +6,7 @@ import (
 	"io"
 )
 
-func newPerfmongerLogReader(source io.Reader) io.Reader {
+func NewPerfmongerLogReader(source io.Reader) io.Reader {
 	var ret io.Reader
 	reader := bufio.NewReader(source)
 

data/lib/perfmonger/command/play.rb

@@ -26,6 +26,10 @@ EOS
       @pretty = true
     end
 
+    @disk_only_regex = nil
+    @parser.on('--disk-only REGEX', "Select disk devices that matches REGEX (Ex. 'sd[b-d]')") do |regex|
+      @disk_only_regex = regex
+    end
   end
 
   def parse_args(argv)
@@ -62,6 +66,12 @@ EOS
     if @pretty
       cmd << "-pretty"
     end
+
+    if @disk_only_regex
+      cmd << "-disk-only"
+      cmd << @disk_only_regex
+    end
+
     cmd << @logfile
 
     Process.exec(*cmd)

data/lib/perfmonger/command/plot.rb

@@ -30,6 +30,7 @@ EOS
     @disk_plot_write = true
     @disk_numkey_threshold = 10
     @plot_iops_max = nil
+    @gnuplot_bin = `which gnuplot`
   end
 
   def parse_args(argv)
@@ -47,6 +48,10 @@ EOS
       @output_dir = dir
     end
 
+    @parser.on('--with-gnuplot GNUPLOT_BIN') do |gnuplot_bin|
+      @gnuplot_bin = gnuplot_bin
+    end
+
     @parser.on('-T', '--output-type TYPE', 'Available: pdf, png') do |typ|
       unless ['pdf', 'png'].include?(typ)
         puts("ERROR: non supported image type: #{typ}")
@@ -119,13 +124,14 @@ EOS
   end
 
   def run(argv)
-    parse_args(argv)
     unless system('which gnuplot >/dev/null 2>&1')
       puts("ERROR: gnuplot not found")
       puts(@parser.help)
       exit(false)
     end
 
+    parse_args(argv)
+
     unless system('gnuplot -e "set terminal" < /dev/null 2>&1 | grep pdfcairo >/dev/null 2>&1')
       puts("ERROR: pdfcairo is not supported by installed gnuplot")
       puts("ERROR: PerfMonger requires pdfcairo-supported gnuplot")
@@ -139,10 +145,13 @@ EOS
 
     @disk_dat = File.expand_path("disk.dat", @tmpdir)
     @cpu_dat = File.expand_path("cpu.dat", @tmpdir)
+    @mem_dat = File.expand_path("mem.dat", @tmpdir)
 
     meta_json = nil
-    cmd = [formatter_bin, "-perfmonger", @data_file, "-cpufile", @cpu_dat,
-      "-diskfile", @disk_dat]
+    cmd = [formatter_bin, "-perfmonger", @data_file,
+           "-cpufile", @cpu_dat,
+           "-diskfile", @disk_dat,
+           "-memfile", @mem_dat]
     if @disk_only_regex
       cmd << "-disk-only"
       cmd << @disk_only
@@ -158,6 +167,7 @@ EOS
 
     plot_disk(meta)
     plot_cpu(meta)
+    plot_mem(meta)
 
     FileUtils.rm_rf(@tmpdir)
 
@@ -502,6 +512,139 @@ EOS
     end
   end # def
 
+  def plot_mem(meta)
+    pdf_filename = @output_prefix + 'mem.pdf'
+    gp_filename  = @output_prefix + 'mem.gp'
+    dat_filename = @output_prefix + 'mem.dat'
+
+    if @output_type != 'pdf'
+      img_filename = @output_prefix + 'cpu.' + @output_type
+    else
+      img_filename = nil
+    end
+
+    start_time = meta["start_time"]
+    end_time = meta["end_time"]
+
+    mem_scaling = 2.0**20 # KB -> GB
+
+# "elapsed_time",     // 1
+# "mem_total",        // 2
+# "mem_used",         // 3
+# "mem_free",         // 4
+# "buffers",          // 5
+# "cached",           // 6
+# "swap_cached",      // 7
+# "active",           // 8
+# "inactive",         // 9
+# "swap_total",       // 10
+# "swap_free",        // 11
+# "dirty",            // 12
+# "writeback",        // 13
+# "anon_pages",       // 14
+# "mapped",           // 15
+# "shmem",            // 16
+# "slab",             // 17
+# "s_reclaimable",    // 18
+# "s_unreclaim",      // 19
+# "kernel_stack",     // 20
+# "page_tables",      // 21
+# "nfs_unstable",     // 22
+# "bounce",           // 23
+# "commit_limit",     // 24
+# "committed_as",     // 25
+# "anon_huge_pages",  // 26
+# "huge_pages_total", // 27
+# "huge_pages_free",  // 28
+# "huge_pages_rsvd",  // 29
+# "huge_pages_surp"}, // 30
+# "hugepagesize"},    // 31
+
+    Dir.chdir(@tmpdir) do
+      total = `tail -n+2 #{dat_filename}|head -n1`.split[1].to_f
+      if total == 0.0
+        raise RuntimeError.new("Failed to get MemTotal value from mem.dat file: #{dat_filename}")
+      end
+
+      gpfile = File.open(gp_filename, 'w')
+
+      pdf_file = File.join(@output_dir, "mem.pdf")
+      gpfile.puts <<EOS
+set term pdfcairo enhanced color size 6in,2.5in
+set title "Memory usage"
+set output "#{pdf_filename}"
+set key outside center bottom horizontal
+set size 1.0, 1.0
+
+set xlabel "elapsed time [sec]"
+set ylabel "memory usage [GB]"
+
+# scaling
+s = #{mem_scaling}
+
+set grid
+set xrange [#{@offset_time}:#{end_time - start_time}]
+set yrange [0:#{total * 1.2}/s]
+
+# line styles
+set style line 1 lt 1 lc rgb '#66C2A5' # teal
+set style line 2 lt 1 lc rgb '#FC8D62' # orange
+set style line 3 lt 1 lc rgb '#8DA0CB' # lilac
+set style line 4 lt 1 lc rgb '#E78AC3' # magentat
+set style line 5 lt 1 lc rgb '#A6D854' # lime green
+set style line 6 lt 1 lc rgb '#FFD92F' # banana
+set style line 7 lt 1 lc rgb '#E5C494' # tan
+set style line 8 lt 1 lc rgb '#B3B3B3' # grey
+
+# palette
+set palette maxcolors 8
+set palette defined ( 0 '#66C2A5',\
+    	    	      1 '#FC8D62',\
+		      2 '#8DA0CB',\
+		      3 '#E78AC3',\
+		      4 '#A6D854',\
+		      5 '#FFD92F',\
+		      6 '#E5C494',\
+		      7 '#B3B3B3' )
+
+used(total, free, cached, buffers, srecl) = \\
+  ( (total - free - cache(cached, srecl) - buffers < 0) ? \\
+    (total - free) : \\
+    (total - free - cache(cached, srecl) - buffers) )
+
+cache(cached, srecl) = cached + srecl
+
+plot "#{dat_filename}" usi 1:($4/s+$5/s+cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 8 ti "free", \\
+     "#{dat_filename}" usi 1:($5/s+cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 3 ti "buffers", \\
+     "#{dat_filename}" usi 1:(cache($6, $18)/s+$16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 5 ti "cached", \\
+     "#{dat_filename}" usi 1:($16/s+used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 6 ti "shared", \\
+     "#{dat_filename}" usi 1:(used($2, $4, $6, $5, $18)/s) wi filledcurves x1 ls 2 ti "used", \\
+     "#{dat_filename}" usi 1:(($27-$28)*$31/s) wi lines dt (4,4) lc rgb 'black' lw 2.5 ti 'used (hugepage)'
+
+EOS
+
+      gpfile.close
+      system("gnuplot #{gpfile.path}")
+
+      if @output_type != 'pdf'
+        system("convert -density 150 -background white #{pdf_filename} #{img_filename}")
+      end
+    end # chdir
+
+    copy_targets = []
+    copy_targets << pdf_filename
+    copy_targets << img_filename if img_filename
+
+    if @save_gpfiles
+      copy_targets << gp_filename
+      copy_targets << dat_filename
+    end
+
+    copy_targets.each do |target|
+      FileUtils.copy(File.join(@tmpdir, target), @output_dir)
+    end
+  end # def
+
   private
   def factors(n)
     (2..([n, n / 2].max).to_i).select do |x|

data/lib/perfmonger/command/record.rb

@@ -31,7 +31,8 @@ class RecordCommand < BaseCommand
     if @option.kill
       unless session_pid
         # There is nothing to be killed
-        return true
+        $stderr.puts("[ERROR] No perfmonger record session is running.")
+        return false
       end
 
       begin
@@ -44,12 +45,32 @@ class RecordCommand < BaseCommand
           f.flock(File::LOCK_UN)
         end
       end
+
+      # wait until the process surely exits
+      sleeptime = 0.05
+      try = 0
+      while true
+        begin
+          Process.kill(:INT, session_pid)
+        rescue Errno::ESRCH
+          # no such process
+          break
+        end
+        sleep(sleeptime)
+        sleeptime *= 2
+        try += 1
+        if try >= 5
+          $stderr.puts("[ERROR] Cannot stop perfmonger record session correctly. PID=#{session_pid}")
+          return false
+        end
+      end
+
       return true
     end
 
     if @option.status
       unless session_pid
-        puts "[ERROR] No perfmonger-recorder is running."
+        puts "[ERROR] No perfmonger record session is running."
         return false
       end
 

data/lib/perfmonger/command/record_option.rb

@@ -63,6 +63,12 @@ class RecordOption
     if @no_intr
       cmd << "-no-intr"
     end
+    if @no_net
+      cmd << "-no-net"
+    end
+    if @no_mem
+      cmd << "-no-mem"
+    end
     if @devices.size > 0
       cmd << "-disks"
       cmd << @devices.join(",")
@@ -98,6 +104,8 @@ class RecordOption
     @no_cpu            = false
     @no_disk           = false
     @no_intr           = true
+    @no_net            = true
+    @no_mem            = false
     @devices           = []
     @logfile           = "perfmonger.pgr"
     @logfile_set       = false
@@ -141,6 +149,14 @@ class RecordOption
       @no_cpu = true
     end
 
+    @parser.on('--no-net', 'Suppress recording network usage') do
+      @no_net = false
+    end
+
+    @parser.on('--no-mem', 'Suppress recording memory usage') do
+      @no_mem = true
+    end
+
     @parser.on('-l', '--logfile FILE') do |file|
       @logfile = file
       @logfile_set = true

data/lib/perfmonger/version.rb

@@ -1,3 +1,3 @@
 module PerfMonger
-  VERSION = "0.11.2"
+  VERSION = "0.14.0"
 end

data/misc/werker-box/Dockerfile

@@ -1,10 +1,11 @@
-FROM golang:1.8
+FROM golang:1.14
 
 WORKDIR /app
 
 ## install packages
 RUN apt-get update
 RUN apt-get install -y build-essential libncurses-dev libreadline-dev libssl-dev gnuplot git gnupg2
+RUN apt-get install -y autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm-dev gawk libsqlite3-dev libtool sqlite3 libgmp-dev # ruby build dep
 
 ## get source code
 RUN git clone https://github.com/hayamiz/perfmonger .
@@ -18,16 +19,16 @@ RUN bash rvm-installer stable
 RUN ln -sf /bin/bash /bin/sh
 
 ## install ruby
-RUN bash -l -c "rvm install 2.2.10"
-RUN bash -l -c "rvm use 2.2.10 && gem install bundler && bundle"
+RUN bash -l -c "rvm install 2.4.9"
+RUN bash -l -c "rvm use 2.4.9 && gem install bundler && bundle"
 
-RUN bash -l -c "rvm install 2.3.8"
-RUN bash -l -c "rvm use 2.3.8 && gem install bundler && bundle"
+RUN bash -l -c "rvm install 2.5.7"
+RUN bash -l -c "rvm use 2.5.7 && gem install bundler && bundle"
 
-RUN bash -l -c "rvm install 2.4.5"
-RUN bash -l -c "rvm use 2.4.5 && gem install bundler && bundle"
+RUN bash -l -c "rvm install 2.6.5"
+RUN bash -l -c "rvm use 2.6.5 && gem install bundler && bundle"
 
-RUN bash -l -c "rvm install 2.5.3"
-RUN bash -l -c "rvm use 2.5.3 && gem install bundler && bundle"
+RUN bash -l -c "rvm install 2.7.0"
+RUN bash -l -c "rvm use 2.7.0 && gem install bundler && bundle"
 
 CMD true

data/misc/werker-box/build-push.sh

@@ -3,5 +3,5 @@
 set -e
 
 docker build -t go-rvm .
-docker tag go-rvm hayamiz/go-rvm:wercker-env
-docker push hayamiz/go-rvm:wercker-env
+docker tag go-rvm hayamiz/go-rvm:wercker-env-0.11.2
+docker push hayamiz/go-rvm:wercker-env-0.11.2

data/wercker.yml

@@ -1,4 +1,4 @@
-box: hayamiz/go-rvm:wercker-env
+box: hayamiz/go-rvm:wercker-env-0.11.2
 # Build definition
 build:
     # The steps that will be executed on build
@@ -9,14 +9,14 @@ build:
             name: prepare perfmonger go subsystem
             code: |
               source /etc/profile.d/rvm.sh
-              rvm use 2.2.10
+              rvm use 2.4.9
               rake go_get
 
         - script:
-            name: switch ruby to 2.2.10
+            name: switch ruby to 2.4.9
             code: |
               source /etc/profile.d/rvm.sh
-              rvm use 2.2.10
+              rvm use 2.4.9
         - bundle-install
         - script:
             name: run rspec
@@ -25,10 +25,10 @@ build:
               bundle exec rake spec
 
         - script:
-            name: switch ruby to 2.3.8
+            name: switch ruby to 2.5.7
             code: |
               source /etc/profile.d/rvm.sh
-              rvm use 2.3.8
+              rvm use 2.5.7
         - bundle-install
         - script:
             name: run rspec
@@ -37,10 +37,10 @@ build:
               bundle exec rake spec
 
         - script:
-            name: switch ruby to 2.4.5
+            name: switch ruby to 2.6.5
             code: |
               source /etc/profile.d/rvm.sh
-              rvm use 2.4.5
+              rvm use 2.6.5
         - bundle-install
         - script:
             name: run rspec
@@ -49,10 +49,10 @@ build:
               bundle exec rake spec
 
         - script:
-            name: switch ruby to 2.5.3
+            name: switch ruby to 2.7.0
             code: |
               source /etc/profile.d/rvm.sh
-              rvm use 2.5.3
+              rvm use 2.7.0
         - bundle-install
         - script:
             name: run rspec