perfmonger 0.8.2 → 0.9.0

data/lib/perfmonger/command/fingerprint.rb

@@ -69,6 +69,18 @@ EOS
         save_disk_info()
       end
 
+      do_with_message("Saving fdisk info") do
+        save_fdisk_info()
+      end
+
+      do_with_message("Saving lsblk info") do
+        save_lsblk_info()
+      end
+
+      do_with_message("Saving LVM info") do
+        save_lvm_info()
+      end
+
       do_with_message("Saving PCI/PCIe info") do
         save_pci_info()
       end
@@ -110,6 +122,8 @@ EOS
 
       FileUtils.mv(tmptar_path, @output_tarball)
     end
+
+    true
   end
 
   private
@@ -210,6 +224,40 @@ EOS
     end
   end
 
+  def save_fdisk_info()
+    fdisk_bin = find_executable("fdisk")
+
+    File.open("#{@output_dir}/fdisk.log", "w") do |f|
+      f.puts(`#{fdisk_bin} -l`)
+    end
+  end
+
+  def save_lsblk_info()
+    lsblk_bin = find_executable("lsblk")
+
+    File.open("#{@output_dir}/lsblk.log", "w") do |f|
+      f.puts(`#{lsblk_bin} -t`)
+    end
+  end
+
+  def save_lvm_info()
+    vgdisplay_bin = find_executable("vgdisplay")
+    lvdisplay_bin = find_executable("lvdisplay")
+    pvdisplay_bin = find_executable("pvdisplay")
+
+    File.open("#{@output_dir}/lvm-vgdisplay.log", "w") do |f|
+      f.puts(`#{vgdisplay_bin}`)
+    end
+
+    File.open("#{@output_dir}/lvm-lvdisplay.log", "w") do |f|
+      f.puts(`#{lvdisplay_bin}`)
+    end
+
+    File.open("#{@output_dir}/lvm-pvdisplay.log", "w") do |f|
+      f.puts(`#{pvdisplay_bin}`)
+    end
+  end
+
   def save_irq_info()
     File.open("#{@output_dir}/irq-smp-affinity.log", "w") do |f|
       Dir.glob('/proc/irq/*/smp_affinity').sort_by do |path|

data/lib/perfmonger/command/init-shell.rb

@@ -0,0 +1,51 @@
+
+require 'optparse'
+require 'json'
+require 'tempfile'
+require 'tmpdir'
+
+module PerfMonger
+module Command
+
+class InitShellCommand < BaseCommand
+  register_command 'init-shell', "Generate shell script to init shell completion"
+
+  def run(argv)
+    gem_dir = File.expand_path("../../../", __dir__)
+
+    shell = `ps -p #{Process.ppid()} -o 'args='`.strip
+    shell = File.basename(shell.split.first)
+
+    case shell
+    when "zsh"
+      if argv.first == "-"
+        puts <<EOS
+source #{File.expand_path("misc/perfmonger.zsh", gem_dir)}
+EOS
+      else
+        puts <<EOS
+# Add a following line to ~/.zshrc
+
+eval "$(perfmonger init-shell -)"
+EOS
+      end
+    when "bash"
+      if argv.first == "-"
+        puts <<EOS
+source #{File.expand_path("misc/perfmonger.bash", gem_dir)}
+EOS
+      else
+        puts <<EOS
+# Add a following line to ~/.bashrc
+
+eval "$(perfmonger init-shell -)"
+EOS
+      end
+    end
+
+    true
+  end
+end
+
+end # module Command
+end # module PerfMonger

data/lib/perfmonger/command/plot.rb

@@ -25,6 +25,9 @@ EOS
     @output_prefix = ''
     @save_gpfiles = false
     @disk_only_regex = nil
+    @disk_plot_read = true
+    @disk_plot_write = true
+    @disk_numkey_threshold = 10
   end
 
   def parse_args(argv)
@@ -77,6 +80,25 @@ EOS
       @disk_only_regex = Regexp.compile(regex)
     end
 
+    @parser.on('--disk-read-only', "Plot only READ performance for disks") do
+      @disk_plot_read = true
+      @disk_plot_write = false
+    end
+
+    @parser.on('--disk-write-only', "Plot only WRITE performance for disks") do
+      @disk_plot_read = false
+      @disk_plot_write = true
+    end
+
+    @parser.on('--disk-read-write', "Plot READ and WRITE performance for disks") do
+      @disk_plot_read = true
+      @disk_plot_write = true
+    end
+
+    @parser.on('--disk-numkey-threshold NUM', "Legends of per-disk plots are turned off if the number of disks is larger than this value.") do |num|
+      @disk_numkey_threshold = num.to_i
+    end
+
     @parser.parse!(argv)
 
     if argv.size == 0
@@ -104,87 +126,133 @@ EOS
       exit(false)
     end
 
-    player_bin = ::PerfMonger::Command::CoreFinder.player()
+    formatter_bin = ::PerfMonger::Command::CoreFinder.plot_formatter()
+
+    @tmpdir = Dir.mktmpdir
+
+    @disk_dat = File.expand_path("disk.dat", @tmpdir)
+    @cpu_dat = File.expand_path("cpu.dat", @tmpdir)
 
-    tmpfile = Tempfile.new("jsondata")
-    IO.popen([player_bin, @data_file], "r").each_line do |line|
-      tmpfile.print(line)
+    meta_json = nil
+    IO.popen([formatter_bin, "-perfmonger", @data_file, "-cpufile", @cpu_dat, "-diskfile", @disk_dat], "r") do |io|
+      meta_json = io.read
     end
-    tmpfile.flush()
+    if $?.exitstatus != 0
+      puts("ERROR: failed to run perfmonger-plot-formatter")
+      exit(false)
+    end
+    meta = JSON.parse(meta_json)
+
+    plot_disk(meta)
+    plot_cpu(meta)
 
-    plot_ioinfo(tmpfile.path)
-    plot_cpuinfo(tmpfile.path)
+    true
   end
 
   private
-  def plot_ioinfo(json_file)
-    iops_pdf_filename = @output_prefix + 'iops.pdf'
-    transfer_pdf_filename = @output_prefix + 'transfer.pdf'
-    gp_filename  = @output_prefix + 'io.gp'
-    dat_filename = @output_prefix + 'io.dat'
+  def plot_disk(meta)
+    iops_pdf_filename = @output_prefix + 'disk-iops.pdf'
+    transfer_pdf_filename = @output_prefix + 'disk-transfer.pdf'
+    total_iops_pdf_filename = @output_prefix + 'disk-total-iops.pdf'
+    total_transfer_pdf_filename = @output_prefix + 'disk-total-transfer.pdf'
+    gp_filename  = @output_prefix + 'disk.gp'
+    dat_filename = @output_prefix + 'disk.dat'
     if @output_type != 'pdf'
-      iops_img_filename = @output_prefix + 'iops.' + @output_type
-      transfer_img_filename = @output_prefix + 'transfer.' + @output_type
+      iops_img_filename = @output_prefix + 'disk-iops.' + @output_type
+      transfer_img_filename = @output_prefix + 'disk-transfer.' + @output_type
+      total_iops_img_filename = @output_prefix + 'disk-total-iops.' + @output_type
+      total_transfer_img_filename = @output_prefix + 'disk-total-transfer.' + @output_type
     else
       iops_img_filename = nil
       transfer_img_filename = nil
+      total_iops_img_filename = nil
+      total_transfer_img_filename = nil
     end
 
-    Dir.mktmpdir do |working_dir|
-      Dir.chdir(working_dir) do
-        datafile = File.open(dat_filename, 'w')
-        gpfile = File.new(gp_filename, 'w')
+    start_time = meta["start_time"]
+    end_time = meta["end_time"]
 
-        start_time = nil
-        devices = nil
+    Dir.chdir(@tmpdir) do
+      gpfile = File.new(gp_filename, 'w')
 
-        File.open(json_file).each_line do |line|
-          record = JSON.parse(line)
-          time = record["time"]
-          diskinfo = record["disk"]
-          return unless diskinfo
+      total_iops_plot_stmt_list = []
+      iops_plot_stmt_list = meta["disk"]["devices"].map do |dev_entry|
+        devname = dev_entry["name"]
+        idx = dev_entry["idx"]
+
+        if devname == "total"
+          if @disk_plot_read
+            total_iops_plot_stmt_list.push("\"disk.dat\" ind #{idx} usi 1:2 with lines lw 2 title \"#{devname} read\"")
+          end
+          if @disk_plot_write
+            total_iops_plot_stmt_list.push("\"disk.dat\" ind #{idx} usi 1:3 with lines lw 2 title \"#{devname} write\"")
+          end
 
-          start_time ||= time
-          devices ||= diskinfo["devices"]
+          []
+        elsif @disk_only_regex && !(devname =~ @disk_only_regex)
+          []
+        else
+          plot_stmt = []
 
-          if @disk_only_regex
-            devices = devices.select do |devname|
-              devname =~ @disk_only_regex
-            end
+          if @disk_plot_read
+            plot_stmt.push("\"disk.dat\" ind #{idx} usi 1:2 with lines lw 2 title \"#{devname} read\"")
+          end
+          if @disk_plot_write
+            plot_stmt.push("\"disk.dat\" ind #{idx} usi 1:3 with lines lw 2 title \"#{devname} write\"")
           end
 
-          datafile.puts([time - start_time,
-                         devices.map{|device|
-                           [diskinfo[device]["riops"], diskinfo[device]["wiops"],
-                            diskinfo[device]["rkbyteps"] * 512 / 1024 / 1024, # in MB/s
-                            diskinfo[device]["wkbyteps"] * 512 / 1024 / 1024, # in MB/s
-                           ]
-                         }].flatten.map(&:to_s).join("\t"))
+          plot_stmt
         end
+      end.flatten
 
-        datafile.close
+      total_transfer_plot_stmt_list = []
+      transfer_plot_stmt_list = meta["disk"]["devices"].map do |dev_entry|
+        devname = dev_entry["name"]
+        idx = dev_entry["idx"]
 
-        col_idx = 2
-        iops_plot_stmt_list = devices.map do |device|
-          plot_stmt = []
-          plot_stmt.push("\"#{dat_filename}\" usi 1:#{col_idx} with lines lw 2 title \"#{device} read\"")
-          plot_stmt.push("\"#{dat_filename}\" usi 1:#{col_idx + 1} with lines lw 2 title \"#{device} write\"")
-          col_idx += 4
-          plot_stmt
-        end.flatten
+        if devname == "total"
+          if @disk_plot_read
+            total_transfer_plot_stmt_list.push("\"disk.dat\" ind #{idx} usi 1:4 with lines lw 2 title \"#{devname} read\"")
+          end
+          if @disk_plot_write
+            total_transfer_plot_stmt_list.push("\"disk.dat\" ind #{idx} usi 1:5 with lines lw 2 title \"#{devname} write\"")
+          end
 
-        col_idx = 4
-        transfer_plot_stmt_list = devices.map do |device|
+          []
+        elsif @disk_only_regex && !(devname =~ @disk_only_regex)
+          []
+        else
           plot_stmt = []
-          plot_stmt.push("\"#{dat_filename}\" usi 1:#{col_idx} with lines lw 2 title \"#{device} read\"")
-          plot_stmt.push("\"#{dat_filename}\" usi 1:#{col_idx + 1} with lines lw 2 title \"#{device} write\"")
-          col_idx += 4
+
+          if @disk_plot_read
+            plot_stmt.push("\"disk.dat\" ind #{idx} usi 1:4 with lines lw 2 title \"#{devname} read\"")
+          end
+          if @disk_plot_write
+            plot_stmt.push("\"disk.dat\" ind #{idx} usi 1:5 with lines lw 2 title \"#{devname} write\"")
+          end
+
           plot_stmt
-        end.flatten
+        end
+      end.flatten
+
+      if iops_plot_stmt_list.size == 0
+        puts("No plot target disk devices.")
+        return
+      end
 
-        gpfile.puts <<EOS
+      num_dev = meta["disk"]["devices"].select do |dev_entry|
+        dev_entry["name"] != "total"
+      end.size
+
+      if num_dev > @disk_numkey_threshold
+        set_key_stmt = "unset key"
+      else
+        set_key_stmt = "set key below center"
+      end
+
+      gpfile.puts <<EOS
 set term pdfcairo enhanced color
-set title "IOPS: #{File.basename(@data_file)}"
+set title "IOPS"
 set size 1.0, 1.0
 set output "#{iops_pdf_filename}"
 
@@ -192,54 +260,67 @@ set xlabel "elapsed time [sec]"
 set ylabel "IOPS"
 
 set grid
-set xrange [#{@offset_time}:*]
+set xrange [#{@offset_time}:#{end_time - start_time}]
 set yrange [0:*]
 
-set key below center
-
+#{set_key_stmt}
 plot #{iops_plot_stmt_list.join(",\\\n     ")}
 
+set title "Total IOPS"
+unset key
+set output "#{total_iops_pdf_filename}"
+plot #{total_iops_plot_stmt_list.join(",\\\n     ")}
+
 
-set title "Transfer rate: #{File.basename(@data_file)}"
+set title "Transfer rate"
 set output "#{transfer_pdf_filename}"
 set ylabel "transfer rate [MB/s]"
+#{set_key_stmt}
 plot #{transfer_plot_stmt_list.join(",\\\n     ")}
-EOS
 
-        gpfile.close
+set title "Total transfer rate"
+set output "#{total_transfer_pdf_filename}"
+unset key
+plot #{total_transfer_plot_stmt_list.join(",\\\n     ")}
+EOS
+      gpfile.close
 
-        system("gnuplot #{gpfile.path}")
+      system("gnuplot #{gpfile.path}")
 
-        if @output_type != 'pdf'
-          system("convert -density 150 -background white #{iops_pdf_filename} #{iops_img_filename}")
-          system("convert -density 150 -background white #{transfer_pdf_filename} #{transfer_img_filename}")
-        end
+      if @output_type != 'pdf'
+        system("convert -density 150 -background white #{iops_pdf_filename} #{iops_img_filename}")
+        system("convert -density 150 -background white #{transfer_pdf_filename} #{transfer_img_filename}")
+        system("convert -density 150 -background white #{total_iops_pdf_filename} #{total_iops_img_filename}")
+        system("convert -density 150 -background white #{total_transfer_pdf_filename} #{total_transfer_img_filename}")
+      end
 
-      end # chdir
+    end # chdir
 
-      copy_targets = [iops_pdf_filename, transfer_pdf_filename]
-      copy_targets.push(iops_img_filename) if iops_img_filename
-      copy_targets.push(transfer_img_filename) if transfer_img_filename
+    copy_targets = []
+    copy_targets += [iops_pdf_filename, transfer_pdf_filename]
+    copy_targets += [total_iops_pdf_filename, total_transfer_pdf_filename]
+    copy_targets.push(iops_img_filename) if iops_img_filename
+    copy_targets.push(transfer_img_filename) if transfer_img_filename
+    copy_targets.push(total_iops_img_filename) if total_iops_img_filename
+    copy_targets.push(total_transfer_img_filename) if total_transfer_img_filename
 
-      if @save_gpfiles
-        copy_targets.push(dat_filename)
-        copy_targets.push(gp_filename)
-      end
+    if @save_gpfiles
+      copy_targets.push(dat_filename)
+      copy_targets.push(gp_filename)
+    end
 
-      copy_targets.each do |target|
-        FileUtils.copy(File.join(working_dir, target), @output_dir)
-      end
-    end # mktempdir
+    copy_targets.each do |target|
+      FileUtils.copy(File.join(@tmpdir, target), @output_dir)
+    end
   end # def
 
-  def plot_cpuinfo(json_file)
+  def plot_cpu(meta)
     pdf_filename = @output_prefix + 'cpu.pdf'
     gp_filename  = @output_prefix + 'cpu.gp'
     dat_filename = @output_prefix + 'cpu.dat'
 
     all_pdf_filename = @output_prefix + 'allcpu.pdf'
     all_gp_filename  = @output_prefix + 'allcpu.gp'
-    all_dat_filename = @output_prefix + 'allcpu.dat'
 
     if @output_type != 'pdf'
       img_filename = @output_prefix + 'cpu.' + @output_type
@@ -249,52 +330,27 @@ EOS
       all_img_filename = nil
     end
 
-    Dir.mktmpdir do |working_dir|
-      Dir.chdir(working_dir) do
-        datafile = File.open(dat_filename, 'w')
-        gpfile = File.open(gp_filename, 'w')
-        all_datafile = File.open(all_dat_filename, 'w')
-        all_gpfile = File.open(all_gp_filename, 'w')
-
-        start_time = nil
-        end_time = 0
-        devices = nil
-        nr_cpu = nil
-
-        File.open(json_file).each_line do |line|
-          record = JSON.parse(line)
-
-          time = record["time"]
-          cpuinfo = record["cpu"]
-          return unless cpuinfo
-          nr_cpu = cpuinfo['num_core']
-
-          cores = cpuinfo['cores']
+    start_time = meta["start_time"]
+    end_time = meta["end_time"]
 
-          start_time ||= time
-          end_time = [end_time, time].max
+    Dir.chdir(@tmpdir) do
+      gpfile = File.open(gp_filename, 'w')
+      all_gpfile = File.open(all_gp_filename, 'w')
 
-          datafile.puts([time - start_time,
-                         %w|usr nice sys iowait hardirq softirq steal guest idle|.map do |key|
-                           cores.map{|core| core[key]}.inject(&:+)
-                         end].flatten.map(&:to_s).join("\t"))
-        end
-        datafile.close
+      devices = nil
+      nr_cpu = meta["cpu"]["num_core"]
 
-        col_idx = 2
-        columns = []
-        plot_stmt_list = []
-        %w|%usr %nice %sys %iowait %hardirq %softirq %steal %guest|.each do |key|
-          columns << col_idx
-          plot_stmt = "\"#{datafile.path}\" usi 1:(#{columns.map{|i| "$#{i}"}.join("+")}) with filledcurve x1 lw 0 lc #{col_idx - 1} title \"#{key}\""
-          plot_stmt_list << plot_stmt
-          col_idx += 1
-        end
+      plot_stmt_list = []
+      %w|%usr %nice %sys %iowait %hardirq %softirq %steal %guest|.each_with_index do |key, idx|
+        stack_columns = (0..idx).to_a.map{|x| x + 2}
+        plot_stmt = "\"cpu.dat\" ind 0 usi 1:(#{stack_columns.map{|i| "$#{i}"}.join("+")}) with filledcurve x1 lw 0 lc #{idx+1} title \"#{key}\""
+        plot_stmt_list << plot_stmt
+      end
 
-        pdf_file = File.join(@output_dir, "cpu.pdf")
-        gpfile.puts <<EOS
+      pdf_file = File.join(@output_dir, "cpu.pdf")
+      gpfile.puts <<EOS
 set term pdfcairo enhanced color
-set title "CPU usage: #{File.basename(@data_file)} (max: #{nr_cpu*100}%)"
+set title "CPU usage (max: #{nr_cpu*100}%)"
 set output "#{pdf_filename}"
 set key outside center bottom horizontal
 set size 1.0, 1.0
@@ -309,24 +365,30 @@ set yrange [0:*]
 plot #{plot_stmt_list.reverse.join(",\\\n     ")}
 EOS
 
-        gpfile.close
-        system("gnuplot #{gpfile.path}")
+      gpfile.close
+      system("gnuplot #{gpfile.path}")
 
-        if @output_type != 'pdf'
-          system("convert -density 150 -background white #{pdf_filename} #{img_filename}")
-        end
+      if @output_type != 'pdf'
+        system("convert -density 150 -background white #{pdf_filename} #{img_filename}")
+      end
 
-        ## Plot all CPUs in a single file
+      ## Plot all CPUs in a single file
 
-        nr_cpu_factors = factors(nr_cpu)
-        nr_cols = nr_cpu_factors.select do |x|
-          x <= Math.sqrt(nr_cpu)
-        end.max
-        nr_cols ||= Math.sqrt(nr_cpu).ceil
-        nr_rows = nr_cpu / nr_cols
+      nr_cpu_factors = factors(nr_cpu)
+      nr_cols = nr_cpu_factors.select do |x|
+        x <= Math.sqrt(nr_cpu)
+      end.max
+      nr_cols ||= Math.sqrt(nr_cpu).ceil
+      nr_rows = nr_cpu / nr_cols
 
-        all_gpfile.puts <<EOS
-set term pdfcairo color enhanced size 8.5inch, 11inch
+      plot_height = 8
+
+      if nr_rows == 1
+        plot_height /= 2.0
+      end
+
+      all_gpfile.puts <<EOS
+set term pdfcairo color enhanced size 8.5inch, #{plot_height}inch
 set output "#{all_pdf_filename}"
 set size 1.0, 1.0
 set multiplot
@@ -336,28 +398,19 @@ set yrange [0:101]
 
 EOS
 
-        legend_height = 0.04
-        nr_cpu.times do |cpu_idx|
-          all_datafile.puts("# cpu #{cpu_idx}")
-          File.open(json_file).each_line do |line|
-            record = JSON.parse(line)
-            time = record["time"]
-            cpurec = record["cpu"]["cores"][cpu_idx]
-            all_datafile.puts([time - start_time,
-                               cpurec["usr"] + cpurec["nice"],
-                               cpurec["sys"],
-                               cpurec["hardirq"],
-                               cpurec["softirq"],
-                               cpurec["steal"] + cpurec["guest"],
-                               cpurec["iowait"]].map(&:to_s).join("\t"))
-          end
-          all_datafile.puts("")
-          all_datafile.puts("")
+      legend_height = 0.04
+      nr_cpu.times do |cpu_idx|
+        xpos = (1.0 / nr_cols) * (cpu_idx % nr_cols)
+        ypos = ((1.0 - legend_height) / nr_rows) * (nr_rows - 1 - (cpu_idx / nr_cols).to_i) + legend_height
 
-          xpos = (1.0 / nr_cols) * (cpu_idx % nr_cols)
-          ypos = ((1.0 - legend_height) / nr_rows) * (nr_rows - 1 - (cpu_idx / nr_cols).to_i) + legend_height
+        plot_stmt_list = []
+        %w|%usr %nice %sys %iowait %hardirq %softirq %steal %guest|.each_with_index do |key, idx|
+          stack_columns = (0..idx).to_a.map{|x| x + 2}
+          plot_stmt = "\"cpu.dat\" ind #{cpu_idx+1} usi 1:(#{stack_columns.map{|i| "$#{i}"}.join("+")}) with filledcurve x1 lw 0 lc #{idx+1} title \"#{key}\""
+          plot_stmt_list << plot_stmt
+        end
 
-          all_gpfile.puts <<EOS
+        all_gpfile.puts <<EOS
 set title 'cpu #{cpu_idx}' offset 0.0,-0.7 font 'Arial,16'
 unset key
 set origin #{xpos}, #{ypos}
@@ -369,20 +422,20 @@ set bmargin 1.3
 set xtics offset 0.0,0.5
 set ytics offset 0.5,0
 set style fill noborder
-plot '#{all_datafile.path}' index #{cpu_idx} using 1:($2+$3+$4+$5+$6+$7) with filledcurve x1 lw 0 lc 6 title '%iowait', \\
-     '#{all_datafile.path}' index #{cpu_idx} using 1:($2+$3+$4+$5+$6) with filledcurve x1 lw 0 lc 5 title '%other', \\
-     '#{all_datafile.path}' index #{cpu_idx} using 1:($2+$3+$4+$5) with filledcurve x1 lw 0 lc 4 title '%soft', \\
-     '#{all_datafile.path}' index #{cpu_idx} using 1:($2+$3+$4) with filledcurve x1 lw 0 lc 3 title '%irq', \\
-     '#{all_datafile.path}' index #{cpu_idx} using 1:($2+$3) with filledcurve x1 lw 0 lc 2 title '%sys', \\
-     '#{all_datafile.path}' index #{cpu_idx} using 1:2 with filledcurve x1 lw 0 lc 1 title '%usr'
+plot #{plot_stmt_list.reverse.join(",\\\n     ")}
 
 EOS
+      end # times
 
-        end
-
-        all_gpfile.puts <<EOS
+      # plot legends
+      plot_stmt_list = []
+      %w|%usr %nice %sys %iowait %hardirq %softirq %steal %guest|.each_with_index do |key, idx|
+        plot_stmt = "-1 with filledcurve x1 lw 0 lc #{idx+1} title \"#{key}\""
+        plot_stmt_list << plot_stmt
+      end
+      all_gpfile.puts <<EOS
 unset title
-set key center center horizontal font "Arial,16"
+set key center center horizontal font "Arial,14"
 set origin 0.0, 0.0
 set size 1.0, #{legend_height}
 set rmargin 0
@@ -394,45 +447,38 @@ set border 0
 set yrange [0:1]
 # plot -1 with filledcurve x1 title '%usr'
 
-plot -1 with filledcurve x1 lw 0 lc 1 title '%usr', \\
-     -1 with filledcurve x1 lw 0 lc 2 title '%sys', \\
-     -1 with filledcurve x1 lw 0 lc 3 title '%irq', \\
-     -1 with filledcurve x1 lw 0 lc 4 title '%soft', \\
-     -1 with filledcurve x1 lw 0 lc 5 title '%other', \\
-     -1 with filledcurve x1 lw 0 lc 6 title '%iowait'
+set xlabel "elapsed time [sec]"
+plot #{plot_stmt_list.reverse.join(",\\\n     ")}
+
 EOS
 
-        all_datafile.fsync
-        all_gpfile.fsync
-        all_datafile.close
-        all_gpfile.close
+      all_gpfile.fsync
+      all_gpfile.close
 
-        system("gnuplot #{all_gpfile.path}")
+      system("gnuplot #{all_gpfile.path}")
 
-        if @output_type != 'pdf'
-          system("convert -density 150 -background white #{all_pdf_filename} #{all_img_filename}")
-        end
+      if @output_type != 'pdf'
+        system("convert -density 150 -background white #{all_pdf_filename} #{all_img_filename}")
+      end
 
-      end # chdir
+    end # chdir
 
-      copy_targets = []
+    copy_targets = []
 
-      copy_targets << pdf_filename
-      copy_targets << img_filename if img_filename
-      copy_targets << all_pdf_filename
-      copy_targets << all_img_filename if all_img_filename
+    copy_targets << pdf_filename
+    copy_targets << img_filename if img_filename
+    copy_targets << all_pdf_filename
+    copy_targets << all_img_filename if all_img_filename
 
-      if @save_gpfiles
-        copy_targets << gp_filename
-        copy_targets << dat_filename
-        copy_targets << all_gp_filename
-        copy_targets << all_dat_filename
-      end
+    if @save_gpfiles
+      copy_targets << gp_filename
+      copy_targets << dat_filename
+      copy_targets << all_gp_filename
+    end
 
-      copy_targets.each do |target|
-        FileUtils.copy(File.join(working_dir, target), @output_dir)
-      end
-    end # mktempdir
+    copy_targets.each do |target|
+      FileUtils.copy(File.join(@tmpdir, target), @output_dir)
+    end
   end # def
 
   private