perfmonger 0.6.1

data/lib/perfmonger/command/summary.rb

@@ -0,0 +1,402 @@
+
+require 'optparse'
+require 'json'
+
+module PerfMonger
+module Command
+
+class SummaryCommand < BaseCommand
+  register_command 'summary', "Show a summary of a perfmonger log file"
+
+  def initialize
+    @parser = OptionParser.new
+    @parser.banner = <<EOS
+Usage: perfmonger summary [options] LOG_FILE
+
+Options:
+EOS
+
+    @json = false
+    @pager = nil
+
+    @parser.on('--json', "Output summary in JSON") do
+      @json = true
+    end
+
+    @parser.on('-p', '--pager [PAGER]', "Use pager to see summary output.") do |pager|
+      if pager.nil?
+        if ENV['PAGER'].nil?
+          puts("ERROR: No pager is available.")
+          puts("ERROR: Please set PAGER or give pager name to --pager option.")
+          puts(@parser.help)
+          exit(false)
+        else
+          @pager = ENV['PAGER']
+        end
+      else
+        @pager = pager
+      end
+    end
+  end
+
+  def parse_args(argv)
+    @parser.parse!(argv)
+
+    if argv.size == 0
+      puts("ERROR: PerfMonger log file is required")
+      puts(@parser.help)
+      exit(false)
+    end
+
+    @logfile = argv.shift
+    if ! File.exists?(@logfile)
+      puts("ERROR: No such file: #{@logfile}")
+      puts(@parser.help)
+      exit(false)
+    end
+  end
+
+  def run(argv, summary_title = nil)
+    parse_args(argv)
+
+    summary_title ||= @logfile
+
+    show_summary(@logfile, summary_title)
+  end
+
+
+  def read_logfile(logfile)
+    File.read(logfile).lines.map do |line|
+      begin
+        JSON.parse(line)
+      rescue JSON::ParserError => err
+        nil
+      end
+    end.compact
+  end
+
+  def make_accumulation(records)
+    unless records.all?{|record| record.has_key?("ioinfo")}
+      return nil
+    end
+    unless records.size > 1
+      return nil
+    end
+
+    accum = Hash.new
+
+    devices = records.first["ioinfo"]["devices"]
+    accum["ioinfo"] = Hash.new
+
+    devices.each do |device|
+      read_requests = 0
+      read_bytes = 0
+      write_requests = 0
+      write_bytes = 0
+
+      (1..(records.size - 1)).each do |idx|
+        last_record = records[idx - 1]
+        record = records[idx]
+        dt = record["time"] - last_record["time"]
+
+        read_requests += record["ioinfo"][device]["riops"] * dt
+        write_requests += record["ioinfo"][device]["wiops"] * dt
+        read_bytes += record["ioinfo"][device]["rsecps"] * 512 * dt
+        write_bytes += record["ioinfo"][device]["wsecps"] * 512 * dt
+      end
+
+      accum["ioinfo"][device] = Hash.new
+      accum["ioinfo"][device]["read_requests"] = read_requests
+      accum["ioinfo"][device]["read_bytes"] = read_bytes
+      accum["ioinfo"][device]["write_requests"] = write_requests
+      accum["ioinfo"][device]["write_bytes"] = write_bytes
+    end
+
+    accum
+  end
+
+  def make_summary(records)
+    if records.empty?
+      return nil
+    elsif records.size == 1
+      return records.first
+    end
+
+    # setup getters and setters all attributes for avg. calculation
+    # getter.call(record) returns value
+    # setter.call(record, value) set value
+    # getters and setters include attribute info as a closure
+    getters = []
+    setters = []
+
+    if records.first.include?("ioinfo")
+      records.first["ioinfo"]["devices"].each do |device|
+        records.first["ioinfo"][device].keys.each do |attr|
+          getters << lambda do |record|
+            record["ioinfo"][device][attr]
+          end
+          setters << lambda do |record, value|
+            record["ioinfo"][device][attr] = value
+          end
+        end
+      end
+
+      records.first["ioinfo"]["total"].keys.each do |attr|
+        getters << lambda do |record|
+          record["ioinfo"]["total"][attr]
+        end
+        setters << lambda do |record, value|
+          record["ioinfo"]["total"][attr] = value
+        end
+      end
+    end
+
+    if records.first.include?("cpuinfo")
+      records.first["cpuinfo"]["all"].keys.each do |attr|
+        getters << lambda do |record|
+          record["cpuinfo"]["all"][attr]
+        end
+        setters << lambda do |record, value|
+          record["cpuinfo"]["all"][attr] = value
+        end
+      end
+
+      records.first["cpuinfo"]["nr_cpu"].times do |cpu_idx|
+        records.first["cpuinfo"]["cpus"][cpu_idx].keys.each do |attr|
+          getters << lambda do |record|
+            record["cpuinfo"]["cpus"][cpu_idx][attr]
+          end
+          setters << lambda do |record, value|
+            record["cpuinfo"]["cpus"][cpu_idx][attr] = value
+          end
+        end
+      end
+    end
+
+    avg_record = Marshal.load(Marshal.dump(records.first)); # deep copy
+
+    setters.each do |setter|
+      setter.call(avg_record, 0.0)
+    end
+
+    (1..(records.size - 1)).each do |idx|
+      record = records[idx]
+
+      last_t = records[idx - 1]["time"]
+      t      = record["time"]
+
+      getters.size.times do |_etters_idx|
+        getter = getters[_etters_idx]
+        setter = setters[_etters_idx]
+
+        setter.call(avg_record,
+                    getter.call(avg_record) + getter.call(record) * (t - last_t))
+      end
+    end
+
+    getters.size.times do |_etters_idx|
+      getter = getters[_etters_idx]
+      setter = setters[_etters_idx]
+
+      setter.call(avg_record,
+                  getter.call(avg_record) / (records[-1]["time"] - records[0]["time"]))
+    end
+
+
+    # r_await/w_await need special handling
+    if records.first.include?("ioinfo")
+      records.first["ioinfo"]["devices"].each do |device|
+        accum_r_io_time = 0.0
+        accum_w_io_time = 0.0
+        r_io_count = 0
+        w_io_count = 0
+
+        (records.size - 1).times do |idx|
+          rec0 = records[idx]
+          rec1 = records[idx + 1]
+          dev_ioinfo = rec1["ioinfo"][device]
+          dt = rec1["time"] - rec0["time"]
+
+          accum_r_io_time += dev_ioinfo["r_await"] * dev_ioinfo["riops"] * dt
+          accum_w_io_time += dev_ioinfo["w_await"] * dev_ioinfo["wiops"] * dt
+          r_io_count += dev_ioinfo["riops"] * dt
+          w_io_count += dev_ioinfo["wiops"] * dt
+        end
+
+        if r_io_count > 0
+          avg_record["ioinfo"][device]["r_await"] = accum_r_io_time / r_io_count
+        else
+          avg_record["ioinfo"][device]["r_await"] = 0.0
+        end
+
+        if w_io_count > 0
+          avg_record["ioinfo"][device]["w_await"] = accum_w_io_time / w_io_count
+        else
+          avg_record["ioinfo"][device]["w_await"] = 0.0
+        end
+      end
+    end
+
+    avg_record
+  end
+
+  def show_summary(logfile, summary_title)
+    records = read_logfile(logfile)
+    summary = make_summary(records)
+    accum = make_accumulation(records)
+
+    duration = records.last["time"] - records.first["time"]
+
+    if @json
+      output = Hash.new
+
+      output["duration"] = duration
+      if summary
+        if summary["cpuinfo"]
+          output["cpuinfo"] = summary["cpuinfo"]
+        end
+
+        if summary['ioinfo']
+          output["ioinfo"] = summary["ioinfo"]
+        end
+      end
+
+      puts output.to_json
+    else
+      if $stdout.tty? && @pager
+        output_file = IO.popen(@pager, "w")
+      else
+        output_file = $stdout
+      end
+
+      output_file.puts("")
+      output_file.puts("== Performance summary of '#{summary_title}' ==")
+      output_file.puts("")
+      duration_str = sprintf("%.2f", duration)
+      output_file.puts("record duration: #{duration_str} sec")
+
+      if summary.nil?
+        output_file.puts("")
+        output_file.puts("No performance info was collected.")
+        output_file.puts("This is because command execution time was too short, or something went wrong.")
+      end
+
+      if summary && summary["cpuinfo"]
+        nr_cpu = records.first["cpuinfo"]["nr_cpu"]
+
+        usr, sys, iowait, irq, soft, idle =
+          [summary['cpuinfo']['all']['usr'] + summary['cpuinfo']['all']['nice'],
+           summary['cpuinfo']['all']['sys'],
+           summary['cpuinfo']['all']['iowait'],
+           summary['cpuinfo']['all']['irq'],
+           summary['cpuinfo']['all']['soft'],
+           summary['cpuinfo']['all']['idle']].map do |val|
+          val * nr_cpu
+        end
+
+        other = [100.0 - (usr + sys + iowait + irq + soft + idle), 0.0].max * nr_cpu
+
+        usr_str, sys_str, iowait_str, irq_str, soft_str, other_str, idle_str =
+          [usr, sys, iowait, irq, soft, other, idle].map do |value|
+          sprintf("%.2f", value)
+        end
+
+        total_non_idle_str = sprintf("%.2f", usr + sys + irq + soft + other)
+        total_idle_str = sprintf("%.2f", iowait + idle)
+
+        output_file.puts("")
+        output_file.puts <<EOS
+* Average CPU usage (MAX: #{100 * nr_cpu} %)
+  * Non idle portion: #{total_non_idle_str}
+       %usr: #{usr_str}
+       %sys: #{sys_str}
+       %irq: #{irq_str}
+      %soft: #{soft_str}
+     %other: #{other_str}
+  * Idle portion: #{total_idle_str}
+    %iowait: #{iowait_str}
+      %idle: #{idle_str}
+EOS
+      end
+
+      if summary && summary['ioinfo']
+        total_r_iops, total_w_iops, total_r_sec, total_w_sec = [0.0] * 4
+
+        summary['ioinfo']['devices'].each do |device|
+          r_iops, w_iops, r_sec, w_sec, r_await, w_await =
+            [summary['ioinfo'][device]['riops'],
+             summary['ioinfo'][device]['wiops'],
+             summary['ioinfo'][device]['rsecps'] * 512 / 1024.0 / 1024.0,
+             summary['ioinfo'][device]['wsecps'] * 512 / 1024.0 / 1024.0,
+             summary['ioinfo'][device]['r_await'],
+             summary['ioinfo'][device]['w_await']]
+
+          total_r_iops += r_iops
+          total_w_iops += w_iops
+          total_r_sec  += r_sec
+          total_w_sec  += w_sec
+
+          r_iops_str, w_iops_str, r_sec_str, w_sec_str = [r_iops, w_iops, r_sec, w_sec].map do |value|
+            sprintf("%.2f", value)
+          end
+
+          r_await_str, w_await_str = [r_await, w_await].map do |await|
+            if await < 1.0
+              sprintf("%.1f usec", await * 1000)
+            else
+              sprintf("%.2f msec", await)
+            end
+          end
+
+          total_r_bytes_str, total_w_bytes_str = ["read_bytes", "write_bytes"].map do |key|
+            bytes = accum["ioinfo"][device][key]
+            if bytes > 2**30
+              sprintf("%.2f GB", bytes / 2**30)
+            elsif bytes > 2**20
+              sprintf("%.2f MB", bytes / 2**20)
+            elsif bytes > 2**10
+              sprintf("%.2f KB", bytes / 2**10)
+            else
+              sprintf("%.2f bytes", bytes)
+            end
+          end
+
+
+          output_file.puts("")
+          output_file.puts("* Average DEVICE usage: #{device}")
+          output_file.puts("        read IOPS: #{r_iops_str}")
+          output_file.puts("       write IOPS: #{w_iops_str}")
+          output_file.puts("  read throughput: #{r_sec_str} MB/s")
+          output_file.puts(" write throughput: #{w_sec_str} MB/s")
+          output_file.puts("     read latency: #{r_await_str}")
+          output_file.puts("    write latency: #{w_await_str}")
+          output_file.puts("      read amount: #{total_r_bytes_str}")
+          output_file.puts("     write amount: #{total_w_bytes_str}")
+        end
+
+        if summary['ioinfo']['devices'].size > 1
+          total_r_iops_str, total_w_iops_str, total_r_sec_str, total_w_sec_str =
+            [total_r_iops, total_w_iops, total_r_sec, total_w_sec].map do |value|
+            sprintf("%.2f", value)
+          end
+
+          output_file.puts("")
+          output_file.puts("* TOTAL DEVICE usage: #{summary['ioinfo']['devices'].join(', ')}")
+          output_file.puts("        read IOPS: #{total_r_iops_str}")
+          output_file.puts("       write IOPS: #{total_w_iops_str}")
+          output_file.puts("  read throughput: #{total_r_sec_str} MB/s")
+          output_file.puts(" write throughput: #{total_w_sec_str} MB/s")
+        end
+
+        output_file.puts("")
+      end
+
+      if output_file != $stdout
+        output_file.close
+      end
+    end
+  end
+end
+
+end # module Command
+end # module PerfMonger

data/lib/perfmonger/config.rb

@@ -0,0 +1,6 @@
+# This file is automatically generated by Autoconf: 2014-06-23 19:42:25
+
+module PerfMonger
+  ROOTDIR = File.expand_path("../../..", __FILE__)
+  DATAROOTDIR = File.expand_path("../../../data", __FILE__)
+end

data/lib/perfmonger/version.rb

@@ -0,0 +1,5 @@
+# This file is automatically generated by Autoconf: 2014-06-23 19:42:25
+
+module PerfMonger
+  VERSION = "0.6.1"
+end

data/lib/perfmonger.rb

@@ -0,0 +1,12 @@
+require 'perfmonger/version.rb'
+require 'perfmonger/config.rb'
+require 'perfmonger/cli.rb'
+require 'perfmonger/command/base_command.rb'
+require 'perfmonger/command/record.rb'
+require 'perfmonger/command/record_option.rb'
+require 'perfmonger/command/stat.rb'
+require 'perfmonger/command/stat_option.rb'
+require 'perfmonger/command/summary.rb'
+require 'perfmonger/command/plot.rb'
+require 'perfmonger/command/server.rb'
+require 'perfmonger/command/fingerprint.rb'

data/misc/release-howto.txt

@@ -0,0 +1,17 @@
+リリース手順
+
+1. make check
+2. rake spec
+3. リリースノート書く
+3. rpm生成
+  * chrootでrpm生成
+  * gpgで署名
+4. tarball生成
+5. タグ付け
+6. ファイル公開
+  * yumにアップロード
+  * tarballアップロード
+7. freecode リリース登録
+8. Webサイト更新
+9. 告知エントリ書く
+

data/perfmonger.gemspec

@@ -0,0 +1,44 @@
+# -*- mode: ruby -*-
+
+$:.push File.expand_path("../lib", __FILE__)
+require 'perfmonger'
+
+Gem::Specification.new do |s|
+  s.name        = 'perfmonger'
+  s.version     = PerfMonger::VERSION
+  s.date        = '2015-01-19'
+  s.summary     = "yet anothor performance measurement/monitoring tool"
+  s.description = "yet anothor performance measurement/monitoring tool"
+  s.authors     = ["Yuto HAYAMIZU"]
+  s.email       = 'y.hayamizu@gmail.com'
+  s.homepage    = 'http://github.com/hayamiz/perfmonger/'
+  s.license     = 'GPL-2'
+
+  s.required_ruby_version = '>= 1.9.0'
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake-compiler"
+
+  s.files       = `git ls-files`.split("\n")
+  s.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map{|f| File.basename(f)}
+
+  s.extensions << 'ext/perfmonger/extconf.rb'
+
+  s.post_install_message = <<EOS
+
+============================================================
+
+Thank you for installing perfmonger.
+Try to start performance monitoring with:
+
+    perfmonger record
+
+Enjoy.
+
+============================================================
+
+EOS
+
+end

data/test/run-test.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+export BASE_DIR="`dirname $0`"
+export TEST_DIR="$(readlink -f $(dirname $0))"
+top_dir="$BASE_DIR/.."
+
+if test -z "$NO_MAKE"; then
+    make -C $top_dir > /dev/null || exit 1
+fi
+
+if test -z "$CUTTER"; then
+    CUTTER="`make -s -C $BASE_DIR echo-cutter`"
+fi
+
+if ! test -z "$DEBUG"; then
+    WRAPPER="gdb --args"
+elif ! test -z "$MEMCHECK"; then
+    VGLOG=`mktemp`
+    WRAPPER="valgrind --leak-check=summary --track-origins=yes --log-file=${VGLOG}"
+fi
+
+# MOCK_SO=$(readlink -f $(find -name mock.so | head -n1))
+# if ! test -z "$MOCK_SO"; then
+#     export LD_PRELOAD="$MOCK_SO"
+# fi
+
+$WRAPPER $CUTTER --keep-opening-modules -s $BASE_DIR "$@" $BASE_DIR
+RET=$?
+
+# show log file of valgrind
+if ! test -z "$MEMCHECK"; then
+    if ! test -z "$PAGER"; then
+	$PAGER $VGLOG
+    else
+	cat $VGLOG
+    fi
+fi
+
+exit $RET

data/test/spec/bin_spec.rb

@@ -0,0 +1,37 @@
+
+require 'spec_helper'
+
+describe "PerfmongerCommand" do
+  before(:each) do
+    @perfmonger_command = File.expand_path('../../src/perfmonger', __FILE__)
+  end
+
+  it "should be an executable" do
+    File.executable?(@perfmonger_command).should be_true
+  end
+
+  it 'should print version number if --version specified' do
+    `#{@perfmonger_command} --version`.should include(PerfMonger::VERSION)
+  end
+
+  describe 'stat subcommand' do
+    it 'should print "Execution time: XXX.XXX"' do
+      if File.exists?("/proc/diskstats")
+        `#{@perfmonger_command} stat -- sleep 1`.should match(/^Execution time: (\d+)\.(\d+)$/)
+      else
+        # do nothing
+        true.should be_true
+      end
+    end
+  end
+
+  describe 'summary subcommand' do
+    it 'should print expected output with 2devices.log' do
+      File.open(data_file('2devices.output'), "w") do |f|
+        f.print(`#{@perfmonger_command} summary #{data_file('2devices.log')}`)
+      end
+
+      system("diff -u #{data_file('2devices.expected')} #{data_file('2devices.output')}").should be_true
+    end
+  end
+end

data/test/spec/data/2devices.expected

@@ -0,0 +1,42 @@
+
+== Performance summary of 'spec/data/2devices.log' ==
+
+record duration: 1.00 sec
+
+* Average CPU usage (MAX: 200 %)
+  * Non idle portion: 200.00
+       %usr: 0.00
+       %sys: 200.00
+       %irq: 0.00
+      %soft: 0.00
+     %other: 0.00
+  * Idle portion: 0.00
+    %iowait: 0.00
+      %idle: 0.00
+
+* Average DEVICE usage: sda
+        read IOPS: 1.00
+       write IOPS: 0.00
+  read throughput: 0.00 MB/s
+ write throughput: 0.00 MB/s
+     read latency: 53.00 msec
+    write latency: 0.0 usec
+      read amount: 512.00 bytes
+     write amount: 0.00 bytes
+
+* Average DEVICE usage: sdb
+        read IOPS: 0.00
+       write IOPS: 0.00
+  read throughput: 0.00 MB/s
+ write throughput: 0.00 MB/s
+     read latency: 0.0 usec
+    write latency: 0.0 usec
+      read amount: 0.00 bytes
+     write amount: 0.00 bytes
+
+* TOTAL DEVICE usage: sda, sdb
+        read IOPS: 1.00
+       write IOPS: 0.00
+  read throughput: 0.00 MB/s
+ write throughput: 0.00 MB/s
+

data/test/spec/data/2devices.output

@@ -0,0 +1,42 @@
+
+== Performance summary of 'spec/data/2devices.log' ==
+
+record duration: 1.00 sec
+
+* Average CPU usage (MAX: 200 %)
+  * Non idle portion: 200.00
+       %usr: 0.00
+       %sys: 200.00
+       %irq: 0.00
+      %soft: 0.00
+     %other: 0.00
+  * Idle portion: 0.00
+    %iowait: 0.00
+      %idle: 0.00
+
+* Average DEVICE usage: sda
+        read IOPS: 1.00
+       write IOPS: 0.00
+  read throughput: 0.00 MB/s
+ write throughput: 0.00 MB/s
+     read latency: 53.00 msec
+    write latency: 0.0 usec
+      read amount: 512.00 bytes
+     write amount: 0.00 bytes
+
+* Average DEVICE usage: sdb
+        read IOPS: 0.00
+       write IOPS: 0.00
+  read throughput: 0.00 MB/s
+ write throughput: 0.00 MB/s
+     read latency: 0.0 usec
+    write latency: 0.0 usec
+      read amount: 0.00 bytes
+     write amount: 0.00 bytes
+
+* TOTAL DEVICE usage: sda, sdb
+        read IOPS: 1.00
+       write IOPS: 0.00
+  read throughput: 0.00 MB/s
+ write throughput: 0.00 MB/s
+

data/test/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+$LOAD_PATH << File.expand_path('../../src/ruby', __FILE__)
+
+TEST_DATA_DIR = File.expand_path('../data', __FILE__)
+
+require 'perfmonger'
+require 'tempfile'
+require 'pathname'
+
+def data_file(rel_path)
+  from = Pathname.new(Dir.pwd)
+  path = Pathname.new(File.expand_path(rel_path, TEST_DATA_DIR))
+
+  path.relative_path_from(from).to_s
+end
+
+RSpec.configure do |config|
+  # RSpec config here
+end