perfmonger 0.10.2 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fec4f6034a16fc8915547917510cb8c69299b095
-  data.tar.gz: 6eb9b12b445f78adf0709d298ce71279abff2c8a
+  metadata.gz: 8e47b0d30a3300a01ae5efcc0bbbe10baf524561
+  data.tar.gz: 88988c5009fcbe62049c7355f3e8374b3c825f5d
 SHA512:
-  metadata.gz: 101f22ee04ff2cdee38ddcae194970ce5af08c70a4850c295e38ecece1c14648bc1953b35558a9735f642423f03cac583b868010a9f35d8fd9a629d0ea9b1f5a
-  data.tar.gz: 31321ba34ecc71ce31668f13e32d58f2d82fd2808dfbabb3c3d869ae78e70fba46020e3dbfe2779e8a2cd202e2c43e392ee1266a4a94db748b726949d7d8a34d
+  metadata.gz: 6f767fae2183025478897926d3b79d705fd30937f8c7f7cd6221dd29ad0d9ae03f7940c29be234613f8100f1c9dafb3e8d079075ab2d2d1879b2de33fdc2e482
+  data.tar.gz: e4f1d714cd58d57db382320224d356f919692f25a4552d5649accf7a27c63cac693049972dd994904cdb4d4a54c9f2c19e374b25d928f392e4bffbdb517d5460

data/.gitignore

@@ -11,3 +11,6 @@
 /cpu.dat
 /disk.dat
 /perfmonger.pgr
+/.wercker/
+/spec/examples.txt
+/perfmonger.pgr.gz

data/.travis.yml

@@ -1,17 +1,18 @@
-
 language: ruby
 install:
   - sudo apt-get update
   - sudo apt-get install gnuplot
   - gnuplot -e "set terminal" < /dev/null 2>&1
   - bundle install
-  - go_version="1.4.2"
+  - go_version="1.8.3"
   - wget http://golang.org/dl/go${go_version}.linux-amd64.tar.gz
   - sudo tar -C /usr/local -xzf go${go_version}.linux-amd64.tar.gz
   - export PATH=$PATH:/usr/local/go/bin
   - export GOPATH="$HOME/go"
   - export PATH="$PATH:$GOPATH/bin"
   - mkdir -p "$HOME/go/{src,pkg,bin}"
+  - go get -u github.com/mattn/go-isatty
+  - go get -u github.com/hayamiz/go-projson
   - go get -u golang.org/x/crypto/ssh/terminal
   - go get -u github.com/hayamiz/perfmonger/core/subsystem
 rvm:
@@ -22,3 +23,6 @@ rvm:
 script:
   - rake spec
   - rake test_core
+notifications:
+  slack:
+    secure: fH8tRyxWHL60OV6QuJlzig9lCLbjfpHx8E6D2EzgQz7+/wqAxtoTUyiN7mbpEJa4hyQeZfpmMpDTnl2tHD6eI8yqjAsY4Q+jt21tCKyrKMegq9Pypd4eMP4o+DupT2mXm0K3cZ2Kgb+yP8AuJPoTy20j3kpmnDFWdqRnhgpBLC8=

data/HOWTO.md

@@ -3,7 +3,6 @@
 ## TODO in release
 
 1. Make a commit for changing version number
-  * Update version number in configure.ac
   * Add a new version number and release date to NEWS
 2. Tag the commit with name `v<VERSION>` ex) `v0.3.0`
 3. Push tags to the github repository

data/NEWS

@@ -1,10 +1,19 @@
-## 2017-09-14: PerfMonger 0.10.2
+## 2018-05-15: PerfMonger 0.11.0
+  * New features
+    * [play] subcommand:
+      * Add --color, --pretty option for pretty JSON output
+    * [live] subcommand:
+      * Add --color, --pretty option for pretty JSON output
+    * [fingerprint] subcommand:
+      * Collect additional info: numactl, ec2-metadata
   * Bug fixes
     * [plot] subcommand:
-      * Delete temporary files after plot
-
-## 2017-09-08: PerfMonger 0.10.1
-  * Yanked wrongly shipped v0.10.0 and pushed up-to-date code
+      * Correctly filter out disk usages by --disk-only option
+  * Changes
+    * Dropped support of i386
+    * Use go-projson for JSON output
+    * [plot] subcommand:
+      * Stacked graphs layout in allcpu.pdf
 
 ## 2017-06-19: PerfMonger 0.10.0
   * New features
@@ -204,4 +213,4 @@ PerfMonger is available on yum repository from this release.
 ## 2011-12-09: PerfMonger 0.1.0 released
 
   * New features
-    * I/O performance monitoring
+    * I/O performance monitoring

data/README.md

@@ -5,31 +5,97 @@
 
 [![wercker status](https://app.wercker.com/status/44c3ade6a2406d337df6d93097a52fdf/m "wercker status")](https://app.wercker.com/project/bykey/44c3ade6a2406d337df6d93097a52fdf)
 
-PerfMonger is an yet anothor performance measurement/monitoring tool
-speaking JSON.
+PerfMonger is a system performance monitor which enables high-resolution and holistic performance measurement with the programmer friendly interface.
+
+* High-resolution: sub-second level monitoring is possible!
+* Holistic performance measurement: monitoring CPU, Disk I/O, Network all at once.
+* Programmer friendly: PerfMonger speaks monitoring results in JSON format, which makes later performance analysis much easier (ex. [jq](https://github.com/stedolan/jq)).
 
 **CAUTION: PerfMonger is still in early stage, so there may be a drastic change in the future. Do not use it for critical jobs**
 
 ## Target platform
 
   * GNU/Linux
+  * Mac OS X (experimental support)
 
-## Prerequisites
-
-  * Ruby 1.9.3 or later
-  * gnuplot 4.6.0 or later (optional)
-
-Note: You need Cutter unit testing framework for building/running tests.
-
-## How to install
+## How to installation
 
     gem install perfmonger
 
+You need gnuplot 4.6.0 or later build with cairo terminals for plotting measurement data with `perfmonger plot` command.
+
 ### Build from source
 
+You need Ruby 1.9.3 or later, and Go 1.8 or later to build perfmonger.
+
+    bundle
     rake build
 
-## How to use: case study
+## Getting started
+
+Basic usage of PerfMonger is:
+
+* Run `perfmonger record` to record performance information logs
+* Run `perfmonger play` to show performance information logs in JSON format
+
+`perfmonger play` repatedly prints records of system performance including CPU
+usages, disk usages, and network usages. One line includes only one record, so
+you can easily process output records with `jq`, or any scripting languages like
+Ruby, Python, and etc.
+
+Pretty-printed structure of a record is as follows:
+
+```
+{"time": 1500043743.504, # timestamp in unix epoch time
+  "cpu": {               # CPU usages
+    "num_core": 2,         # the number of cores
+    "all": {               # aggregated CPU usage (max = num_core * 100%)
+      "usr": 50.0,
+      "sys": 50.0,
+      "idle": 100.0,
+      ...
+    },
+    "cores": [             # Usage of each CPU core
+      {
+        "usr": 25.0,
+        "sys": 25.0,
+        "idle": 50.0,
+        ...
+      },
+      {
+        "usr": 25.0,
+        "sys": 25.0,
+        "idle": 50.0,
+        ...
+      }
+    ]
+  },
+  "disk": {             # Disk usages
+    "devices": ["sda"],   # List of disk devices
+    "sda": {              # Usage of device 'sda'
+      "riops": 10.0,        # The number of read I/O per second
+      "wiops": 20.0,        # The number of write I/O per second
+      "rkbyteps": 80.0,     # Read transfer rate in KB/s
+      "wkbyteps": 160.0,    # Write transfer rate in KB/s
+      ...
+    }
+    "total": {           # Aggregated usage of all devices
+      "riops": 10.0,
+      "wiops": 20.0,
+      "rkbyteps": 80.0,
+      "wkbyteps": 160.0,
+      ...
+    }
+  }
+}
+```
+
+
+## Typical use cases
+
+### `perfmonger live`: live performance monitoring
+
+    $ perfmonger live
 
 ### Monitor IO performance of /dev/sda for each 0.1 second
 
@@ -42,19 +108,3 @@ Note: You need Cutter unit testing framework for building/running tests.
 ### Monitor CPU usage and IO performance of /dev/sda, sdb for each 0.1 second
 
     $ perfmonger record -i 0.1 -d sda -d sdb
-
-### Plot CPU and IOPS
-
-    $ perfmonger record -i 0.1 -C -d sda > /tmp/perfmonger.log & sleep 10; pkill perfmonger
-    $ perfmonger plot -o /path/to/output_dir/ -Tpng /tmp/perfmonger.log
-    $ display /path/to/output_dir/read-iops.png
-    $ display /path/to/output_dir/cpu.png
-
-![Sample image of IOPS graph](https://raw.github.com/hayamiz/perfmonger/master/misc/sample-read-iops.png)
-![Sample image of CPU usage graph](https://raw.github.com/hayamiz/perfmonger/master/misc/sample-cpu.png)
-
-## Special Thanks
-
-Large portion of PerfMonger comes from
-[SYSSTAT](http://sebastien.godard.pagesperso-orange.fr/) codebase. Thanks for
-their great work.

data/Rakefile

@@ -7,7 +7,7 @@ task :default => [:spec, :test_core]
 desc "Run all specs in spec directory"
 RSpec::Core::RakeTask.new(:spec)
 
-task :spec => [:self_build_core]
+task :spec => [:cross_build_core]
 
 desc "Cross build core recorder/player"
 task :cross_build_core do
@@ -18,20 +18,29 @@ task :cross_build_core do
   end
 end
 
-desc "Self build core recorder/player"
-task :self_build_core do
-  Dir.chdir("./core") do
-    sh "./build.sh -"
-  end
-end
-
 task :build => :cross_build_core
 
+desc "Install Golang libraries"
+task :go_get do
+  sh "go get -u github.com/hayamiz/go-projson"
+  sh "go get -u github.com/hayamiz/perfmonger/core/subsystem"
+  sh "go get -u golang.org/x/crypto/ssh/terminal"
+  sh "go get -u github.com/mattn/go-isatty"
+end
+
 desc "Run tests of core recorder/player"
-task :test_core do
+task :test_core => [:cross_build_core] do
   Dir.chdir("./core/subsystem") do
     sh "go test -v -cover"
+
+    # running static analysis
+    sh "go vet *.go"
   end
+
+  Dir.chdir("./core") do
+    sh "go vet *.go"
+  end
+
 end
 
 desc "Removed generated files"

data/core/Makefile

@@ -9,22 +9,6 @@ GO_SRC := utils.go
 all: build
 
 
-../lib/exec/perfmonger-recorder_linux_386: perfmonger-recorder.go $(GO_DEPS)
-	go build -o $@ perfmonger-recorder.go $(GO_SRC)
-
-
-../lib/exec/perfmonger-player_linux_386: perfmonger-player.go $(GO_DEPS)
-	go build -o $@ perfmonger-player.go $(GO_SRC)
-
-
-../lib/exec/perfmonger-summarizer_linux_386: perfmonger-summarizer.go $(GO_DEPS)
-	go build -o $@ perfmonger-summarizer.go $(GO_SRC)
-
-
-../lib/exec/perfmonger-plot-formatter_linux_386: perfmonger-plot-formatter.go $(GO_DEPS)
-	go build -o $@ perfmonger-plot-formatter.go $(GO_SRC)
-
-
 ../lib/exec/perfmonger-recorder_linux_amd64: perfmonger-recorder.go $(GO_DEPS)
 	go build -o $@ perfmonger-recorder.go $(GO_SRC)
 
@@ -57,8 +41,8 @@ all: build
 	go build -o $@ perfmonger-plot-formatter.go $(GO_SRC)
 
 
-build: ../lib/exec/perfmonger-recorder_linux_386 ../lib/exec/perfmonger-player_linux_386 ../lib/exec/perfmonger-summarizer_linux_386 ../lib/exec/perfmonger-plot-formatter_linux_386 ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-plot-formatter_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64 ../lib/exec/perfmonger-plot-formatter_darwin_amd64
+build: ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-plot-formatter_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64 ../lib/exec/perfmonger-plot-formatter_darwin_amd64
 
 clean:
-	rm -f ../lib/exec/perfmonger-recorder_linux_386 ../lib/exec/perfmonger-player_linux_386 ../lib/exec/perfmonger-summarizer_linux_386 ../lib/exec/perfmonger-plot-formatter_linux_386 ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-plot-formatter_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64 ../lib/exec/perfmonger-plot-formatter_darwin_amd64
+	rm -f ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-plot-formatter_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64 ../lib/exec/perfmonger-plot-formatter_darwin_amd64
 

data/core/build.sh

@@ -28,7 +28,7 @@ if [[ $1 = "-" ]]; then
     TARGET=("${os} ${arch}")
 else
     # cross build
-    TARGET=("linux 386" "linux amd64"  "darwin amd64")
+    TARGET=("linux amd64"  "darwin amd64")
 fi
 
 set -e

data/core/perfmonger-player.go

@@ -4,39 +4,38 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/gob"
+	"flag"
 	"fmt"
 	"io"
 	"os"
 
+	projson "github.com/hayamiz/go-projson"
 	ss "github.com/hayamiz/perfmonger/core/subsystem"
 )
 
-func showCpuStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
+type PlayerOption struct {
+	logfile string
+	color   bool
+	pretty  bool
+}
+
+var option PlayerOption
+var init_rec ss.StatRecord
+
+func showCpuStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
 	cusage, err := ss.GetCpuUsage(prev_rec.Cpu, cur_rec.Cpu)
 	if err != nil {
 		return err
 	}
-	buffer.WriteString(`,"cpu":`)
-	cusage.WriteJsonTo(buffer)
+
+	printer.PutKey("cpu")
+	cusage.WriteJsonTo(printer)
 
 	return nil
 }
 
-func showInterruptStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
-	// intr_usage, err := ss.GetInterruptUsage(
-	// 	prev_rec.Time, prev_rec.Interrupt,
-	// 	cur_rec.Time, cur_rec.Interrupt)
-	// if err != nil {
-	// 	return err
-	// }
-	//
-	// buffer.WriteString(`,"intr":`)
-	// intr_usage.WriteJsonTo(buffer)
-	//
-	// return nil
-
+func showInterruptStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
 	intr_usage, err := ss.GetInterruptUsage(
 		prev_rec.Time, prev_rec.Interrupt,
 		cur_rec.Time, cur_rec.Interrupt)
@@ -44,13 +43,13 @@ func showInterruptStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *s
 		return err
 	}
 
-	buffer.WriteString(`,"intr":`)
-	intr_usage.WriteJsonTo(buffer)
+	printer.PutKey("intr")
+	intr_usage.WriteJsonTo(printer)
 
 	return nil
 }
 
-func showDiskStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
+func showDiskStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
 	dusage, err := ss.GetDiskUsage(
 		prev_rec.Time, prev_rec.Disk,
 		cur_rec.Time, cur_rec.Disk)
@@ -58,13 +57,14 @@ func showDiskStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *ss.Sta
 		return err
 	}
 
-	buffer.WriteString(`,"disk":`)
-	dusage.WriteJsonTo(buffer)
+	printer.PutKey("disk")
+
+	dusage.WriteJsonTo(printer)
 
 	return nil
 }
 
-func showNetStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
+func showNetStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
 	dusage, err := ss.GetNetUsage(
 		prev_rec.Time, prev_rec.Net,
 		cur_rec.Time, cur_rec.Net,
@@ -73,52 +73,77 @@ func showNetStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *ss.Stat
 		return err
 	}
 
-	buffer.WriteString(`,"net":`)
-	dusage.WriteJsonTo(buffer)
+	printer.PutKey("net")
+
+	dusage.WriteJsonTo(printer)
 
 	return nil
 }
 
-func showStat(buffer *bytes.Buffer, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
-	buffer.WriteString(fmt.Sprintf(`{"time":%.3f`, float64(cur_rec.Time.UnixNano())/1e9))
+func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
+	printer.Reset()
+	if option.pretty {
+		printer.SetStyle(projson.SmartStyle)
+	}
+	if option.color {
+		printer.SetColor(true)
+	}
+	printer.BeginObject()
+	printer.PutKey("time")
+	printer.PutFloatFmt(float64(cur_rec.Time.UnixNano())/1e9, "%.3f")
+	printer.PutKey("elapsed_time")
+	printer.PutFloatFmt((float64(cur_rec.Time.UnixNano())-float64(init_rec.Time.UnixNano()))/1e9,
+		"%.3f")
+
 	if cur_rec.Cpu != nil {
-		err := showCpuStat(buffer, prev_rec, cur_rec)
+		err := showCpuStat(printer, prev_rec, cur_rec)
 		if err != nil {
 			return err
 		}
 	}
 	if cur_rec.Interrupt != nil {
-		err := showInterruptStat(buffer, prev_rec, cur_rec)
+		err := showInterruptStat(printer, prev_rec, cur_rec)
 		if err != nil {
 			return err
 		}
 	}
 	if cur_rec.Disk != nil {
-		err := showDiskStat(buffer, prev_rec, cur_rec)
+		err := showDiskStat(printer, prev_rec, cur_rec)
 		if err != nil {
 			return err
 		}
 	}
 	if cur_rec.Net != nil {
-		err := showNetStat(buffer, prev_rec, cur_rec)
+		err := showNetStat(printer, prev_rec, cur_rec)
 		if err != nil {
 			return err
 		}
 	}
-	buffer.WriteString("}\n")
+
+	printer.FinishObject()
 
 	return nil
 }
 
+func parseArgs() {
+	flag.BoolVar(&option.color, "color", false, "Use colored JSON output")
+	flag.BoolVar(&option.pretty, "pretty", false, "Use human readable JSON output")
+
+	flag.Parse()
+
+	option.logfile = flag.Arg(0)
+}
+
 func main() {
-	args := os.Args
 	var in *os.File
 	var out *bufio.Writer
 
-	if len(args) < 2 {
+	parseArgs()
+
+	if option.logfile == "" {
 		in = os.Stdin
 	} else {
-		f, err := os.Open(args[1])
+		f, err := os.Open(option.logfile)
 		if err != nil {
 			panic(err)
 		}
@@ -159,9 +184,10 @@ func main() {
 	} else if err != nil {
 		panic(err)
 	}
+	init_rec = records[curr]
 	curr ^= 1
 
-	buffer := bytes.NewBuffer([]byte{})
+	printer := projson.NewPrinter()
 	for {
 		prev_rec := &records[curr^1]
 		cur_rec := &records[curr]
@@ -173,21 +199,26 @@ func main() {
 			panic(err)
 		}
 
-		err = showStat(buffer, prev_rec, cur_rec)
+		err = showStat(printer, prev_rec, cur_rec)
 		if err != nil {
-			buffer.Reset()
+			printer.Reset()
 			fmt.Fprintln(os.Stderr, "skip by err")
 			continue
 		}
 
-		_, err = out.WriteString(buffer.String())
+		if str, err := printer.String(); err != nil {
+			fmt.Println("error", err)
+			fmt.Println(str)
+		} else {
+			_, err = out.WriteString(str + "\n")
+		}
 		err = out.Flush()
 		if err != nil {
 			// stdout is closed
 			break
 		}
 
-		buffer.Reset()
+		printer.Reset()
 
 		curr ^= 1
 	}