perfmonger 0.8.2 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4a10e8d2334c72c7e5c730a741cb461e71df2e50
-  data.tar.gz: fda2f62538fdcba060ef6e6d68377fca3208fc5b
+  metadata.gz: c6c4a036c08d2f75fc25b25da1b773e025461880
+  data.tar.gz: 87182b819268cdcd289ea1cfb89f8672cc307947
 SHA512:
-  metadata.gz: 098aad2506d627754ac570a6f9ae733d7bf7e2a118fa33dd0114fe290838e33926cea8d2f6675da1aea20a5c026df871f0a147098c711d431fde895b4f9e42ee
-  data.tar.gz: bb2f83fe43a83203f49ebc55c65bc64902e056dd469eb43d64e6508eaa0d652c11c9774d0ec80b6dbd4fca1394e9c4872f60d872738cf028f1dc287e8bed4e72
+  metadata.gz: dcb1dd5f6086bff622f66dc7eb15a835604d0ce63fb2ea7b3d6ab022d3b5724384ca54ec38525c31984bc06dc08c87ca89cb21da7a62696bfaf665df6474d8a8
+  data.tar.gz: cb33cd2fe403d279687a6ad680fbda8af96c778b65b485f6f4b6024eac8e80c8fc6efd79e4e225f1a02fcbfa2f79cfe5061857716ff62fc3b7c4a1a828ca2568

data/.gitignore

@@ -8,3 +8,6 @@
 /lib/exec/
 /ext/perfmonger/Makefile
 /core/perfmonger-summarizer
+/cpu.dat
+/disk.dat
+/perfmonger.pgr

data/NEWS

@@ -1,5 +1,13 @@
-## XXXX-XX-XX: PerfMonger 0.9.0
-  * ...
+## 2016-12-12: PerfMonger 0.9.0
+  * New features
+    * [init-shell] subcommand: Added for loading shell completion function from rcfile
+      * zsh support
+      * bash support
+    * [fingerprint]
+      * log LVM, fdisk, and lsblk info
+    * [plot]
+      * Fast plot data formatting with new core implementation
+    * Improved shell completion
 
 ## 2016-11-22: PerfMonger 0.8.2
   * Bug fixes

data/core/Makefile

@@ -20,6 +20,10 @@ all: build
 	go build -o $@ perfmonger-summarizer.go
 
 
+../lib/exec/perfmonger-plot-formatter_linux_386: perfmonger-plot-formatter.go $(GOSRC)
+	go build -o $@ perfmonger-plot-formatter.go
+
+
 ../lib/exec/perfmonger-recorder_linux_amd64: perfmonger-recorder.go $(GOSRC)
 	go build -o $@ perfmonger-recorder.go
 
@@ -32,6 +36,10 @@ all: build
 	go build -o $@ perfmonger-summarizer.go
 
 
+../lib/exec/perfmonger-plot-formatter_linux_amd64: perfmonger-plot-formatter.go $(GOSRC)
+	go build -o $@ perfmonger-plot-formatter.go
+
+
 ../lib/exec/perfmonger-recorder_darwin_amd64: perfmonger-recorder.go $(GOSRC)
 	go build -o $@ perfmonger-recorder.go
 
@@ -44,8 +52,12 @@ all: build
 	go build -o $@ perfmonger-summarizer.go
 
 
-build: ../lib/exec/perfmonger-recorder_linux_386 ../lib/exec/perfmonger-player_linux_386 ../lib/exec/perfmonger-summarizer_linux_386 ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64
+../lib/exec/perfmonger-plot-formatter_darwin_amd64: perfmonger-plot-formatter.go $(GOSRC)
+	go build -o $@ perfmonger-plot-formatter.go
+
+
+build: ../lib/exec/perfmonger-recorder_linux_386 ../lib/exec/perfmonger-player_linux_386 ../lib/exec/perfmonger-summarizer_linux_386 ../lib/exec/perfmonger-plot-formatter_linux_386 ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-plot-formatter_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64 ../lib/exec/perfmonger-plot-formatter_darwin_amd64
 
 clean:
-	rm -f ../lib/exec/perfmonger-recorder_linux_386 ../lib/exec/perfmonger-player_linux_386 ../lib/exec/perfmonger-summarizer_linux_386 ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64
+	rm -f ../lib/exec/perfmonger-recorder_linux_386 ../lib/exec/perfmonger-player_linux_386 ../lib/exec/perfmonger-summarizer_linux_386 ../lib/exec/perfmonger-plot-formatter_linux_386 ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-plot-formatter_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64 ../lib/exec/perfmonger-plot-formatter_darwin_amd64
 

data/core/build.sh

@@ -54,7 +54,7 @@ for idx in $(seq 0 $((${#TARGET[@]}-1))); do
     export GOOS=$1
     export GOARCH=$2
 
-    for subcmd in recorder player summarizer; do
+    for subcmd in recorder player summarizer plot-formatter; do
         TARGETS+=(../lib/exec/perfmonger-${subcmd}_${GOOS}_${GOARCH})
 
         cat <<EOF >> Makefile

data/core/perfmonger-plot-formatter.go

@@ -0,0 +1,325 @@
+//usr/bin/env go run $0 $@ ; exit
+
+package main
+
+import (
+	"bufio"
+	"encoding/gob"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sort"
+
+	ss "github.com/hayamiz/perfmonger/core/subsystem"
+)
+
+type CmdOption struct {
+	DiskFile       string
+	CpuFile        string
+	PerfmongerFile string
+}
+
+func parseArgs() *CmdOption {
+	opt := new(CmdOption)
+
+	flag.StringVar(&opt.DiskFile, "diskfile", "./disk.dat", "Disk performance data file")
+	flag.StringVar(&opt.CpuFile, "cpufile", "./cpu.dat", "CPU performance data file")
+	flag.StringVar(&opt.PerfmongerFile, "perfmonger", "", "Perfmonger log file")
+
+	flag.Parse()
+
+	if opt.PerfmongerFile == "" {
+		os.Stderr.WriteString("[ERROR] perfmonger log file is required.\n")
+		os.Exit(1)
+	}
+
+	return opt
+}
+
+type DiskMetaEntry struct {
+	Name string `json:"name"`
+	Idx  int    `json:"idx"`
+}
+
+type DiskMeta struct {
+	Devices []DiskMetaEntry `json:"devices"`
+}
+
+type CpuMeta struct {
+	NumCore int `json:"num_core"`
+}
+
+type PlotMeta struct {
+	Disk      DiskMeta `json:"disk"`
+	Cpu       CpuMeta  `json:"cpu"`
+	StartTime float64  `json:"start_time"`
+	EndTime   float64  `json:"end_time"`
+}
+
+type DiskDatTmpFile struct {
+	Name   string
+	Path   string
+	File   *os.File
+	Writer *bufio.Writer
+	Idx    int
+}
+
+type CpuDatTmpFile struct {
+	CoreId int
+	Path   string
+	File   *os.File
+	Writer *bufio.Writer
+}
+
+func makeDiskDatTmpFile(dname string, idx int) *DiskDatTmpFile {
+	ret := new(DiskDatTmpFile)
+	ret.Name = dname
+	ret.Idx = idx
+
+	f, err := ioutil.TempFile("", "perfmonger-"+dname)
+	if err != nil {
+		panic(err)
+	}
+	ret.File = f
+	ret.Path = f.Name()
+	ret.Writer = bufio.NewWriter(f)
+
+	return ret
+}
+
+func makeCpuDatTmpFile(coreid int) *CpuDatTmpFile {
+	ret := new(CpuDatTmpFile)
+	ret.CoreId = coreid
+
+	f, err := ioutil.TempFile("", fmt.Sprintf("perfmonger-core%d-", coreid))
+	if err != nil {
+		panic(err)
+	}
+	ret.File = f
+	ret.Path = f.Name()
+	ret.Writer = bufio.NewWriter(f)
+
+	return ret
+}
+
+func printCoreUsage(writer *bufio.Writer, elapsed_time float64, coreusage *ss.CpuCoreUsage) {
+	writer.WriteString(
+		fmt.Sprintf("%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\n",
+			elapsed_time,
+			coreusage.User,
+			coreusage.Nice,
+			coreusage.Sys,
+			coreusage.Iowait,
+			coreusage.Hardirq,
+			coreusage.Softirq,
+			coreusage.Steal,
+			coreusage.Guest,
+			coreusage.Idle))
+}
+
+func main() {
+	opt := parseArgs()
+
+	var in *os.File
+
+	f, err := os.Open(opt.PerfmongerFile)
+	if err != nil {
+		panic(err)
+	}
+	in = f
+	defer f.Close()
+
+	dec := gob.NewDecoder(bufio.NewReader(in))
+
+	var cheader ss.CommonHeader
+	var pheader ss.PlatformHeader
+	var records = make([]ss.StatRecord, 2)
+	curr := 0
+
+	err = dec.Decode(&cheader)
+	if err == io.EOF {
+		return
+	}
+	if err != nil {
+		panic(err)
+	}
+	err = dec.Decode(&pheader)
+	if err == io.EOF {
+		return
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	// read first record
+	err = dec.Decode(&records[curr])
+	if err == io.EOF {
+		return
+	} else if err != nil {
+		panic(err)
+	}
+	t0 := records[curr].Time
+	curr ^= 1
+
+	meta_set := false
+	meta := PlotMeta{}
+	meta.StartTime = float64(records[0].Time.UnixNano()) / 1.0e9
+
+	disk_dat_files := map[string]*DiskDatTmpFile{}
+	cpu_dat_files := make([]*CpuDatTmpFile, records[0].Cpu.NumCore)
+	meta.Cpu.NumCore = records[0].Cpu.NumCore
+
+	f, err = os.Create(opt.CpuFile)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	cpu_writer := bufio.NewWriter(f)
+
+	cpu_writer.WriteString("# All cpu usage\n")
+	cpu_writer.WriteString("# elapsed_time	%usr	%nice	%sys	%iowait	%hardirq	%softirq	%steal	%guest	%idle\n")
+
+	for {
+		prev_rec := &records[curr^1]
+		cur_rec := &records[curr]
+
+		err := dec.Decode(cur_rec)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			panic(err)
+		}
+
+		// Disk usage
+		dusage, err := ss.GetDiskUsage(prev_rec.Time, prev_rec.Disk, cur_rec.Time, cur_rec.Disk)
+		if err != nil {
+			panic(err)
+		}
+		didx := 0
+
+		var dnames []string
+		for dname, _ := range *dusage {
+			if dname != "total" {
+				dnames = append(dnames, dname)
+			}
+		}
+		sort.Strings(dnames)
+		dnames = append(dnames, "total")
+
+		// for dname, dusage_entry := range *dusage {
+		for _, dname := range dnames {
+			dusage_entry, ok := (*dusage)[dname]
+			if !ok {
+				panic("device '" + dname + "' not found")
+			}
+
+			if !meta_set {
+				meta.Disk.Devices =
+					append(meta.Disk.Devices,
+						DiskMetaEntry{Name: dname, Idx: didx})
+			}
+
+			disk_dat, ok := disk_dat_files[dname]
+			if !ok {
+				disk_dat = makeDiskDatTmpFile(dname, didx)
+				disk_dat_files[dname] = disk_dat
+				defer disk_dat.File.Close()
+
+				disk_dat.Writer.WriteString("\n\n\n")
+				disk_dat.Writer.WriteString("# device: " + disk_dat.Name + "\n")
+				disk_dat.Writer.WriteString(fmt.Sprintln(
+					"# elapsed_time	r_iops	w_iops	r_MB/s	w_MB/s	r_latency	w_latency	r_avgsz	w_avgsz	qdepth"))
+			}
+
+			elapsed_time := prev_rec.Time.Sub(t0).Seconds()
+			disk_dat.Writer.WriteString(
+				fmt.Sprintf("%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\n",
+					elapsed_time,
+					dusage_entry.RdIops,
+					dusage_entry.WrIops,
+					dusage_entry.RdSecps*512.0/1024.0/1024.0,
+					dusage_entry.WrSecps*512.0/1024.0/1024.0,
+					dusage_entry.RdLatency,
+					dusage_entry.WrLatency,
+					dusage_entry.AvgRdSize,
+					dusage_entry.AvgWrSize,
+					dusage_entry.ReqQlen))
+
+			didx += 1
+		}
+
+		// Cpu usage
+		cusage, err := ss.GetCpuUsage(prev_rec.Cpu, cur_rec.Cpu)
+		if err != nil {
+			panic(err)
+		}
+		for coreid, coreusage := range cusage.CoreUsages {
+			cpu_dat := cpu_dat_files[coreid]
+			if cpu_dat == nil {
+				cpu_dat = makeCpuDatTmpFile(coreid)
+				cpu_dat_files[coreid] = cpu_dat
+				defer cpu_dat.File.Close()
+
+				cpu_dat.Writer.WriteString(fmt.Sprintf("\n\n\n# core: %d\n", coreid))
+				cpu_dat.Writer.WriteString("# elapsed_time	%usr	%nice	%sys	%iowait	%hardirq	%softirq	%steal	%guest	%idle\n")
+			}
+
+			printCoreUsage(cpu_dat.Writer, prev_rec.Time.Sub(t0).Seconds(), coreusage)
+		}
+		printCoreUsage(cpu_writer, prev_rec.Time.Sub(t0).Seconds(), cusage.All)
+
+		curr ^= 1
+		meta_set = true
+	}
+
+	meta.EndTime = float64(records[curr^1].Time.UnixNano()) / 1.0e9
+
+	for _, disk_dat := range disk_dat_files {
+		disk_dat.Writer.Flush()
+		disk_dat.File.Close()
+	}
+
+	f, err = os.Create(opt.DiskFile)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	df_writer := bufio.NewWriter(f)
+
+	for _, dev := range meta.Disk.Devices {
+		disk_dat, ok := disk_dat_files[dev.Name]
+		if !ok {
+			panic(dev.Name)
+		}
+
+		content, err := ioutil.ReadFile(disk_dat.Path)
+		if err != nil {
+			panic(err)
+		}
+
+		df_writer.Write(content)
+
+		os.Remove(disk_dat.Path)
+	}
+	df_writer.Flush()
+
+	for _, cpu_dat := range cpu_dat_files {
+		cpu_dat.Writer.Flush()
+		cpu_dat.File.Close()
+
+		content, err := ioutil.ReadFile(cpu_dat.Path)
+		if err != nil {
+			panic(err)
+		}
+
+		cpu_writer.Write(content)
+		os.Remove(cpu_dat.Path)
+	}
+	cpu_writer.Flush()
+
+	json_enc := json.NewEncoder(os.Stdout)
+	json_enc.Encode(meta)
+}

data/core/subsystem/usage.go

@@ -96,7 +96,19 @@ func GetCpuCoreUsage(c1 *CpuCoreStat, c2 *CpuCoreStat) (*CpuCoreUsage, error) {
 	itv := c2.Uptime() - c1.Uptime()
 
 	if itv == 0 {
-		return nil, errors.New("uptime difference is zero")
+		// return nil, errors.New("uptime difference is zero")
+		usage.User = 0
+		usage.Nice = 0
+		usage.Sys = 0
+		usage.Idle = 0
+		usage.Iowait = 0
+		usage.Hardirq = 0
+		usage.Softirq = 0
+		usage.Steal = 0
+		usage.Guest = 0
+		usage.GuestNice = 0
+
+		return usage, nil
 	} else if itv < 0 {
 		return nil, errors.New("uptime difference is negative")
 	}

data/lib/perfmonger.rb

@@ -13,3 +13,4 @@ require 'perfmonger/command/play.rb'
 require 'perfmonger/command/plot.rb'
 require 'perfmonger/command/server.rb'
 require 'perfmonger/command/fingerprint.rb'
+require 'perfmonger/command/init-shell.rb'

data/lib/perfmonger/command/core.rb

@@ -55,6 +55,10 @@ class CoreFinder
     def summarizer
       self.find("summarizer")
     end
+
+    def plot_formatter
+      self.find("plot-formatter")
+    end
   end
 end