perfmonger 0.6.1 → 0.7.0

data/core/perfmonger-recorder.go

@@ -0,0 +1,296 @@
+//usr/bin/env go run $0 $@ ; exit
+
+package main
+
+import (
+	"bufio"
+	"encoding/gob"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/ssh/terminal"
+
+	ss "github.com/hayamiz/perfmonger/core/subsystem"
+)
+
+type RecorderOption struct {
+	interval            time.Duration
+	no_interval_backoff bool
+	timeout             time.Duration
+	start_delay         time.Duration
+	devsParts           []string
+	output              string
+	no_cpu              bool
+	no_disk             bool
+	no_net              bool
+	debug               bool
+	listDevices         bool
+	player_bin          string
+	disks               string
+	targetDisks         *map[string]bool
+}
+
+var option RecorderOption
+
+// By default, measurement interval backoff is enabled.
+// Minimum resoluton guaranteed: BACKOFF_RATIO / BACKOFF_THRESH
+const (
+	BACKOFF_THRESH = 1000.0
+	BACKOFF_RATIO  = 2.0
+)
+
+func parseArgs() {
+	// set options
+	flag.DurationVar(&option.interval, "interval",
+		time.Second, "Measurement interval")
+	flag.BoolVar(&option.no_interval_backoff, "no-interval-backoff",
+		false, "Disable interval backoff")
+	flag.DurationVar(&option.timeout, "timeout",
+		time.Second*0, "Measurement timeout")
+	flag.DurationVar(&option.start_delay, "start-delay",
+		time.Second*0, "Wait time before measurement")
+	flag.StringVar(&option.output, "output",
+		"-", "Output file name")
+	flag.BoolVar(&option.no_cpu, "no-cpu",
+		false, "Do not record CPU usage")
+	flag.BoolVar(&option.no_disk, "no-disk",
+		false, "Do not record disk usage")
+	flag.BoolVar(&option.no_net, "no-net",
+		false, "Do not record net usage")
+	flag.BoolVar(&option.debug, "debug",
+		false, "Enable debug mode")
+	flag.BoolVar(&option.listDevices, "list-devices",
+		false, "List devices and exits")
+	flag.StringVar(&option.disks, "disks",
+		"", "Disk devices to be monitored")
+	flag.StringVar(&option.player_bin, "player-bin",
+		"", "Run perfmonger-player to show JSON output")
+
+	flag.Parse()
+
+	if option.player_bin == "" && terminal.IsTerminal(int(os.Stdout.Fd())) &&
+		option.output == "-" {
+		fmt.Fprintf(os.Stderr, "[recording to data.pgr]\n")
+		option.output = "data.pgr"
+	}
+
+	if option.disks == "" {
+		option.targetDisks = nil
+	} else {
+		option.targetDisks = new(map[string]bool)
+		*option.targetDisks = make(map[string]bool)
+		for _, dev := range strings.Split(option.disks, ",") {
+			(*option.targetDisks)[dev] = true
+		}
+	}
+
+	if option.debug {
+		os.Stderr.WriteString(
+			fmt.Sprintf(
+				`== option
+  - output   : %s
+  - interval : %s
+  - debug    : %t
+  - remainings: %s
+`,
+				option.output,
+				option.interval.String(),
+				option.debug,
+				fmt.Sprint(flag.Args())))
+	}
+}
+
+func main() {
+	var enc *gob.Encoder
+	var out *bufio.Writer
+	var err error
+
+	parseArgs()
+
+	hostname, _ := os.Hostname()
+	cheader := &ss.CommonHeader{ss.Linux, hostname, time.Now()}
+
+	platform_header := ss.NewPlatformHeader()
+
+	if option.listDevices {
+		for _, name := range platform_header.DevsParts {
+			os.Stderr.WriteString(name + "\n")
+		}
+		return
+	}
+
+	var player_cmd *exec.Cmd = nil
+	var player_stdin io.WriteCloser = nil
+	var player_stdout io.ReadCloser = nil
+
+	if option.player_bin != "" {
+		player_cmd = exec.Command(option.player_bin)
+		player_stdin, err = player_cmd.StdinPipe()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to get stdin of %s", option.player_bin)
+			player_cmd = nil
+			player_stdin = nil
+		}
+		player_stdout, err = player_cmd.StdoutPipe()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to get stdout of %s", option.player_bin)
+			player_cmd = nil
+			player_stdin = nil
+			player_stdout = nil
+		}
+
+		err = player_cmd.Start()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to start %s", option.player_bin)
+			player_cmd = nil
+			player_stdin = nil
+			player_stdout = nil
+		}
+
+		// read stdout of player and write to stdout
+		go func() {
+			var buf = make([]byte, 4096)
+			for {
+				n, err := player_stdout.Read(buf)
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					panic(err)
+				}
+
+				if n == 0 {
+					continue
+				}
+				os.Stdout.Write(buf[0:n])
+			}
+		}()
+	}
+
+	if option.output == "-" {
+		out = bufio.NewWriter(os.Stdout)
+		if player_stdin != nil {
+			out = bufio.NewWriter(player_stdin)
+		}
+	} else {
+		file, err := os.Create(option.output)
+		if err != nil {
+			panic(err)
+		}
+		defer file.Close()
+
+		if player_stdin != nil {
+			out = bufio.NewWriter(io.MultiWriter(file, player_stdin))
+		} else {
+			out = bufio.NewWriter(file)
+		}
+	}
+
+	enc = gob.NewEncoder(out)
+
+	// Write the beginning sections
+	err = enc.Encode(cheader)
+	if err != nil {
+		panic(err)
+	}
+
+	err = enc.Encode(platform_header)
+	if err != nil {
+		panic(err)
+	}
+
+	// start delay
+	time.Sleep(option.start_delay)
+
+	var timeout_ch <-chan time.Time
+	var timeout_time time.Time
+	if option.timeout == time.Second*0 {
+		// dummy channel
+		timeout_ch = make(<-chan time.Time)
+		timeout_time = time.Now()
+	} else {
+		timeout_ch = time.After(option.timeout)
+		timeout_time = time.Now().Add(option.timeout)
+	}
+
+	// Loops
+	sigint_ch := make(chan os.Signal, 1)
+	running := true
+	next_time := time.Now()
+	record := ss.NewStatRecord()
+	backoff_counter := 0
+
+	// cause SIGINT to break loop
+	signal.Notify(sigint_ch, os.Interrupt)
+
+	for {
+		record.Time = time.Now()
+
+		if !option.no_cpu {
+			ss.ReadCpuStat(record)
+		}
+		if !option.no_disk {
+			ss.ReadDiskStats(record, option.targetDisks)
+		}
+		if !option.no_net {
+			ss.ReadNetStat(record)
+		}
+
+		err = enc.Encode(record)
+		if err != nil {
+			break
+		}
+		out.Flush()
+
+		if !running {
+			break
+		}
+
+		if !option.no_interval_backoff {
+			backoff_counter++
+			if backoff_counter >= BACKOFF_THRESH {
+				backoff_counter -= BACKOFF_THRESH
+
+				option.interval *= BACKOFF_RATIO
+				if option.interval.Seconds() > 3600.0 {
+					option.interval = time.Hour
+				}
+			}
+		}
+
+		next_time = next_time.Add(option.interval)
+
+		// wait for next iteration
+		select {
+		case <-sigint_ch:
+			running = false
+			break
+		case <-timeout_ch:
+			running = false
+			break
+		case <-time.After(next_time.Sub(time.Now())):
+			break
+		}
+
+		// If next_time and timeout_time is very close,
+		// avoid recording twice in a very short time
+		if option.timeout != time.Second*0 &&
+			timeout_time.Sub(next_time).Seconds() < 0.01 {
+			running = false
+		}
+	}
+
+	out.Flush()
+
+	if player_stdin != nil {
+		player_stdin.Close()
+		_ = player_cmd.Wait()
+	}
+
+	return
+}

data/core/perfmonger-summarizer.go

@@ -0,0 +1,207 @@
+//usr/bin/env go run $0 $@ ; exit
+
+package main
+
+import (
+	"bytes"
+	"encoding/gob"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+
+	ss "github.com/hayamiz/perfmonger/core/subsystem"
+)
+
+type SummaryOption struct {
+	logfile string
+	title   string
+	json    bool
+}
+
+var option SummaryOption
+
+func parseArgs() {
+	flag.BoolVar(&option.json, "json",
+		false, "Show summary in JSON")
+	flag.StringVar(&option.title, "title",
+		"", "Title of summary")
+
+	flag.Parse()
+
+	if len(flag.Args()) < 1 {
+		os.Exit(1)
+	}
+
+	option.logfile = flag.Args()[0]
+}
+
+func main() {
+	var cheader ss.CommonHeader
+	var pheader ss.PlatformHeader
+
+	parseArgs()
+
+	f, err := os.Open(option.logfile)
+	if err != nil {
+		panic(err)
+	}
+	dec := gob.NewDecoder(f)
+
+	err = dec.Decode(&cheader)
+	if err == io.EOF {
+		return
+	}
+	if err != nil {
+		panic(err)
+	}
+	err = dec.Decode(&pheader)
+	if err == io.EOF {
+		return
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	var fst_record ss.StatRecord
+	// read first record
+	err = dec.Decode(&fst_record)
+	if err == io.EOF {
+		return
+	} else if err != nil {
+		panic(err)
+	}
+
+	// loop until last line
+	var lst_records [2]ss.StatRecord
+	idx := 0
+	for {
+		err = dec.Decode(&lst_records[idx])
+		if err == io.EOF {
+			idx ^= 1
+			break
+		} else if err != nil {
+			panic(err)
+		}
+
+		idx ^= 1
+	}
+
+	lst_record := lst_records[idx]
+
+	var cpu_usage *ss.CpuUsage = nil
+	var disk_usage *ss.DiskUsage = nil
+	var net_usage *ss.NetUsage = nil
+
+	if fst_record.Cpu != nil && lst_record.Cpu != nil {
+		cpu_usage, err = ss.GetCpuUsage(fst_record.Cpu, lst_record.Cpu)
+	}
+
+	if fst_record.Disk != nil && lst_record.Disk != nil {
+		disk_usage, err = ss.GetDiskUsage(
+			fst_record.Time, fst_record.Disk,
+			lst_record.Time, lst_record.Disk)
+	}
+
+	if fst_record.Disk != nil && lst_record.Disk != nil {
+		net_usage, err = ss.GetNetUsage(
+			fst_record.Time, fst_record.Net,
+			lst_record.Time, lst_record.Net)
+	}
+
+	interval := lst_record.Time.Sub(fst_record.Time)
+
+	if option.json {
+		buf := bytes.NewBuffer([]byte{})
+
+		buf.WriteString(fmt.Sprintf(`{"exectime":%.3f`, interval.Seconds()))
+		if cpu_usage != nil {
+			buf.WriteString(`,"cpu":`)
+			cpu_usage.WriteJsonTo(buf)
+		}
+
+		if disk_usage != nil {
+			buf.WriteString(`,"disk":`)
+			disk_usage.WriteJsonTo(buf)
+		}
+
+		if disk_usage != nil {
+			buf.WriteString(`,"net":`)
+			net_usage.WriteJsonTo(buf)
+		}
+
+		buf.WriteByte('}')
+
+		fmt.Println(buf.String())
+	} else {
+		if option.title == "" {
+			fmt.Println("== performance summary ==")
+		} else {
+			fmt.Printf("== performance summary of '%s' ==\n", option.title)
+		}
+		fmt.Printf(`
+Duration: %.3f sec
+
+`,
+			interval.Seconds())
+		if cpu_usage != nil {
+			fmt.Printf(`* Average CPU usage (MAX: %d %%)
+  * Non-idle usage: %.2f %%
+       %%usr: %.2f %%
+       %%sys: %.2f %%
+       %%irq: %.2f %%
+      %%soft: %.2f %%
+     %%other: %.2f %%
+  * Idle usage: %.2f %%
+    %%iowait: %.2f %%
+      %%idle: %.2f %%
+
+`,
+				100*cpu_usage.NumCore,
+				100.0*float64(cpu_usage.NumCore)-cpu_usage.All.Idle-cpu_usage.All.Iowait,
+				cpu_usage.All.User+cpu_usage.All.Nice,
+				cpu_usage.All.Sys,
+				cpu_usage.All.Hardirq,
+				cpu_usage.All.Softirq,
+				cpu_usage.All.Steal,
+				cpu_usage.All.Idle+cpu_usage.All.Iowait,
+				cpu_usage.All.Iowait, cpu_usage.All.Idle)
+		}
+
+		if disk_usage != nil {
+			devices := []string{}
+
+			for device, _ := range *disk_usage {
+				if device != "total" {
+					devices = append(devices, device)
+				}
+			}
+			sort.Strings(devices)
+			if len(devices) > 1 {
+				devices = append(devices, "total")
+			}
+
+			for _, device := range devices {
+				e := (*disk_usage)[device]
+				fmt.Printf(`* Average DEVICE usage: %s
+        read IOPS: %.2f
+       write IOPS: %.2f
+  read throughput: %.2f MB/s
+ write throughput: %.2f MB/s
+     read latency: %.1f usec
+    write latency: %.1f usec
+      read amount: %.2f MB
+     write amount: %.2f MB
+
+`,
+					device,
+					e.RdIops, e.WrIops,
+					e.RdSecps*512.0/1024.0/1024.0, e.WrSecps*512.0/1024.0/1024.0,
+					e.RdLatency*1000.0, e.WrLatency*1000.0,
+					float64(e.RdSectors*512)/1024.0/1024.0,
+					float64(e.WrSectors*512)/1024.0/1024.0)
+			}
+		}
+	}
+}

data/core/subsystem/Makefile

@@ -0,0 +1,3 @@
+
+all:
+	make -C ../

data/core/subsystem/perfmonger.go

@@ -0,0 +1,60 @@
+package subsystem
+
+import (
+	"time"
+)
+
+/*
+
+PerfMonger binary format:
+
+1. Common header
+  * platform type tag
+  * host info
+  * timestamp
+  * ...
+2. Platform-dependent header
+  * List of devices
+  * List of NICs
+  * List of IRQs
+  * CPU topology
+  * ...
+3. Record section
+  * Platform-dependent record data stream
+
+*/
+
+//
+// Common header
+//
+
+const (
+	Linux  = 1
+	Darwin = 2
+)
+
+type PlatformType int
+
+type CommonHeader struct {
+	Platform  PlatformType
+	Hostname  string
+	StartTime time.Time
+}
+
+//
+// Platform-dependent header
+//
+
+type LinuxDevice struct {
+	Name  string
+	Parts []string
+}
+
+type LinuxHeader struct {
+	Devices   map[string]LinuxDevice
+	DevsParts []string
+}
+
+type DarwinHeader struct {
+	DevsParts []string
+}

data/core/subsystem/perfmonger_darwin.go

@@ -0,0 +1,22 @@
+// +build darwin
+
+package subsystem
+
+type PlatformHeader DarwinHeader
+
+func NewPlatformHeader() *DarwinHeader {
+	header := new(DarwinHeader)
+	return header
+}
+
+func ReadCpuStat(record *StatRecord) error {
+	return nil
+}
+
+func ReadDiskStats(record *StatRecord, targets *map[string]bool) error {
+	return nil
+}
+
+func ReadNetStat(record *StatRecord) error {
+	return nil
+}