perfmonger 0.11.2 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a60a8aaed2af9077a0027baf15afc68856ed6c5d9a08267dde6a903abeea762
-  data.tar.gz: 46113a4949d34a74737acac64f8d032b31cf3bddfdad51dbc158fc0599fa1151
+  metadata.gz: 7cafa0eb63dce619c0f5cf2f23042db020e435905eb7d00e2338d3c499495d02
+  data.tar.gz: ab6875fdfe16bd70461db8569b4c5572bbd261e931a6efa98c2bede6660e9d1b
 SHA512:
-  metadata.gz: 4eacfdf595f3bd39bc64f5516ef8d14eff0a49e9f8befdf7a2102b642dd604be88328783fceb7a1b3163a883554eb624d1b46e88a3a13c23b9265ef0a4b1343c
-  data.tar.gz: 1b416767ae0201a1cbb7f500c99f14c88566a86c7a8e03c5f0d56a3b1ba0ba8b12b5491a0a5ac35d2b06263b6af5fbd6b22e7550df8720bd78a183e8c4b68eae
+  metadata.gz: 17bb023ddebbde5fda03b2ed90411274a335e34ac5ddc83e4df7821454e40484957688ed7dfe72b9245190006fee0c2e1e39aa375fe17c12f95bba9838f2d767
+  data.tar.gz: 259636bb0f5049e404ec752edf0e659b23b8bc1adcaea69ef2f0db4cc3bfccac028793fd48920d9e4a54e5cda86a3a52581eb2dc250abd0b5f8476dec815772f

data/.travis.yml

@@ -4,7 +4,7 @@ install:
   - sudo apt-get install gnuplot
   - gnuplot -e "set terminal" < /dev/null 2>&1
   - bundle install
-  - go_version="1.8.3"
+  - go_version="1.14.1"
   - wget http://golang.org/dl/go${go_version}.linux-amd64.tar.gz
   - sudo tar -C /usr/local -xzf go${go_version}.linux-amd64.tar.gz
   - export PATH=$PATH:/usr/local/go/bin
@@ -12,10 +12,10 @@ install:
   - export PATH="$PATH:$GOPATH/bin"
   - mkdir -p "$HOME/go/{src,pkg,bin}"
 rvm:
-  - 2.2.10
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
+  - 2.4.9
+  - 2.5.7
+  - 2.6.5
+  - 2.7.0
 script:
   - rake go_get
   - rake spec

data/NEWS

@@ -1,9 +1,46 @@
-## 20XX-XX-XX: PerfMonger 0.12.0
+## 2021-XX-XX: PerfMonger 0.15.0
+
+## 2021-07-27: PerfMonger 0.14.0
+  * New features
+    * [plot] subcommand:
+      * Added memory usage plot
+  * Changes
+    * [record] subcommand:
+      * Additionaly record 'Hugepagesize' in memory usage
+
+## 2021-07-26: PerfMonger 0.13.1
+  * New features
+    * [play] subcommand:
+      * Added --disk-only option to filter results of not interesting disks
+  * Note: v0.13.0 is yanked from rubygems.org
+
+## 2021-07-08: PerfMonger 0.12.1
+  * Bug fixes
+    * [record] subcommand:
+      * --kill waits until a recorder process surely exits
+
+## 2021-05-10: PerfMonger 0.12.0
+  * New features
+    * [record] subcommand:
+      * Add memory usage collection
+
+## 2021-02-10: PerfMonger 0.11.3
+  * Bug fixes
+    * [plot] subcommand:
+      * Fall back to available gnuplot terminal
+  * Changes
+    * [plot] subcommand:
+      * Added --with-gnuplot option for specifying gnuplot binary
 
 ## 2020-03-31: PerfMonger 0.11.2
   * Bug fixes
     * [plot] subcommand:
       * Remove temporary directory correctly
+  * Changes
+    * CI environment
+      * Changed golang version to 1.14
+      * Dropped support of ruby older than 2.4
+      * Added ruby 2.6 and 2.7
 
 ## 2018-05-15: PerfMonger 0.11.1
   * Bug fixes

data/Rakefile

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'rspec/core/rake_task'
 require "bundler/gem_tasks"
 
-task :default => [:spec, :test_core]
+task :default => [:spec, :test_core, :analyze_core]
 
 desc "Run all specs in spec directory"
 RSpec::Core::RakeTask.new(:spec)
@@ -30,19 +30,30 @@ task :go_get do
   sh "go get -u github.com/jroimartin/gocui"
 end
 
-desc "Run tests of core recorder/player"
+desc "Run tests of golang core library"
 task :test_core => [:cross_build_core] do
   Dir.chdir("./core/subsystem") do
+    # check coverage
     sh "go test -v -cover"
-
-    # running static analysis
-    sh "go vet *.go"
   end
+end
 
-  Dir.chdir("./core") do
-    sh "go vet *.go"
+desc "Run static-analysis of golang core library"
+task :analyze_core => [:cross_build_core] do
+  # running static analysis
+  Dir.chdir("./core/subsystem") do
+    ["linux", "darwin"].each do |platform|
+      puts "* ./core/subsystem"
+      sh "go vet perfmonger_#{platform}.go $(ls *.go | grep -v perfmonger_)"
+    end
   end
 
+  Dir["./core", "./core/cmd/*"].each do |dir|
+    Dir.chdir(dir) do
+      puts "* #{dir}"
+      sh "go vet *.go"
+    end
+  end
 end
 
 desc "Removed generated files"

data/core/Makefile

@@ -2,51 +2,50 @@
 # generated by build.sh
 
 GO_DEPS := subsystem/perfmonger_darwin.go subsystem/perfmonger.go subsystem/perfmonger_linux.go subsystem/perfmonger_linux_test.go subsystem/stat.go subsystem/stat_test.go subsystem/usage.go subsystem/usage_test.go utils.go
-GO_SRC := utils.go
 
 .PHONY: all build clean
 
 all: build
 
 
-../lib/exec/perfmonger-recorder_linux_amd64: perfmonger-recorder.go $(GO_DEPS)
-	go build -o $@ perfmonger-recorder.go $(GO_SRC)
+../lib/exec/perfmonger-recorder_linux_amd64: cmd/perfmonger-recorder/perfmonger-recorder.go $(GO_DEPS)
+	cd cmd/perfmonger-recorder && go build -o ../../$@ perfmonger-recorder.go
 
 
-../lib/exec/perfmonger-player_linux_amd64: perfmonger-player.go $(GO_DEPS)
-	go build -o $@ perfmonger-player.go $(GO_SRC)
+../lib/exec/perfmonger-player_linux_amd64: cmd/perfmonger-player/perfmonger-player.go $(GO_DEPS)
+	cd cmd/perfmonger-player && go build -o ../../$@ perfmonger-player.go
 
 
-../lib/exec/perfmonger-viewer_linux_amd64: perfmonger-viewer.go $(GO_DEPS)
-	go build -o $@ perfmonger-viewer.go $(GO_SRC)
+../lib/exec/perfmonger-viewer_linux_amd64: cmd/perfmonger-viewer/perfmonger-viewer.go $(GO_DEPS)
+	cd cmd/perfmonger-viewer && go build -o ../../$@ perfmonger-viewer.go
 
 
-../lib/exec/perfmonger-summarizer_linux_amd64: perfmonger-summarizer.go $(GO_DEPS)
-	go build -o $@ perfmonger-summarizer.go $(GO_SRC)
+../lib/exec/perfmonger-summarizer_linux_amd64: cmd/perfmonger-summarizer/perfmonger-summarizer.go $(GO_DEPS)
+	cd cmd/perfmonger-summarizer && go build -o ../../$@ perfmonger-summarizer.go
 
 
-../lib/exec/perfmonger-plot-formatter_linux_amd64: perfmonger-plot-formatter.go $(GO_DEPS)
-	go build -o $@ perfmonger-plot-formatter.go $(GO_SRC)
+../lib/exec/perfmonger-plot-formatter_linux_amd64: cmd/perfmonger-plot-formatter/perfmonger-plot-formatter.go $(GO_DEPS)
+	cd cmd/perfmonger-plot-formatter && go build -o ../../$@ perfmonger-plot-formatter.go
 
 
-../lib/exec/perfmonger-recorder_darwin_amd64: perfmonger-recorder.go $(GO_DEPS)
-	go build -o $@ perfmonger-recorder.go $(GO_SRC)
+../lib/exec/perfmonger-recorder_darwin_amd64: cmd/perfmonger-recorder/perfmonger-recorder.go $(GO_DEPS)
+	cd cmd/perfmonger-recorder && go build -o ../../$@ perfmonger-recorder.go
 
 
-../lib/exec/perfmonger-player_darwin_amd64: perfmonger-player.go $(GO_DEPS)
-	go build -o $@ perfmonger-player.go $(GO_SRC)
+../lib/exec/perfmonger-player_darwin_amd64: cmd/perfmonger-player/perfmonger-player.go $(GO_DEPS)
+	cd cmd/perfmonger-player && go build -o ../../$@ perfmonger-player.go
 
 
-../lib/exec/perfmonger-viewer_darwin_amd64: perfmonger-viewer.go $(GO_DEPS)
-	go build -o $@ perfmonger-viewer.go $(GO_SRC)
+../lib/exec/perfmonger-viewer_darwin_amd64: cmd/perfmonger-viewer/perfmonger-viewer.go $(GO_DEPS)
+	cd cmd/perfmonger-viewer && go build -o ../../$@ perfmonger-viewer.go
 
 
-../lib/exec/perfmonger-summarizer_darwin_amd64: perfmonger-summarizer.go $(GO_DEPS)
-	go build -o $@ perfmonger-summarizer.go $(GO_SRC)
+../lib/exec/perfmonger-summarizer_darwin_amd64: cmd/perfmonger-summarizer/perfmonger-summarizer.go $(GO_DEPS)
+	cd cmd/perfmonger-summarizer && go build -o ../../$@ perfmonger-summarizer.go
 
 
-../lib/exec/perfmonger-plot-formatter_darwin_amd64: perfmonger-plot-formatter.go $(GO_DEPS)
-	go build -o $@ perfmonger-plot-formatter.go $(GO_SRC)
+../lib/exec/perfmonger-plot-formatter_darwin_amd64: cmd/perfmonger-plot-formatter/perfmonger-plot-formatter.go $(GO_DEPS)
+	cd cmd/perfmonger-plot-formatter && go build -o ../../$@ perfmonger-plot-formatter.go
 
 
 build: ../lib/exec/perfmonger-recorder_linux_amd64 ../lib/exec/perfmonger-player_linux_amd64 ../lib/exec/perfmonger-viewer_linux_amd64 ../lib/exec/perfmonger-summarizer_linux_amd64 ../lib/exec/perfmonger-plot-formatter_linux_amd64 ../lib/exec/perfmonger-recorder_darwin_amd64 ../lib/exec/perfmonger-player_darwin_amd64 ../lib/exec/perfmonger-viewer_darwin_amd64 ../lib/exec/perfmonger-summarizer_darwin_amd64 ../lib/exec/perfmonger-plot-formatter_darwin_amd64

data/core/build.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-READLINK=$(type -p greadlink readlink | head -1)
-cd $(dirname $($READLINK -f $0))
+cd $(cd $(dirname $0); pwd)
 
 if [[ $1 = "-" ]]; then
     # do self build
@@ -33,7 +32,7 @@ fi
 
 set -e
 
-GO_DEPS=$(ls subsystem/*.go; ls utils.go)
+GO_DEPS=$(ls subsystem/*.go utils.go)
 
 makefile=`mktemp`
 
@@ -42,7 +41,6 @@ cat <<EOF > $makefile
 # generated by build.sh
 
 GO_DEPS := $(echo ${GO_DEPS})
-GO_SRC := utils.go
 
 .PHONY: all build clean
 
@@ -62,8 +60,8 @@ for idx in $(seq 0 $((${#TARGET[@]}-1))); do
 
         cat <<EOF >> $makefile
 
-../lib/exec/perfmonger-${subcmd}_${var_GOOS}_${var_GOARCH}: perfmonger-${subcmd}.go \$(GO_DEPS)
-	go build -o \$@ perfmonger-$subcmd.go \$(GO_SRC)
+../lib/exec/perfmonger-${subcmd}_${var_GOOS}_${var_GOARCH}: cmd/perfmonger-${subcmd}/perfmonger-${subcmd}.go \$(GO_DEPS)
+	cd cmd/perfmonger-${subcmd} && go build -o ../../\$@ perfmonger-$subcmd.go
 
 EOF
     done

data/core/{perfmonger-player.go → cmd/perfmonger-player/perfmonger-player.go}

@@ -9,15 +9,19 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
 
 	projson "github.com/hayamiz/go-projson"
+	core "github.com/hayamiz/perfmonger/core"
 	ss "github.com/hayamiz/perfmonger/core/subsystem"
 )
 
 type PlayerOption struct {
-	logfile string
-	color   bool
-	pretty  bool
+	logfile         string
+	color           bool
+	pretty          bool
+	disk_only       string
+	disk_only_regex *regexp.Regexp
 }
 
 var option PlayerOption
@@ -49,10 +53,11 @@ func showInterruptStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cu
 	return nil
 }
 
-func showDiskStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
-	dusage, err := ss.GetDiskUsage(
+func showDiskStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord, disk_only_regex *regexp.Regexp) error {
+	dusage, err := ss.GetDiskUsage1(
 		prev_rec.Time, prev_rec.Disk,
-		cur_rec.Time, cur_rec.Disk)
+		cur_rec.Time, cur_rec.Disk,
+		option.disk_only_regex)
 	if err != nil {
 		return err
 	}
@@ -80,7 +85,22 @@ func showNetStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec
 	return nil
 }
 
-func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord) error {
+func showMemStat(printer *projson.JsonPrinter, cur_rec *ss.StatRecord) error {
+	musage, err := ss.GetMemUsage(cur_rec.Mem)
+	if err != nil {
+		return err
+	}
+
+	printer.PutKey("mem")
+
+	musage.WriteJsonTo(printer)
+
+	return nil
+}
+
+func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss.StatRecord,
+	disk_only_regex *regexp.Regexp) error {
+
 	printer.Reset()
 	if option.pretty {
 		printer.SetStyle(projson.SmartStyle)
@@ -88,6 +108,7 @@ func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss
 	if option.color {
 		printer.SetColor(true)
 	}
+
 	printer.BeginObject()
 	printer.PutKey("time")
 	printer.PutFloatFmt(float64(cur_rec.Time.UnixNano())/1e9, "%.3f")
@@ -108,7 +129,7 @@ func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss
 		}
 	}
 	if cur_rec.Disk != nil {
-		err := showDiskStat(printer, prev_rec, cur_rec)
+		err := showDiskStat(printer, prev_rec, cur_rec, disk_only_regex)
 		if err != nil {
 			return err
 		}
@@ -119,6 +140,12 @@ func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss
 			return err
 		}
 	}
+	if cur_rec.Mem != nil {
+		err := showMemStat(printer, cur_rec)
+		if err != nil {
+			return err
+		}
+	}
 
 	printer.FinishObject()
 
@@ -128,9 +155,25 @@ func showStat(printer *projson.JsonPrinter, prev_rec *ss.StatRecord, cur_rec *ss
 func parseArgs() {
 	flag.BoolVar(&option.color, "color", false, "Use colored JSON output")
 	flag.BoolVar(&option.pretty, "pretty", false, "Use human readable JSON output")
+	flag.StringVar(&option.disk_only, "disk-only", "", "Select disk devices by regex")
 
 	flag.Parse()
 
+	if len(flag.Args()) < 1 {
+		fmt.Fprintln(os.Stderr, "Insufficient argument")
+		os.Exit(1)
+	}
+
+	option.disk_only_regex = nil
+
+	if option.disk_only != "" {
+		var err error
+		option.disk_only_regex, err = regexp.Compile(option.disk_only)
+		if err != nil {
+			panic(err)
+		}
+	}
+
 	option.logfile = flag.Arg(0)
 }
 
@@ -150,7 +193,7 @@ func main() {
 		in = f
 		defer f.Close()
 	}
-	input_reader := newPerfmongerLogReader(in)
+	input_reader := core.NewPerfmongerLogReader(in)
 	dec := gob.NewDecoder(input_reader)
 
 	out = bufio.NewWriter(os.Stdout)
@@ -199,7 +242,7 @@ func main() {
 			panic(err)
 		}
 
-		err = showStat(printer, prev_rec, cur_rec)
+		err = showStat(printer, prev_rec, cur_rec, option.disk_only_regex)
 		if err != nil {
 			printer.Reset()
 			fmt.Fprintln(os.Stderr, "skip by err")

data/core/cmd/perfmonger-plot-formatter/README.md

@@ -0,0 +1,24 @@
+# perfmonger-plot-formatter
+
+`perfmonger-plot-formatter` is a command to generate data files for plotting
+various metrices recorded by `perfmonger-recorder`.
+
+## Notes on memory usage records
+
+### `free(1)` compatible summarization
+
+- total: `MemStat.MemTotal`
+- used: `if (memUsed < 0) { MemStat.MemTotal - MemStat.MemFree } else { memUsed }`
+  - memUsed := `MemStat.MemTotal - MemStat.MemFree - mainCached - MemStat.Buffers`
+  - mainCached := `MemStat.Cached + MemStat.SReclaimable`
+- free: `MemStat.MemFree`
+- shared: `MemStat.Shmem`
+- buffers: `MemStat.Buffers`
+- cache: `mainCached`
+  - mainCached := `MemStat.Cached + MemStat.SReclaimable`
+- available: `MemStat.MemAvailable`
+
+### Additional info
+
+- hugeTotal: `MemStat.HugePages_Total`
+- hugeUsed: `MemStat.HugePages_Total - MemStat.HugePages_Free`

data/core/{perfmonger-plot-formatter.go → cmd/perfmonger-plot-formatter/perfmonger-plot-formatter.go}

@@ -13,13 +13,16 @@ import (
 	"os"
 	"regexp"
 	"sort"
+	"strings"
 
+	"github.com/hayamiz/perfmonger/core"
 	ss "github.com/hayamiz/perfmonger/core/subsystem"
 )
 
 type CmdOption struct {
 	DiskFile        string
 	CpuFile         string
+	MemFile         string
 	PerfmongerFile  string
 	disk_only       string
 	disk_only_regex *regexp.Regexp
@@ -30,8 +33,9 @@ func parseArgs() *CmdOption {
 
 	opt := new(CmdOption)
 
-	flag.StringVar(&opt.DiskFile, "diskfile", "./disk.dat", "Disk performance data file")
-	flag.StringVar(&opt.CpuFile, "cpufile", "./cpu.dat", "CPU performance data file")
+	flag.StringVar(&opt.DiskFile, "diskfile", "./disk.dat", "Disk usage data file for gnuplot")
+	flag.StringVar(&opt.CpuFile, "cpufile", "./cpu.dat", "CPU usage data file for gnuplot")
+	flag.StringVar(&opt.MemFile, "memfile", "./mem.dat", "Memory usage data file for gnuplot")
 	flag.StringVar(&opt.PerfmongerFile, "perfmonger", "", "Perfmonger log file")
 	flag.StringVar(&opt.disk_only, "disk-only",
 		"", "Select disk devices by regex")
@@ -79,6 +83,13 @@ type DiskDatTmpFile struct {
 	Idx    int
 }
 
+type MemDatTmpFile struct {
+	Name   string
+	Path   string
+	File   *os.File
+	Writer *bufio.Writer
+}
+
 type CpuDatTmpFile struct {
 	CoreId int
 	Path   string
@@ -117,6 +128,20 @@ func makeCpuDatTmpFile(coreid int) *CpuDatTmpFile {
 	return ret
 }
 
+func makeMemDatTmpFile() *MemDatTmpFile {
+	ret := new(MemDatTmpFile)
+
+	f, err := ioutil.TempFile("", "perfmonger-mem")
+	if err != nil {
+		panic(err)
+	}
+	ret.File = f
+	ret.Path = f.Name()
+	ret.Writer = bufio.NewWriter(f)
+
+	return ret
+}
+
 func printCoreUsage(writer *bufio.Writer, elapsed_time float64, coreusage *ss.CpuCoreUsage) {
 	writer.WriteString(
 		fmt.Sprintf("%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\n",
@@ -132,6 +157,80 @@ func printCoreUsage(writer *bufio.Writer, elapsed_time float64, coreusage *ss.Cp
 			coreusage.Idle))
 }
 
+func printMemUsage(writer *bufio.Writer, elapsed_time float64, mem *ss.MemStat) {
+	if mem == nil {
+		writer.WriteString("#")
+		writer.WriteString(
+			strings.Join([]string{
+				"elapsed_time",     // 1
+				"mem_total",        // 2
+				"mem_used",         // 3
+				"mem_free",         // 4
+				"buffers",          // 5
+				"cached",           // 6
+				"swap_cached",      // 7
+				"active",           // 8
+				"inactive",         // 9
+				"swap_total",       // 10
+				"swap_free",        // 11
+				"dirty",            // 12
+				"writeback",        // 13
+				"anon_pages",       // 14
+				"mapped",           // 15
+				"shmem",            // 16
+				"slab",             // 17
+				"s_reclaimable",    // 18
+				"s_unreclaim",      // 19
+				"kernel_stack",     // 20
+				"page_tables",      // 21
+				"nfs_unstable",     // 22
+				"bounce",           // 23
+				"commit_limit",     // 24
+				"committed_as",     // 25
+				"anon_huge_pages",  // 26
+				"huge_pages_total", // 27
+				"huge_pages_free",  // 28
+				"huge_pages_rsvd",  // 29
+				"huge_pages_surp",  // 30
+				"hugepagesize"},    // 31
+				"\t"))
+		writer.WriteString("\n")
+	} else {
+		writer.WriteString(fmt.Sprintf("%f\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
+			elapsed_time,
+			mem.MemTotal,
+			mem.MemTotal-mem.MemFree-mem.Buffers-mem.Cached-mem.SReclaimable,
+			mem.MemFree,
+			mem.Buffers,
+			mem.Cached,
+			mem.SwapCached,
+			mem.Active,
+			mem.Inactive,
+			mem.SwapTotal,
+			mem.SwapFree,
+			mem.Dirty,
+			mem.Writeback,
+			mem.AnonPages,
+			mem.Mapped,
+			mem.Shmem,
+			mem.Slab,
+			mem.SReclaimable,
+			mem.SUnreclaim,
+			mem.KernelStack,
+			mem.PageTables,
+			mem.NFS_Unstable,
+			mem.Bounce,
+			mem.CommitLimit,
+			mem.Committed_AS,
+			mem.AnonHugePages,
+			mem.HugePages_Total,
+			mem.HugePages_Free,
+			mem.HugePages_Rsvd,
+			mem.HugePages_Surp,
+			mem.Hugepagesize))
+	}
+}
+
 func main() {
 	opt := parseArgs()
 
@@ -141,7 +240,7 @@ func main() {
 	}
 	defer f.Close()
 
-	input_reader := newPerfmongerLogReader(f)
+	input_reader := core.NewPerfmongerLogReader(f)
 	dec := gob.NewDecoder(input_reader)
 
 	var cheader ss.CommonHeader
@@ -192,6 +291,13 @@ func main() {
 	cpu_writer.WriteString("# All cpu usage\n")
 	cpu_writer.WriteString("# elapsed_time	%usr	%nice	%sys	%iowait	%hardirq	%softirq	%steal	%guest	%idle\n")
 
+	f, err = os.Create(opt.MemFile)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	mem_writer := bufio.NewWriter(f)
+
 	for {
 		prev_rec := &records[curr^1]
 		cur_rec := &records[curr]
@@ -283,6 +389,12 @@ func main() {
 		}
 		printCoreUsage(cpu_writer, prev_rec.Time.Sub(t0).Seconds(), cusage.All)
 
+		if !meta_set {
+			// print column labels
+			printMemUsage(mem_writer, prev_rec.Time.Sub(t0).Seconds(), nil)
+		}
+		printMemUsage(mem_writer, prev_rec.Time.Sub(t0).Seconds(), cur_rec.Mem)
+
 		curr ^= 1
 		meta_set = true
 	}
@@ -331,6 +443,7 @@ func main() {
 		os.Remove(cpu_dat.Path)
 	}
 	cpu_writer.Flush()
+	mem_writer.Flush()
 
 	json_enc := json.NewEncoder(os.Stdout)
 	json_enc.Encode(meta)