perfmonger 0.10.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +3 -0
  3. data/.travis.yml +9 -8
  4. data/HOWTO.md +0 -1
  5. data/NEWS +47 -3
  6. data/README.md +77 -27
  7. data/Rakefile +20 -9
  8. data/core/Makefile +10 -18
  9. data/core/build.sh +2 -2
  10. data/core/perfmonger-player.go +90 -40
  11. data/core/perfmonger-plot-formatter.go +18 -4
  12. data/core/perfmonger-recorder.go +22 -2
  13. data/core/perfmonger-summarizer.go +20 -14
  14. data/core/perfmonger-viewer.go +164 -0
  15. data/core/subsystem/Makefile +4 -0
  16. data/core/subsystem/perfmonger_linux.go +95 -0
  17. data/core/subsystem/perfmonger_linux_test.go +40 -0
  18. data/core/subsystem/stat.go +70 -0
  19. data/core/subsystem/stat_test.go +9 -0
  20. data/core/subsystem/usage.go +223 -66
  21. data/core/subsystem/usage_test.go +62 -32
  22. data/{bin → exe}/perfmonger +0 -0
  23. data/lib/perfmonger/command/fingerprint.rb +26 -1
  24. data/lib/perfmonger/command/live.rb +19 -0
  25. data/lib/perfmonger/command/play.rb +16 -0
  26. data/lib/perfmonger/command/plot.rb +25 -9
  27. data/lib/perfmonger/command/record.rb +1 -1
  28. data/lib/perfmonger/command/record_option.rb +16 -0
  29. data/lib/perfmonger/command/server.rb +1 -1
  30. data/lib/perfmonger/version.rb +1 -1
  31. data/misc/werker-box/Dockerfile +34 -0
  32. data/misc/werker-box/build-push.sh +7 -0
  33. data/perfmonger.gemspec +2 -1
  34. data/spec/data/busy100.pgr.played +3 -3
  35. data/spec/fingerprint_spec.rb +1 -1
  36. data/spec/live_spec.rb +2 -3
  37. data/spec/perfmonger_spec.rb +1 -1
  38. data/spec/play_spec.rb +1 -1
  39. data/spec/plot_spec.rb +16 -1
  40. data/spec/record_spec.rb +10 -1
  41. data/spec/spec_helper.rb +28 -3
  42. data/spec/stat_spec.rb +2 -2
  43. data/spec/summary_spec.rb +1 -1
  44. data/wercker.yml +29 -16
  45. metadata +28 -10
@@ -1,7 +1,6 @@
1
1
  package subsystem
2
2
 
3
3
  import (
4
- "bytes"
5
4
  "encoding/json"
6
5
  "math"
7
6
  "regexp"
@@ -9,6 +8,8 @@ import (
9
8
  "strings"
10
9
  "testing"
11
10
  "time"
11
+
12
+ projson "github.com/hayamiz/go-projson"
12
13
  )
13
14
 
14
15
  func isValidJson(byt []byte) bool {
@@ -89,7 +90,7 @@ func TestGetCoreUsage(t *testing.T) {
89
90
  usage.Steal == 0 &&
90
91
  usage.Guest == 0 &&
91
92
  usage.GuestNice == 0) {
92
- t.Error("usage is not 0%, want 0%")
93
+ t.Error("usage is not 0 percent, want 0 percent")
93
94
  }
94
95
 
95
96
  // should return error if c1.Uptime() is larger than c2.Uptime()
@@ -120,10 +121,14 @@ func TestGetCoreUsage(t *testing.T) {
120
121
  t.Errorf("usage.Sys = %v, want 25.0", usage.User)
121
122
  }
122
123
 
123
- buf := bytes.NewBuffer([]byte{})
124
- usage.WriteJsonTo(buf)
125
- if !isValidJson(buf.Bytes()) {
126
- t.Errorf("Invalid JSON: %s", buf.String())
124
+ printer := projson.NewPrinter()
125
+ usage.WriteJsonTo(printer)
126
+ if str, err := printer.String(); err != nil {
127
+ t.Errorf("failed to print JSON")
128
+ } else {
129
+ if !isValidJson([]byte(str)) {
130
+ t.Errorf("Invalid JSON: %s", str)
131
+ }
127
132
  }
128
133
  }
129
134
 
@@ -207,15 +212,20 @@ func TestGetCpuUsage(t *testing.T) {
207
212
  t.Errorf("usage.Sys = %v, want 25.0", usage.CoreUsages[1].User)
208
213
  }
209
214
 
210
- buf := bytes.NewBuffer([]byte{})
211
- usage.WriteJsonTo(buf)
212
- if !isValidJson(buf.Bytes()) {
213
- t.Errorf("Invalid JSON: %s", buf.String())
215
+ printer := projson.NewPrinter()
216
+ usage.WriteJsonTo(printer)
217
+ str, err := printer.String()
218
+ if err != nil {
219
+ t.Errorf("failed printing JSON")
220
+ } else {
221
+ if !isValidJson([]byte(str)) {
222
+ t.Errorf("Invalid JSON: %s", str)
223
+ }
214
224
  }
215
225
 
216
226
  assertHasKey := func(key_path string) {
217
- if !jsonHasKey(buf.Bytes(), key_path) {
218
- t.Errorf("%v is not present in JSON:\n%v", key_path, buf.String())
227
+ if !jsonHasKey([]byte(str), key_path) {
228
+ t.Errorf("%v is not present in JSON:\n%v", key_path, str)
219
229
  }
220
230
  }
221
231
  assertHasKey("num_core")
@@ -291,10 +301,15 @@ func TestDiskUsage(t *testing.T) {
291
301
  t.Errorf("sda.RdSectors = %v, want %v", (*usage)["sda"].RdSectors, 350)
292
302
  }
293
303
 
294
- buf := bytes.NewBuffer([]byte{})
295
- usage.WriteJsonTo(buf)
296
- if !isValidJson(buf.Bytes()) {
297
- t.Errorf("invalid json: %s", buf.String())
304
+ printer := projson.NewPrinter()
305
+ usage.WriteJsonTo(printer)
306
+ str, err := printer.String()
307
+ if err != nil {
308
+ t.Errorf("failed printing JSON")
309
+ } else {
310
+ if !isValidJson([]byte(str)) {
311
+ t.Errorf("invalid json: %s", str)
312
+ }
298
313
  }
299
314
 
300
315
  d1.Entries = append(d1.Entries, NewDiskStatEntry())
@@ -347,15 +362,20 @@ func TestDiskUsage(t *testing.T) {
347
362
  t.Errorf("total.RdSectors = %v, want %v", (*usage)["total"].RdSectors, 350)
348
363
  }
349
364
 
350
- buf = bytes.NewBuffer([]byte{})
351
- usage.WriteJsonTo(buf)
352
- if !isValidJson(buf.Bytes()) {
353
- t.Errorf("invalid json: %s", buf.String())
365
+ printer = projson.NewPrinter()
366
+ usage.WriteJsonTo(printer)
367
+ str, err = printer.String()
368
+ if err != nil {
369
+ t.Errorf("failed printing JSON")
370
+ } else {
371
+ if !isValidJson([]byte(str)) {
372
+ t.Errorf("invalid json: %s", str)
373
+ }
354
374
  }
355
375
 
356
376
  assertHasKey := func(key_path string) {
357
- if !jsonHasKey(buf.Bytes(), key_path) {
358
- t.Errorf("%v is not present in JSON:\n%v", key_path, buf.String())
377
+ if !jsonHasKey([]byte(str), key_path) {
378
+ t.Errorf("%v is not present in JSON:\n%v", key_path, str)
359
379
  }
360
380
  }
361
381
  assertHasKey("devices")
@@ -428,10 +448,15 @@ func TestGetNetUsage(t *testing.T) {
428
448
  t.Errorf("lo.RxPacketsPerSec = %v, want %v", (*usage)["lo"].RxPacketsPerSec, 100.0/interval)
429
449
  }
430
450
 
431
- buf := bytes.NewBuffer([]byte{})
432
- usage.WriteJsonTo(buf)
433
- if !isValidJson(buf.Bytes()) {
434
- t.Errorf("invalid json: %s", buf.String())
451
+ printer := projson.NewPrinter()
452
+ usage.WriteJsonTo(printer)
453
+ str, err := printer.String()
454
+ if err != nil {
455
+ t.Errorf("failed printing JSON")
456
+ } else {
457
+ if !isValidJson([]byte(str)) {
458
+ t.Errorf("invalid json: %s", str)
459
+ }
435
460
  }
436
461
 
437
462
  n1.Entries = append(n1.Entries, NewNetStatEntry())
@@ -483,15 +508,20 @@ func TestGetNetUsage(t *testing.T) {
483
508
  (100.0+150.0)/interval)
484
509
  }
485
510
 
486
- buf = bytes.NewBuffer([]byte{})
487
- usage.WriteJsonTo(buf)
488
- if !isValidJson(buf.Bytes()) {
489
- t.Errorf("invalid json: %s", buf.String())
511
+ printer = projson.NewPrinter()
512
+ usage.WriteJsonTo(printer)
513
+ str, err = printer.String()
514
+ if err != nil {
515
+ t.Errorf("failed printing JSON")
516
+ } else {
517
+ if !isValidJson([]byte(str)) {
518
+ t.Errorf("invalid json: %s", str)
519
+ }
490
520
  }
491
521
 
492
522
  assertHasKey := func(key_path string) {
493
- if !jsonHasKey(buf.Bytes(), key_path) {
494
- t.Errorf("%v is not present in JSON:\n%v", key_path, buf.String())
523
+ if !jsonHasKey([]byte(str), key_path) {
524
+ t.Errorf("%v is not present in JSON:\n%v", key_path, str)
495
525
  }
496
526
  }
497
527
  assertHasKey("devices")
File without changes
@@ -57,6 +57,10 @@ EOS
57
57
  save_proc_info()
58
58
  end
59
59
 
60
+ do_with_message("Saving numactl info") do
61
+ save_numactl_info()
62
+ end
63
+
60
64
  do_with_message("Saving IRQ info") do
61
65
  save_irq_info()
62
66
  end
@@ -109,9 +113,14 @@ EOS
109
113
  save_nvme_info()
110
114
  end
111
115
 
112
-
113
116
  ## Collect vendor specific info
114
117
 
118
+ # https://aws.amazon.com/jp/code/ec2-instance-metadata-query-tool/
119
+ if find_executable("ec2-metadata")
120
+ do_with_message("Saving EC2 metadata info") do
121
+ save_ec2metadata_info()
122
+ end
123
+ end
115
124
 
116
125
  # LSI MegaRAID
117
126
  megacli_bin = "/opt/MegaRAID/MegaCli/MegaCli64"
@@ -241,6 +250,22 @@ EOS
241
250
  end
242
251
  end
243
252
 
253
+ def save_numactl_info()
254
+ numactl_bin = find_executable("numactl")
255
+
256
+ File.open("#{@output_dir}/numactl.log", "w") do |f|
257
+ f.puts(`#{numactl_bin} --hardware`)
258
+ end
259
+ end
260
+
261
+ def save_ec2metadata_info()
262
+ ec2metadata_bin = find_executable("ec2-metadata")
263
+
264
+ File.open("#{@output_dir}/ec2-metadata.log", "w") do |f|
265
+ f.puts(`#{ec2metadata_bin} --all`)
266
+ end
267
+ end
268
+
244
269
  def save_lsblk_info()
245
270
  lsblk_bin = find_executable("lsblk")
246
271
 
@@ -7,10 +7,29 @@ module PerfMonger
7
7
  module Command
8
8
 
9
9
  class LiveOption < RecordOption
10
+ def initialize
11
+ super
12
+
13
+ @color = false
14
+ @parser.on("-c", "--color", "Use colored JSON output") do
15
+ @color = true
16
+ end
17
+
18
+ @pretty
19
+ @parser.on("--pretty", "Use human readable JSON output") do
20
+ @pretty = true
21
+ end
22
+
23
+ end
24
+
10
25
  def make_command
11
26
  cmd = super()
12
27
  @player_bin = ::PerfMonger::Command::CoreFinder.player()
13
28
  cmd += ["-player-bin", @player_bin]
29
+ cmd << "-color" if @color
30
+ cmd << "-pretty" if @pretty
31
+
32
+ cmd
14
33
  end
15
34
  end
16
35
 
@@ -16,6 +16,16 @@ Usage: perfmonger play [options] LOG_FILE
16
16
  Options:
17
17
  EOS
18
18
 
19
+ @color = false
20
+ @parser.on("-c", "--color", "Use colored JSON ouptut") do
21
+ @color = true
22
+ end
23
+
24
+ @pretty = false
25
+ @parser.on("-p", "--pretty", "Use human readable JSON ouptut") do
26
+ @pretty = true
27
+ end
28
+
19
29
  end
20
30
 
21
31
  def parse_args(argv)
@@ -46,6 +56,12 @@ EOS
46
56
  end
47
57
 
48
58
  cmd = [@player_bin]
59
+ if @color
60
+ cmd << "-color"
61
+ end
62
+ if @pretty
63
+ cmd << "-pretty"
64
+ end
49
65
  cmd << @logfile
50
66
 
51
67
  Process.exec(*cmd)
@@ -24,11 +24,13 @@ EOS
24
24
  @output_type = 'pdf'
25
25
  @output_prefix = ''
26
26
  @save_gpfiles = false
27
+ @disk_only = nil
27
28
  @disk_only_regex = nil
28
29
  @disk_plot_read = true
29
30
  @disk_plot_write = true
30
31
  @disk_numkey_threshold = 10
31
32
  @plot_iops_max = nil
33
+ @gnuplot_bin = `which gnuplot`
32
34
  end
33
35
 
34
36
  def parse_args(argv)
@@ -46,6 +48,10 @@ EOS
46
48
  @output_dir = dir
47
49
  end
48
50
 
51
+ @parser.on('--with-gnuplot GNUPLOT_BIN') do |gnuplot_bin|
52
+ @gnuplot_bin = gnuplot_bin
53
+ end
54
+
49
55
  @parser.on('-T', '--output-type TYPE', 'Available: pdf, png') do |typ|
50
56
  unless ['pdf', 'png'].include?(typ)
51
57
  puts("ERROR: non supported image type: #{typ}")
@@ -78,6 +84,7 @@ EOS
78
84
  end
79
85
 
80
86
  @parser.on('--disk-only REGEX', "Select disk devices that matches REGEX") do |regex|
87
+ @disk_only = regex
81
88
  @disk_only_regex = Regexp.compile(regex)
82
89
  end
83
90
 
@@ -117,13 +124,14 @@ EOS
117
124
  end
118
125
 
119
126
  def run(argv)
120
- parse_args(argv)
121
127
  unless system('which gnuplot >/dev/null 2>&1')
122
128
  puts("ERROR: gnuplot not found")
123
129
  puts(@parser.help)
124
130
  exit(false)
125
131
  end
126
132
 
133
+ parse_args(argv)
134
+
127
135
  unless system('gnuplot -e "set terminal" < /dev/null 2>&1 | grep pdfcairo >/dev/null 2>&1')
128
136
  puts("ERROR: pdfcairo is not supported by installed gnuplot")
129
137
  puts("ERROR: PerfMonger requires pdfcairo-supported gnuplot")
@@ -139,7 +147,13 @@ EOS
139
147
  @cpu_dat = File.expand_path("cpu.dat", @tmpdir)
140
148
 
141
149
  meta_json = nil
142
- IO.popen([formatter_bin, "-perfmonger", @data_file, "-cpufile", @cpu_dat, "-diskfile", @disk_dat], "r") do |io|
150
+ cmd = [formatter_bin, "-perfmonger", @data_file, "-cpufile", @cpu_dat,
151
+ "-diskfile", @disk_dat]
152
+ if @disk_only_regex
153
+ cmd << "-disk-only"
154
+ cmd << @disk_only
155
+ end
156
+ IO.popen(cmd, "r") do |io|
143
157
  meta_json = io.read
144
158
  end
145
159
  if $?.exitstatus != 0
@@ -151,6 +165,8 @@ EOS
151
165
  plot_disk(meta)
152
166
  plot_cpu(meta)
153
167
 
168
+ FileUtils.rm_rf(@tmpdir)
169
+
154
170
  true
155
171
  end
156
172
 
@@ -389,10 +405,10 @@ EOS
389
405
  nr_cols = nr_cpu_factors.select do |x|
390
406
  x <= Math.sqrt(nr_cpu)
391
407
  end.max
392
- nr_cols ||= Math.sqrt(nr_cpu).ceil
408
+ nr_cols = 1
393
409
  nr_rows = nr_cpu / nr_cols
394
410
 
395
- plot_height = 8
411
+ plot_height = ([nr_cpu, 8].min) + ([nr_cpu - 8, 0].max) * 0.5
396
412
 
397
413
  if nr_rows == 1
398
414
  plot_height /= 2.0
@@ -422,14 +438,14 @@ EOS
422
438
  end
423
439
 
424
440
  all_gpfile.puts <<EOS
425
- set title 'cpu #{cpu_idx}' offset 0.0,-0.7 font 'Arial,16'
441
+ set title 'cpu #{cpu_idx}' offset -61,-3 font 'Arial,16'
426
442
  unset key
427
443
  set origin #{xpos}, #{ypos}
428
444
  set size #{1.0/nr_cols}, #{(1.0 - legend_height)/nr_rows}
429
- set rmargin 0.5
430
- set lmargin 3.5
431
- set tmargin 1.3
432
- set bmargin 1.3
445
+ set rmargin 2
446
+ set lmargin 12
447
+ set tmargin 0.5
448
+ set bmargin 0.5
433
449
  set xtics offset 0.0,0.5
434
450
  set ytics offset 0.5,0
435
451
  set style fill noborder
@@ -19,7 +19,7 @@ class RecordCommand < BaseCommand
19
19
  def run(argv)
20
20
  @argv, @option = PerfMonger::Command::RecordOption.parse(argv)
21
21
 
22
- session_file = File.expand_path(sprintf("perfmonger-%s-session.pid", Etc.getlogin),
22
+ session_file = File.expand_path(sprintf("perfmonger-%s-session.pid", Etc.getpwuid.name),
23
23
  Dir.tmpdir)
24
24
  begin
25
25
  session_pid = File.read(session_file).to_i
@@ -63,6 +63,12 @@ class RecordOption
63
63
  if @no_intr
64
64
  cmd << "-no-intr"
65
65
  end
66
+ if @no_net
67
+ cmd << "-no-net"
68
+ end
69
+ if @no_mem
70
+ cmd << "-no-mem"
71
+ end
66
72
  if @devices.size > 0
67
73
  cmd << "-disks"
68
74
  cmd << @devices.join(",")
@@ -98,6 +104,8 @@ class RecordOption
98
104
  @no_cpu = false
99
105
  @no_disk = false
100
106
  @no_intr = true
107
+ @no_net = true
108
+ @no_mem = false
101
109
  @devices = []
102
110
  @logfile = "perfmonger.pgr"
103
111
  @logfile_set = false
@@ -141,6 +149,14 @@ class RecordOption
141
149
  @no_cpu = true
142
150
  end
143
151
 
152
+ @parser.on('--no-net', 'Suppress recording network usage') do
153
+ @no_net = false
154
+ end
155
+
156
+ @parser.on('--no-mem', 'Suppress recording memory usage') do
157
+ @no_mem = true
158
+ end
159
+
144
160
  @parser.on('-l', '--logfile FILE') do |file|
145
161
  @logfile = file
146
162
  @logfile_set = true
@@ -114,7 +114,7 @@ EOS
114
114
  _, record_option = PerfMonger::Command::RecordOption.parse(@record_cmd_args)
115
115
 
116
116
  # find perfmonger command
117
- perfmonger_bin = File.expand_path('bin/perfmonger', PerfMonger::ROOTDIR)
117
+ perfmonger_bin = File.expand_path('exe/perfmonger', PerfMonger::ROOTDIR)
118
118
  if ! File.executable?(perfmonger_bin)
119
119
  puts("ERROR: perfmonger not found!")
120
120
  exit(false)
@@ -1,3 +1,3 @@
1
1
  module PerfMonger
2
- VERSION = "0.10.1"
2
+ VERSION = "0.12.0"
3
3
  end
@@ -0,0 +1,34 @@
1
+ FROM golang:1.14
2
+
3
+ WORKDIR /app
4
+
5
+ ## install packages
6
+ RUN apt-get update
7
+ RUN apt-get install -y build-essential libncurses-dev libreadline-dev libssl-dev gnuplot git gnupg2
8
+ RUN apt-get install -y autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm-dev gawk libsqlite3-dev libtool sqlite3 libgmp-dev # ruby build dep
9
+
10
+ ## get source code
11
+ RUN git clone https://github.com/hayamiz/perfmonger .
12
+
13
+ ## install RVM
14
+ RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
15
+ RUN curl -O https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer
16
+ RUN curl -O https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer.asc
17
+ RUN gpg --verify rvm-installer.asc
18
+ RUN bash rvm-installer stable
19
+ RUN ln -sf /bin/bash /bin/sh
20
+
21
+ ## install ruby
22
+ RUN bash -l -c "rvm install 2.4.9"
23
+ RUN bash -l -c "rvm use 2.4.9 && gem install bundler && bundle"
24
+
25
+ RUN bash -l -c "rvm install 2.5.7"
26
+ RUN bash -l -c "rvm use 2.5.7 && gem install bundler && bundle"
27
+
28
+ RUN bash -l -c "rvm install 2.6.5"
29
+ RUN bash -l -c "rvm use 2.6.5 && gem install bundler && bundle"
30
+
31
+ RUN bash -l -c "rvm install 2.7.0"
32
+ RUN bash -l -c "rvm use 2.7.0 && gem install bundler && bundle"
33
+
34
+ CMD true