server_scripts 0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f1fcfa7acf474ee5e2995af309c00516764836c4
-  data.tar.gz: 3e3567be1f67c495143ecabe583be364235cadf1
+  metadata.gz: 4b72b201ae506a4e955c8b19ce8e0f3b1f3880cb
+  data.tar.gz: 3644cad87a968c5032e6265d371ac299a001f39d
 SHA512:
-  metadata.gz: 8996a1076526a5e44e58b649ae8daf11a520ed7cecbc2c26bd1eff94f2c6a7109e609f3ce995c00d333895303a9c9e6c47242b440306a551e2b803d88701176d
-  data.tar.gz: 99e4065047cba125b88baf6a8f4f7391f926a7b9909ba7136d395f4bc584f1e671c9f65ae5caea0ef2aa5e74076c8ff6637124c0acd13f416c0a357ab57a627e
+  metadata.gz: b72fdf3f6c2340e4599326191c7e37033d04ff4a7a93716401c21fe13bf4143aba8a214f1a63300874d7283335eaf3615abdd81d6fb61546b983ff2f0d405848
+  data.tar.gz: c29c3f522a6325caf5ee70e8ebde1a7edfd9ae62954871a4301b564ab9174a939b87fc3642e0f0204c909e60eb0f1622a0d7a3da16dffe8262f712f0e057a1b0

data/README.md

@@ -6,6 +6,21 @@ The following functionality is provided:
 * Generate job scripts and run batch jobs on TSUBAME 3.0, ABCI and reedbush machines.
 * Parse various kinds of profiling files and generate meaningful output.
 
+<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-generate-toc again -->
+**Table of Contents**
+
+- [server-scripts](#server-scripts)
+- [Usage](#usage)
+    - [ENV variables](#env-variables)
+    - [Writing job scripts](#writing-job-scripts)
+        - [Simple openMPI job script](#simple-openmpi-job-script)
+        - [Intel MPI profiling job script](#intel-mpi-profiling-job-script)
+    - [Parse intel ITAC output](#parse-intel-itac-output)
+    - [Parse starpu worker info](#parse-starpu-worker-info)
+        - [Usage](#usage)
+
+<!-- markdown-toc end -->
+
 # Usage
 
 ## ENV variables
@@ -55,6 +70,8 @@ The intel ITAC tool can be helpful for generating traces of parallel MPI program
 This class can be used for converting an ITAC file to an ideal trace and then generating
 the function profile for obtaining things like the MPI wait time.
 
+### Usage
+
 For extracting the MPI wait time from an ITAC trace, do the following:
 ``` ruby
 require 'server_scripts'
@@ -72,4 +89,21 @@ puts itac.event_time("getrf_start", how: :per_proc, kind: :real)
 
 ## Parse starpu worker info
 
+The `ServerScripts::Parser::StarpuProfile` class has various functions for parsing the
+`*.starpu_profile` files that are generated by starpu with per-worker CPU execution info.
+These can be batch-processed using server\_scripts by specifying a regex that will match the
+profile for each process that produces it. You can either get per-worker or per-process
+information from this.
 
+### Usage
+
+``` ruby
+parser = Parser::StarpuProfile.new("test/artifacts/4_proc_profile_8_*.starpu_profile")
+    
+puts parser.total_time
+puts parser.total_exec_time
+puts parser.total_sleep_time
+puts parser.total_overhead_time
+puts parser.time(event: :total_time, proc_id: 0, worker_id: 4)
+puts parser.proc_time event: :exec_time, proc_id: 2
+```

data/bin/mem_monitor

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'server_scripts'
+
+if ARGV[0] == "--help"
+  puts "Usage:"
+  puts "\tmem_monitor [--pname=pname|--pid=pid] --duration=HH:MM:SS --interval(HH:MM:SS)"
+  exit
+end
+
+pname = nil
+pid = nil
+duration = nil
+interval = nil
+ofile = nil
+ARGV.each do |arg|
+  pname = arg.match(/--pname=(\w+)/)[1] if arg.match(/--pname/)
+  pid = arg.match(/--pid=(\w+)/)[1].to_i if arg.match(/--pid/)
+  duration = arg.match(/--duration=(\w+:\w+:\w+)/)[1] if arg.match(/--duration/)
+  interval = arg.match(/--interval=(\w+:\w+:\w+)/)[1] if arg.match(/--interval/)
+  ofile = arg.match(/--output=(.*)/)[1] if arg.match(/--output/)
+end
+
+if !pname && pid
+  pname = `ps -p #{pid} -o comm=`.strip
+end
+
+if !pid && pname
+  pid = `pidof #{pname}`.strip
+  raise ArgumentError, "Process #{pname} does not exist."
+end
+
+duration = "00:00:30" if !duration
+interval = "00:00:01" if !interval
+
+puts "Using params:"
+puts "\tPID: #{pid}"
+puts "\tPNAME: #{pname}"
+puts "\tDURATION: #{duration}"
+puts "\tINTERVAL: #{interval}"
+puts "\tOFILE: #{ofile}" if ofile
+
+monitor = ServerScripts::MemoryMonitor.new(pid: pid, duration: duration,
+                                           interval: interval)
+monitor.start!
+
+if ofile
+  file = File.open ofile, "w+"
+  file << "VmRSS (MB): #{monitor.vmrss}\n"
+  file << "VmSize (MB): #{monitor.vmsize}\n"
+  file.close
+else
+  puts "VmRSS (MB): #{monitor.vmrss}"
+  puts "VmSize (MB): #{monitor.vmsize}"  
+end
+

data/lib/server_scripts.rb

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 require 'ptools'
+require 'csv'
+
+require 'server_scripts/version'
 
 require 'server_scripts/node_type'
 require 'server_scripts/executor'
@@ -7,7 +10,7 @@ require 'server_scripts/computer'
 require 'server_scripts/parser'
 require 'server_scripts/experiment'
 require 'server_scripts/batch_job'
-require 'server_scripts/version'
+require 'server_scripts/memory_monitor'
 
 module ServerScripts
   class << self

data/lib/server_scripts/memory_monitor.rb

@@ -0,0 +1,56 @@
+module ServerScripts
+  class MemoryMonitor
+    attr_reader :pid
+    attr_reader :pname
+    attr_reader :duration_sec
+    attr_reader :interval_sec
+    attr_reader :vmrss
+    attr_reader :vmsize
+    
+    def initialize(pid: nil, pname: nil, duration: "00:00:30", interval: "00:00:01")
+      @vmrss = []
+      @vmsize = []
+      
+      if pname.nil? && pid
+        @pname = `ps -p #{pid} -o comm=`.strip
+        @pid = pid
+      end
+      
+      if pid.nil? && pname
+        @pname = pname
+        @pid = `pidof #{pname}`.strip.to_i
+      end
+
+      parse_time_intervals duration, interval
+    end
+
+    def start!
+      @duration_sec.times do
+        file = File.open "/proc/#{@pid}/status"
+        file.each_line do |l|
+          rss = l.match(/VmRSS:\s+(\d+)\s+kB/)
+          @vmrss << (rss[1].to_i / 1e3) if rss
+
+          size = l.match(/VmSize:\s+(\d+)\s+kB/)
+          @vmsize << (size[1].to_i / 1e3) if size
+        end
+
+        sleep @interval_sec
+      end
+    end
+
+    private
+
+    def parse_time_intervals dur, inter
+      hours, min, sec = get_hms(dur)
+      @duration_sec = hours * 3600 + min * 60 + sec
+      hours, min, sec = get_hms(inter)
+      @interval_sec = hours * 3600 + min * 60 + sec
+    end
+
+    def get_hms(time)
+      matcher = time.match(/(\d+):(\d+):(\d+)/)
+      [matcher[1].to_i, matcher[2].to_i, matcher[3].to_i]
+    end
+  end # class MemoryMonitor
+end # module ServerScripts

data/lib/server_scripts/parser.rb

@@ -1,2 +1,3 @@
 require 'server_scripts/parser/starpu_profile'
 require 'server_scripts/parser/itac'
+require 'server_scripts/parser/vtune'

data/lib/server_scripts/parser/starpu_profile.rb

@@ -2,7 +2,9 @@ module ServerScripts
   module Parser
     class StarpuProfile
 
+      # The Hash containing the time records of various workers and processes.
       attr_reader :time_hash
+      
       # Specify the regex that will allow finding the profile files for the given starpu
       # processes. Each process will output one file.
       #
@@ -13,24 +15,30 @@ module ServerScripts
         @regex = regex
         @time_hash = {}
         extract_data_from_profiles
+        raise ArgumentError, "could not find any starpu profiles." if @time_hash.empty?
       end
 
+      # Get the sum of total time in seconds summed over all processes and workers.
       def total_time
         extract_from_time_hash :total_time
       end
 
+      # Get the sum of exec total time in seconds summed over all processes and workers.
       def total_exec_time
         extract_from_time_hash :exec_time
       end
-
+      
+      # Get the sum of sleep total time in seconds summed over all processes and workers.
       def total_sleep_time
         extract_from_time_hash :sleep_time        
       end
 
+      # Get the sum of overhead total time in seconds summed over all processes and workers.
       def total_overhead_time
         extract_from_time_hash :overhead_time
       end
 
+      # Get the total time for an event summed over all the workers in a given process.
       def proc_time event:, proc_id:
         time = 0.0
         @time_hash[proc_id].each_value do |thread_info|
@@ -40,6 +48,11 @@ module ServerScripts
         time
       end
 
+      # Get the time in seconds for the given event, worker and process.
+      # 
+      # :event can be one of :total_time, :exec_time, :sleep_time or :overhead_time.
+      # :proc_id should be a number specifying process number.
+      # :worker_id should be a number specifying the worker ID.
       def time event:, proc_id:, worker_id:
         @time_hash[proc_id][worker_id][event]
       end
@@ -68,8 +81,7 @@ module ServerScripts
           if ServerScripts.verbose
             puts "--------------------------------------"
             puts "Reading file #{fname}..."
-          end
-          
+          end          
           proc_id = fname.match(@regex.gsub("*", "(\\d+)"))[1].to_i
           @time_hash[proc_id] = {}
           output = File.read(fname).split("\n")

data/lib/server_scripts/parser/vtune.rb

@@ -0,0 +1 @@
+require 'server_scripts/parser/vtune/hotspots'

data/lib/server_scripts/parser/vtune/hotspots.rb

@@ -0,0 +1,3 @@
+require 'server_scripts/parser/vtune/hotspots/base'
+require 'server_scripts/parser/vtune/hotspots/threads'
+require 'server_scripts/parser/vtune/hotspots/threads/starpu'

data/lib/server_scripts/parser/vtune/hotspots/base.rb

@@ -0,0 +1,20 @@
+module ServerScripts
+  module Parser
+    module VTune
+      module Hotspots
+        class Base
+          CPU_TIME = "CPU Time"
+          CPU_EFFECTIVE_TIME = "CPU Time:Effective Time"
+          CPU_OVERHEAD_TIME = "CPU Time:Overhead Time"
+          CPU_SPIN_TIME = "CPU Time:Spin Time"
+          WAIT_TIME = "Wait Time"
+
+          def initialize fname
+            @threads = {}
+            parse_csv! fname
+          end
+        end
+      end # class Base
+    end # module VTune
+  end # module Parser
+end # module ServerScripts

data/lib/server_scripts/parser/vtune/hotspots/threads.rb

@@ -0,0 +1,72 @@
+module ServerScripts
+  module Parser
+    module VTune
+      # Classes for analysing files showing hotspot analysis.
+      module Hotspots
+        # Parse a file with a hotspots report and grouped by threads. This class
+        # is made for parsing things from a single node, multi threaded execution.
+        # CSV delimiter should be a comma.
+        # 
+        # Example command:
+        # vtune -collect threading -report hotspots -group-by thread -csv-delimiter=, a.out 65536
+        class Threads < Base
+          # Get time for a particular event in a particular thread.
+          def time event:, tid:
+            @threads[tid][event]
+          end
+
+          # Sum of total CPU and wait time.
+          def total_time
+            total_cpu_time + total_wait_time
+          end
+          
+          # Total CPU time of all the threads. Does not include the wait time.
+          def total_cpu_time
+            @total_cpu_time ||= parse_for_event(:cpu_time)
+            @total_cpu_time
+          end
+          
+          # Total Effective CPU time.
+          def total_cpu_effective_time
+            @total_cpu_effective_time ||= parse_for_event(:cpu_effective_time)
+            @total_cpu_effective_time
+          end
+          
+          # Total CPU overhead: sum of CPU Spin Time + CPU Overhead Time
+          def total_cpu_overhead_time
+            @total_cpu_overhead_time ||= parse_for_event(:cpu_overhead_time)
+            @total_cpu_overhead_time
+          end
+          
+          # Total Wait Time.
+          def total_wait_time
+            @total_wait_time ||= parse_for_event(:wait_time)
+            @total_wait_time            
+          end
+
+          private
+
+          def parse_for_event event
+            total = 0.0
+            @threads.each_value do |thread|
+              total += thread[event]
+            end
+            total
+          end
+
+          def parse_csv! fname
+            data = CSV.parse(File.read(fname), headers: true)
+            data.each_with_index do |row, i|
+              @threads[i] = {}
+              @threads[i][:cpu_time] = data[CPU_TIME][i].to_f
+              @threads[i][:cpu_effective_time] = data[CPU_EFFECTIVE_TIME][i].to_f
+              @threads[i][:cpu_overhead_time] = data[CPU_OVERHEAD_TIME][i].to_f +
+                data[CPU_SPIN_TIME][i].to_f
+              @threads[i][:wait_time] = data[WAIT_TIME][i].to_f
+            end
+          end
+        end # class Threads
+      end # module Hotspots
+    end # class ITAC
+  end # module Parser
+end # module ServerScripts

data/lib/server_scripts/parser/vtune/hotspots/threads/starpu.rb

@@ -0,0 +1,22 @@
+module ServerScripts
+  module Parser
+    module VTune
+      module Hotspots
+        class Starpu < Base
+          # Get time for a particular event of a particular worker, master thread
+          # or MPI thread. Specify :tid as :CPU_#ID for worker, :MPI for MPI thread,
+          # and :master for the task submission thread.
+          def time event:, tid:
+            
+          end
+
+          private
+
+          def parse_csv! fname
+            
+          end
+        end # class Starpu
+      end # module Hotspots
+    end # module VTune
+  end # module Parser
+end # module ServerScripts

data/lib/server_scripts/version.rb

@@ -1,3 +1,3 @@
 module ServerScripts
-  VERSION = "0.1"
+  VERSION = "0.1.1"
 end

data/server_scripts.gemspec

@@ -34,6 +34,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
+  spec.bindir = "bin"
+  spec.executables << "mem_monitor"
 
   spec.add_runtime_dependency 'ptools'
   

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: server_scripts
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: 0.1.1
 platform: ruby
 authors:
 - Sameer Deshmukh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-28 00:00:00.000000000 Z
+date: 2020-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ptools
@@ -71,13 +71,15 @@ description: 'Easily write scripts for submitted jobs to various machines.
 '
 email:
 - sameer.deshmukh93@gmail.com
-executables: []
+executables:
+- mem_monitor
 extensions: []
 extra_rdoc_files: []
 files:
 - Gemfile
 - README.md
 - Rakefile
+- bin/mem_monitor
 - lib/server_scripts.rb
 - lib/server_scripts/batch_job.rb
 - lib/server_scripts/computer.rb
@@ -91,10 +93,16 @@ files:
 - lib/server_scripts/executor/valgrind.rb
 - lib/server_scripts/executor/vanilla.rb
 - lib/server_scripts/experiment.rb
+- lib/server_scripts/memory_monitor.rb
 - lib/server_scripts/node_type.rb
 - lib/server_scripts/parser.rb
 - lib/server_scripts/parser/itac.rb
 - lib/server_scripts/parser/starpu_profile.rb
+- lib/server_scripts/parser/vtune.rb
+- lib/server_scripts/parser/vtune/hotspots.rb
+- lib/server_scripts/parser/vtune/hotspots/base.rb
+- lib/server_scripts/parser/vtune/hotspots/threads.rb
+- lib/server_scripts/parser/vtune/hotspots/threads/starpu.rb
 - lib/server_scripts/version.rb
 - server_scripts.gemspec
 homepage: https://github.com/v0dro/server-scripts