darshan-ruby 3.1.1 → 3.1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e67929f858f4576da2761e4ff3f1aac6abe604d9
-  data.tar.gz: d3111becbbe77576d8fa6809d76ebae77a78a826
+  metadata.gz: 9380f68e1e53b12918ce4ba9e3258d2e20f8534c
+  data.tar.gz: 6d75c01826d8f6a29fe3edda3afb04f7aea45aef
 SHA512:
-  metadata.gz: 733b3ef723604deb9d01b5034d983cd902628e861ff6d57aa5b8d7f05425cccdc8e3a040d6df007a1e56235723a5cd18cf7b6fc6113b5b5e73386ab3897873a2
-  data.tar.gz: d30095626d87f7a9f86219d63d6656a15730dd557f7993061f7746e07d4de5b22b37bba080b1c5911bb77598d080ba27d1317aa5ad7fa11d3009d5b602da7a73
+  metadata.gz: 9257397a634d9913b57a94fce91fe435bf3f795e473b7dcd1dce111d76a97f3ea0da24cb761752e0b88fee39aa485d70ca5b48238201ccb2ef76235afac1d2c4
+  data.tar.gz: b6da20235ce6b10b24e3a97db57d0e99a43beb536df390a287eb44583ee0a59d39d393daaad77033636e91c311d384d143790cbce6d4d90fb921193a041faeb8

data/bin/quarshan

@@ -0,0 +1,283 @@
+#!/usr/bin/env ruby
+#######################################################################
+# Author:       Matthieu Dorier
+# Institution:  Argonne National Laboratory
+# Mail:         mdorier [at] anl.gov
+# Date:         October 2016
+#######################################################################
+
+$LOAD_PATH << '../lib'
+$LOAD_PATH << '../ext/darshan'
+
+require 'rubygems' 
+require 'darshan'
+require 'optparse'
+
+# List of Darshan modules
+$darshan_mods = { "POSIX" 	=> Darshan::POSIX, 
+		  "MPI-IO" 	=> Darshan::MPIIO,
+		  "HDF5" 	=> Darshan::HDF5, 
+		  "PNETCDF"	=> Darshan::PNETCDF,
+		  "BG/Q"	=> Darshan::BGQ, 
+		  "LUSTRE"	=> Darshan::LUSTRE,
+		  "STDIO"	=> Darshan::STDIO }
+
+# List of available reduction operations
+$reduction_ops = ['min','max','avg','var','std','med', 'sum']
+
+# Structure for option parsing
+Options = Struct.new(:counters,:query,:reductions,:header,:files,:mod,:qcounters)
+
+# Option parsing class
+class Parser
+
+	def self.parse(options)
+		args = Options.new([],'true',[],false,[],nil,[])
+		opt_parser = OptionParser.new do |opts|
+			opts.banner = "Usage: quarshan file1 [file2 [...]] [options]"
+
+			opts.on("-o COUNTERS", "--output COUNTERS", "Comma-separated list of counters") do |o|
+				args.counters = o.split(',')
+			end
+
+			opts.on("-v", "--verbose", "Display a header before results") do |h|
+				args.header = h
+			end
+
+			opts.on("-s CONDITION", "--select CONDITION", "Record selection condition") do |q|
+				args.query = q
+			end
+
+			opts.on("-r", "--reductions OPERATIONS", "Comma-separated list of reduction operations") do |r|
+				args.reductions = r.split(',')
+			end
+
+			opts.on("-h", "--help", "Prints this help") do
+				puts opts
+				exit
+			end
+		end
+		opt_parser.parse!(options)
+		args.files = options
+		validate(args)
+		return args
+	end
+
+	def self.validate(args)
+		# check that at least one counter is provided, otherwise we can exit right now
+		if(args.counters.size == 0)
+			exit(0)
+		end
+		# check that if reductions are present, there are as many as counters, or 1
+		if(args.reductions.size > 1)
+			if(args.reductions.size != args.counters.size)
+				$stderr << "Number of reduction operations does not match number of output counters\n"
+				exit(-1)
+			end
+		end
+		# if 1 reduction is provided, extend it to all counters
+		if(args.reductions.size == 1 && args.counters.size > 1)
+			args.reductions *= args.counters.size
+		end
+		# check that reduction operation is valid
+		for r in args.reductions
+			if(!$reduction_ops.include?(r))
+				$stderr << "Unknown reduction operation \"" << r << "\"\n"
+				exit(-1)
+			end
+		end
+		# deduce the module based on the first counter's name
+		for name,mod in $darshan_mods
+			counter = args.counters[0]
+			if(mod::NAMES.include?(counter))
+				args.mod = mod
+				break
+			end
+		end
+		# if module not found, error
+		if(args.mod == nil)
+			$stderr << "Could not deduce Darshan module from provided counters\n"
+			exit(-1)
+		end
+		# check that all counters belong to the deduced module
+		for c in args.counters
+			if(!args.mod::NAMES.include?(c))
+				$stderr << "Counter " << c << " does not belong to module "
+				$stderr << "deduced from first provided counter\n"
+				exit(-1)
+			end
+		end
+		# build the list of counters required for the query
+		pass = false
+		qcounters = []
+		query = args.query.dup
+		# tries to evaluate the query and look for unknown names to
+		# find out the list of counters
+		while(!pass)
+			begin
+				str = "x__=1; "+query
+				eval(str)
+				pass=true
+			rescue NameError => e
+				qcounters << e.name.to_s
+				query.gsub!(e.name.to_s,"x__")
+			rescue Exception => other
+				$stderr << "Error while building query counters list\n"
+				$stderr << "(Ruby error is: " << other << ")\n"
+				exit(-1)
+			end
+		end
+		# make sure the query counters are part of the module
+		for q in qcounters
+			if(!args.mod::NAMES.include?(q))
+				$stderr << "Unknown query counter " << q << " in module " << args.mod.name << "\n"
+				exit(-1)
+			end
+		end
+		args.qcounters = qcounters
+	end
+end
+
+# The Query object is initialized with an Option structure created from
+# the command line options. It accumulates results from parsed log files.
+class Query
+
+	def initialize(args)
+		@header = args.header
+		@mod = args.mod
+		@query = args.query
+		@counters = args.counters
+		@qcounters = args.qcounters
+		@counters_idx = [nil]*@counters.size
+		@qcounters_idx = [nil]*@counters.size
+		for i in 0...@counters.size
+			@counters_idx[i] = @mod::NAMES.index(@counters[i])
+		end
+		for i in 0...@qcounters.size
+			@qcounters_idx[i] = @mod::NAMES.index(@qcounters[i])
+		end
+		@reductions = args.reductions
+		@results = []
+		(@counters.size+1).times do
+			@results << []
+		end
+	end
+
+	def read_file(filename)
+		Darshan::LogFile.open(filename) do |file|
+			file.each_module do | m |
+				next if @mod != $darshan_mods[m.name]
+				m.each_record do | r |
+					process_record(r)
+				end
+			end
+		end
+	end
+
+	def process_record(record)
+		return if(!query_satisfied(record))
+		@results[0] << record.name
+		for i in 0...@counters_idx.size
+			@results[i+1] << record.counter(@counters_idx[i])
+		end
+	end
+
+	def query_satisfied(record)
+		str = ""
+		for i in 0...@qcounters.size
+			str += "#{@qcounters[i].downcase}=#{record.counter(@qcounters_idx[i])}; "
+		end
+		str += @query.downcase
+		return eval(str)
+	end
+
+	def reduce
+		return if(@reductions.size == 0)
+		@results[0] = [@results[0].size]
+		for i in 0...@reductions.size
+			eval("@results[i+1] = [#{@reductions[i]}(@results[i+1])]")
+		end
+	end
+
+	def to_s
+		res = ""
+		if(@header)
+			res += "# records\t"
+			for i in 0...@counters.size
+				if(@reductions.size == 0)
+					res += @counters[i]+"\t"
+				else
+					res += @reductions[i]+"("+@counters[i]+")\t"
+				end
+			end
+			res += "\n"
+		end
+		for j in 0...@results[0].size
+			for i in 0...@results.size
+				res += "#{@results[i][j]}\t"
+			end
+			res += "\n"
+		end
+		return res
+	end
+
+	private
+
+	def max(arr)
+		return arr.max
+	end
+
+	def min(arr)
+		return arr.min
+	end
+
+	def var(arr)
+		x = 0.0
+		for i in arr
+			x += i**2
+		end
+		x /= arr.size
+		a = avg(arr)
+		return x - a**2
+	end
+
+	def med(arr)
+		arr2 = arr.sort
+		if(arr.size % 2 == 1)
+			return arr2[arr.size/2]
+		else
+			return (arr2[arr.size/2]+arr2[arr.size/2+1])/2.0
+		end
+	end
+
+	def std(arr)
+		v = var(arr)
+		return Math.sqrt(v)
+	end
+
+	def avg(arr)
+		x = 0.0
+		for i in arr
+			x += i
+		end
+		x /= arr.size
+		return x
+	end
+
+	def sum(arr)
+		x = 0.0
+		for i in arr
+			x += i
+		end
+		return x
+	end
+end
+
+options = Parser.parse(ARGV)
+files = options.files
+query = Query.new(options)
+for f in files
+	query.read_file(f)
+end
+query.reduce
+print query

data/ext/darshan/darshan-ruby.c

@@ -206,6 +206,9 @@ static VALUE rb_darshan_next_record(VALUE self)
 	case DARSHAN_BGQ_MOD:
 		res = Darshan3rb_get_bgq_record(fd,&rec_id);
 		break;
+	case DARSHAN_STDIO_MOD:
+		res = Darshan3rb_get_stdio_record(fd,&rec_id);
+		break;
 	case DARSHAN_LUSTRE_MOD:
 		res = Darshan3rb_get_lustre_record(fd,&rec_id);
 		break;
@@ -242,6 +245,7 @@ void Init_Darshan3rb() {
 	Darshan3rb_init_hdf5();
 	Darshan3rb_init_pnetcdf();
 	Darshan3rb_init_bgq();
+	Darshan3rb_init_stdio();
 	Darshan3rb_init_lustre();
 
 	rb_define_method(cDarshanLogFile,"next_module",

data/ext/darshan/darshan-ruby.h

@@ -16,5 +16,6 @@
 #include "pnetcdf-module.h"
 #include "bgq-module.h"
 #include "lustre-module.h"
+#include "stdio-module.h"
 
 #endif

data/ext/darshan/lustre-module.c

@@ -9,8 +9,8 @@
 extern VALUE cDarshanRecord;
 extern VALUE mDarshan;
 
-VALUE cDarshanLustreRecord;
-VALUE mDarshanLustre;
+VALUE cDarshanLUSTRERecord;
+VALUE mDarshanLUSTRE;
 
 static VALUE Darshan3rb_lustre_get_counter(VALUE self, VALUE index)
 {
@@ -37,19 +37,19 @@ static VALUE Darshan3rb_lustre_get_osts(VALUE self)
 
 void Darshan3rb_init_lustre()
 {
-	mDarshanLustre = rb_define_module_under(mDarshan,"Lustre");
+	mDarshanLUSTRE = rb_define_module_under(mDarshan,"LUSTRE");
 
 	VALUE cnames = rb_ary_new2(LUSTRE_NUM_INDICES);
 	int i;
 	for(i=0; i < LUSTRE_NUM_INDICES; i++) {
 		rb_ary_store(cnames,i,rb_str_new2(mpiio_counter_names[i]));
-		rb_define_const(mDarshanLustre,mpiio_counter_names[i],INT2NUM(i));
+		rb_define_const(mDarshanLUSTRE,mpiio_counter_names[i],INT2NUM(i));
 	}
-	rb_define_const(mDarshanLustre,"NAMES",cnames);
+	rb_define_const(mDarshanLUSTRE,"NAMES",cnames);
 
-	cDarshanLustreRecord = rb_define_class_under(mDarshanLustre,"Record",cDarshanRecord);
-	rb_define_method(cDarshanLustreRecord,"counter",Darshan3rb_lustre_get_counter,1);
-	rb_define_method(cDarshanLustreRecord,"osts",Darshan3rb_lustre_get_osts,0);
+	cDarshanLUSTRERecord = rb_define_class_under(mDarshanLUSTRE,"Record",cDarshanRecord);
+	rb_define_method(cDarshanLUSTRERecord,"counter",Darshan3rb_lustre_get_counter,1);
+	rb_define_method(cDarshanLUSTRERecord,"osts",Darshan3rb_lustre_get_osts,0);
 }
 
 VALUE Darshan3rb_get_lustre_record(darshan_fd fd, darshan_record_id* rec_id)
@@ -58,6 +58,6 @@ VALUE Darshan3rb_get_lustre_record(darshan_fd fd, darshan_record_id* rec_id)
 	int r = mod_logutils[DARSHAN_LUSTRE_MOD]->log_get_record(fd, (void**)&c_record);
 	if(r != 1) return Qnil;
 	*rec_id = c_record->base_rec.id;
-	VALUE rb_record = Data_Wrap_Struct(cDarshanLustreRecord, NULL , free, c_record);
+	VALUE rb_record = Data_Wrap_Struct(cDarshanLUSTRERecord, NULL , free, c_record);
 	return rb_record;
 }

data/ext/darshan/stdio-module.c

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#include "darshan-ruby.h"
+
+extern VALUE cDarshanRecord;
+extern VALUE mDarshan;
+
+VALUE cDarshanSTDIORecord;
+VALUE mDarshanSTDIO;
+
+static VALUE Darshan3rb_stdio_get_rank(VALUE self)
+{
+	struct darshan_stdio_file* c_record = NULL;
+	Data_Get_Struct(self, struct darshan_stdio_file, c_record);
+	if(c_record) return LL2NUM(c_record->base_rec.rank);
+	else return Qnil;
+}
+
+static VALUE Darshan3rb_stdio_get_counter(VALUE self, VALUE index)
+{
+	struct darshan_stdio_file* c_record = NULL;
+	Data_Get_Struct(self,struct darshan_stdio_file, c_record);
+	int i = NUM2INT(index);
+	if((i < 0) || (c_record == NULL)) return Qnil;
+	if(i < STDIO_NUM_INDICES) return LL2NUM(c_record->counters[i]);
+	if(i < STDIO_NUM_INDICES+STDIO_F_NUM_INDICES) return rb_float_new(c_record->fcounters[i-STDIO_NUM_INDICES]);
+	else return Qnil;
+}
+
+void Darshan3rb_init_stdio()
+{
+	mDarshanSTDIO = rb_define_module_under(mDarshan,"STDIO");
+
+	VALUE cnames = rb_ary_new2(STDIO_NUM_INDICES+STDIO_F_NUM_INDICES);
+	int i;
+	for(i=0; i < STDIO_NUM_INDICES; i++) {
+		rb_ary_store(cnames,i,rb_str_new2(stdio_counter_names[i]));
+		rb_define_const(mDarshanSTDIO,stdio_counter_names[i],INT2NUM(i));
+	}
+	for(i=0; i < STDIO_F_NUM_INDICES; i++) {
+		int j = i+STDIO_NUM_INDICES;
+		rb_ary_store(cnames,j,rb_str_new2(stdio_f_counter_names[i]));
+		rb_define_const(mDarshanSTDIO,stdio_f_counter_names[i],INT2NUM(j));
+	}
+	rb_define_const(mDarshanSTDIO,"NAMES",cnames);
+
+	cDarshanSTDIORecord = rb_define_class_under(mDarshanSTDIO,"Record",cDarshanRecord);
+	rb_define_method(cDarshanSTDIORecord,"rank",Darshan3rb_stdio_get_rank,0);
+	rb_define_method(cDarshanSTDIORecord,"counter",Darshan3rb_stdio_get_counter,1);
+}
+
+VALUE Darshan3rb_get_stdio_record(darshan_fd fd, darshan_record_id* rec_id)
+{
+	struct darshan_stdio_file* c_record = NULL;
+	int r = mod_logutils[DARSHAN_STDIO_MOD]->log_get_record(fd, (void**)&c_record);
+	if(r != 1) return Qnil;
+	*rec_id = c_record->base_rec.id;
+	VALUE rb_record = Data_Wrap_Struct(cDarshanSTDIORecord, NULL , free, c_record);
+	return rb_record;
+}

data/ext/darshan/stdio-module.h

@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#include <ruby.h>
+
+void Darshan3rb_init_stdio();
+
+VALUE Darshan3rb_get_stdio_record(darshan_fd fd, darshan_record_id* rec_id);

metadata

@@ -1,22 +1,24 @@
 --- !ruby/object:Gem::Specification
 name: darshan-ruby
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.1.1.1
 platform: ruby
 authors:
 - Matthieu Dorier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-28 00:00:00.000000000 Z
+date: 2016-11-01 00:00:00.000000000 Z
 dependencies: []
 description: Ruby binding to ANL's Darshan library for HPC I/O tracing and analysis
 email: mdorier@anl.gov
-executables: []
+executables:
+- quarshan
 extensions:
 - ext/darshan/extconf.rb
 extra_rdoc_files: []
 files:
+- bin/quarshan
 - ext/darshan/bgq-module.c
 - ext/darshan/bgq-module.h
 - ext/darshan/darshan-ruby.c
@@ -32,6 +34,8 @@ files:
 - ext/darshan/pnetcdf-module.h
 - ext/darshan/posix-module.c
 - ext/darshan/posix-module.h
+- ext/darshan/stdio-module.c
+- ext/darshan/stdio-module.h
 - lib/darshan.rb
 homepage: http://www.mcs.anl.gov/research/projects/darshan/
 licenses: