oar-scripting 1.0.4

data/bin/oar-scripting-graph

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Wed Feb 29 15:14:47 +0100 2012
+#
+
+$: << File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'oar/scripting/application/graph.rb'
+
+OAR::Scripting::Application::Graph.new.run
+

data/lib/oar/scripting.rb

@@ -0,0 +1,64 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Wed Feb 22 17:36:57 +0100 2012
+#
+
+require 'oar/scripting/constants'
+require 'oar/scripting/script'
+require 'oar/scripting/config'
+require 'logger'
+require 'backports'
+
+module OAR
+  module Scripting
+
+    def sh(cmd, options = {})
+      options = options.symbolize_keys
+      options[:stderr] ||= true
+      cmd += options[:stderr] == false ? " 2>/dev/null" : " 2>&1"
+      result = ""
+      Script.logger.info "[command] " + cmd
+      IO.popen(cmd) do |io|
+        result = io.read.strip
+        Script.logger.debug "[result]\n" + result
+      end
+      $? != 0 ? Script.logger.debug("[command_failure]") : Script.logger.debug("[command_success]")
+      result if options[:return]
+    end # def:: sh(cmd)
+
+    def step(name, *args, &block)
+      step = Hash.new
+      raise(ArgumentError, "Step name must be a symbol") unless name.kind_of?(Symbol)
+      if Script.disabled_steps.include? name
+        Script.logger.info "[disable_step]#{name}"
+        return
+      end
+      step = args.first unless args.first.nil?
+      step[:name] = name
+      step[:order] ||= OAR::Scripting::Config[:default_step_order]
+      step[:continue] ||= true
+      step[:proc] = block
+      pre_step = Script.steps.select { |a| a[:name] == name }
+      unless pre_step.empty?
+        if step[:overwrite]
+          Script.logger.info "[step_overwrites] replace #{pre_step.first.inspect} by #{step.inspect}"
+          Script.steps.delete pre_step.first
+          Script.steps << step
+        else
+          Script.logger.info "[step_already_defined] skip #{step.inspect}"
+        end
+      else
+        Script.steps << step
+      end
+    end # def:: step(name)
+
+    def job
+      Script.job
+    end # def:: job
+
+    def oarstat
+      Script.oarstat
+    end # def:: oarstat
+
+  end # module:: Scripting
+end # module:: OAR
+

data/lib/oar/scripting/application.rb

@@ -0,0 +1,47 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Wed Feb 29 15:07:38 +0100 2012
+#
+
+require 'mixlib/cli'
+require 'oar/scripting/constants'
+require 'oar/scripting/config'
+
+class OAR::Scripting::Application
+  include Mixlib::CLI
+
+  def initialize
+    super
+
+    trap("TERM") do
+      OAR::Scripting::Application.fatal!("SIGTERM received, stopping", 1)
+    end
+
+    trap("INT") do
+      OAR::Scripting::Application.fatal!("SIGINT received, stopping", 2)
+    end
+  end # Definition:: initialize
+
+  def run
+    # use parse_options from Mixlib
+    parse_options
+
+    # Run application
+    run_application
+  end # Definition:: run
+
+  def run_application
+    raise "#{self.to_s}: you must override run_application"
+  end # Definition:: run_application
+
+  class << self
+    def fatal!(msg, err = -1)
+      STDERR.puts("FATAL: #{msg}")
+      Process.exit err
+    end # Definition:: fatal!(msg, err = -1)
+
+    def exit!(msg, err = -1)
+      Process.exit err
+    end # Definition:: exit!(msg, err = -1)
+  end
+
+end # class:: OAR::Scripting::Application

data/lib/oar/scripting/application/graph.rb

@@ -0,0 +1,191 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Wed Feb 29 15:09:48 +0100 2012
+#
+
+require 'oar/scripting/application'
+require 'fileutils'
+require 'rubyvis'
+require 'json'
+
+class OAR::Scripting::Application::Graph < OAR::Scripting::Application
+
+  include OAR::Scripting
+
+  CMD_NAME = "oar-scripting-graph"
+
+  option :help,
+    :short        => "-h",
+    :long         => "--help",
+    :description  => "Show this message",
+    :on           => :tail,
+    :boolean      => true,
+    :show_options => true,
+    :exit         => 0
+
+  option :version,
+    :short        => "-v",
+    :long         => "--version",
+    :description  => "Show oar-scripting version",
+    :boolean      => true,
+    :proc         => lambda {|v| puts "#{CMD_NAME}: #{OAR::Scripting::VERSION}"},
+    :exit         => 0
+
+  option :output,
+    :short        => "-o PATH",
+    :long         => "--output PATH",
+    :description  => "Output directory",
+    :default      => Config[:output]
+
+  def initialize
+    super
+  end # def:: initialize
+
+  def run_application
+    Config[:timeout] = 120
+    raise "Directory does not exist : #{config[:output]}" until File.exist?(config[:output])
+    FileUtils.rm_rf File.join(config[:output], "oar-scripting-graph") if File.exist?(File.join(config[:output], "oar-scripting-graph"))
+    FileUtils.mkdir File.join(config[:output], "oar-scripting-graph")
+    FileUtils.mkdir File.join(config[:output], "oar-scripting-graph", "css")
+    FileUtils.mkdir File.join(config[:output], "oar-scripting-graph", "js")
+    FileUtils.mkdir File.join(config[:output], "oar-scripting-graph", "images")
+    FileUtils.mkdir File.join(config[:output], "oar-scripting-graph", "logs")
+    FileUtils.cp(File.join(File.dirname(__FILE__), 'graph', 'sh_emacs.css'),File.join(config[:output], "oar-scripting-graph", "css"))
+    FileUtils.cp(File.join(File.dirname(__FILE__), 'graph', 'sh_main.min.js'),File.join(config[:output], "oar-scripting-graph", "js"))
+    FileUtils.cp(File.join(File.dirname(__FILE__), 'graph', 'sh_ruby.min.js'),File.join(config[:output], "oar-scripting-graph", "js"))
+
+    html_path = File.join(config[:output], "oar-scripting-graph")
+
+    dataset = {}
+
+    %w{prologue epilogue}.each do |script|
+      dataset[script.to_sym] = {}
+      Dir["/var/log/oar/*-#{script}-*.log"].each do |file|
+        log = File.read(file).scan(/^.*\[stats\].*$/).first
+        log.nil? ? next : json = log.gsub(/^.*\[stats\](.*)$/, '\1')
+        stats = JSON.parse json
+        p stats
+        dataset[script.to_sym][:global] ||= []
+        dataset[script.to_sym][:global] << OpenStruct.new( {"x" => stats["job"]["resources_count"], "y" => stats["job"]["host_count"], "z" => stats["duration"]} )
+        dataset[script.to_sym][:steps_list] ||= []
+        stats["steps"].each do |step|
+          dataset[script.to_sym][:steps_list] << step["name"] if !dataset[script.to_sym][:steps_list].include?(step["name"])
+          dataset[script.to_sym][:steps] ||= {}
+          dataset[script.to_sym][:steps][step["name"].to_sym] ||= []
+          dataset[script.to_sym][:steps][step["name"].to_sym] << OpenStruct.new( {"x" => stats["job"]["resources_count"], "y" => stats["job"]["host_count"], "z" => step["duration"]} )
+        end
+      end
+    end
+
+    dataset.each do |script,datas|
+      graph_gen("#{script.to_s.capitalize} duration (all steps)", File.join(html_path, "images", "#{script.to_s}_all.svg"), dataset[script][:global])
+      dataset[script][:steps_list].each do |step|
+        graph_gen("#{script.to_s.capitalize} duration (#{step})", File.join(html_path, "images", "#{script.to_s}_#{step}.svg"), dataset[script][:steps][step.to_sym])
+      end
+    end
+
+    open(File.join(html_path, "index.html"), "w") do |file|
+      file.puts <<-EOH
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+    <html>
+      <head>
+        <title>OAR Scripting graph</title>
+      <link rel="stylesheet" title="Default" href="./css/sh_emacs.css">
+      <script type="text/javascript" src="./js/sh_main.min.js"></script>
+      <script type="text/javascript" src="./js/sh_ruby.min.js"></script>
+      </head>
+    EOH
+      file.puts <<-EOH
+      <body onload="sh_highlightDocument('js/', 'sh_ruby.min.js');">
+        <h1>OAR Scripting graph</h1>
+        <h3>PROLOGUE_EPILOGUE_TIMEOUT=#{Config[:timeout]}</h3>
+        <h4>Generated at #{Time.now}</h4>
+    EOH
+      dataset.each do |script,datas|
+        file.puts "<div>"
+        file.puts "<pre class=\"sh_ruby\">"
+        file.puts File.read("/etc/oar/#{script.to_s}")
+        file.puts <<-EOH
+        </pre>
+        </div>
+        <div class='image'>
+          <!--[if IE]>
+            <embed class='svg' height='600' src='#{File.join("./images", "#{script.to_s}_all.svg")}' width='1100'></embed>
+          <![endif]-->
+          <object class='svg' data='#{File.join("./images", "#{script.to_s}_all.svg")}' height='600' type='image/svg+xml' width='1100'></object>
+        </div>
+    EOH
+        dataset[script][:steps_list].each do |step|
+          file.puts <<-EOH
+        <div class='image'>
+          <!--[if IE]>
+            <embed class='svg' height='600' src='#{File.join("./images", "#{script.to_s}_#{step}.svg")}' width='1100'></embed>
+          <![endif]-->
+          <object class='svg' data='#{File.join("./images", "#{script.to_s}_#{step}.svg")}' height='600' type='image/svg+xml' width='1100'></object>
+        </div>
+    EOH
+        end
+      end
+      file.puts <<-EOH
+      </body>
+    </html>
+    EOH
+    end
+  end # def:: run_application
+
+  def graph_gen(title, file_path, data)
+
+    x_max = 0
+    y_max = 0
+    data.each do |entry|
+      x_max = entry.x if entry.x > x_max
+      y_max = entry.y if entry.y > y_max
+    end
+
+    axis = {  :x => { :title => "Recources count", :max => x_max },
+              :y => { :title => "Nodes count", :max => y_max } }
+
+    panel = { :title  => title,
+              :bottom => 60,
+              :left   => 60,
+              :right  => 40,
+              :top    => 40 }
+
+    w = 1000
+    h = 400
+
+    timeout = Config[:timeout]
+
+    x = pv.Scale.linear(0, axis[:x][:max]).range(0, w)
+    y = pv.Scale.linear(0, axis[:y][:max]).range(0, h)
+
+    c = pv.Scale.log(1, timeout).range("green", "brown")
+
+    # The root panel.
+    vis = pv.Panel.new().width(w).height(h).bottom(panel[:bottom]).left(panel[:left]).right(panel[:right]).top(panel[:top])
+
+    # Main title
+    vis.add(pv.Label).left(((w + panel[:left] + panel[:right]) - panel[:title].length) / 2 ).bottom(h).textAngle(0).text(panel[:title]).font( "35" + "px sans-serif")
+
+    # Axis titles
+    vis.add(pv.Label).left(-20).bottom(h/2 - (axis[:y][:title].length / 2)).textAngle(-Math::PI/2).text(axis[:y][:title]).font( "20" + "px sans-serif")
+    vis.add(pv.Label).left(w/2 - (axis[:x][:title].length / 2)).bottom(-40).textAngle(0).text(axis[:x][:title]).font( "20" + "px sans-serif")
+
+    # Y-axis and ticks.
+    vis.add(pv.Rule).data(y.ticks()).bottom(y).strokeStyle(lambda {|d| d!=0 ? "#eee" : "#000"}).anchor("left").add(pv.Label).visible(lambda {|d|  d > 0 and d < axis[:y][:max]}).text(y.tick_format)
+
+    # X-axis and ticks.
+    vis.add(pv.Rule).data(x.ticks()).left(x).stroke_style(lambda {|d| d!=0 ? "#eee" : "#000"}).anchor("bottom").add(pv.Label).visible(lambda {|d|  d > 0 and d < axis[:x][:max]}).text(x.tick_format)
+
+    #/* The dot plot! */
+    vis.add(pv.Panel).data(data).add(pv.Dot).left(lambda {|d| x.scale(d.x)}).bottom(lambda {|d| y.scale(d.y)}).stroke_style(lambda {|d| c.scale(d.z)}).fill_style(lambda {|d| c.scale(d.z).alpha(0.2)}).shape_size(lambda {|d| d.z * 5}).anchor("center").add(pv.Label).visible(lambda {|d| d.z > 30}).textAngle(0).text(lambda {|d| "%0.1f" %  d.z});
+
+    vis.render()
+
+    open(file_path, "w") do |file|
+      file.puts vis.to_svg
+    end
+
+  end # def:: graph_gen(title, file_path)
+
+
+end # class:: OAR::Scripting::Application::Graph < OAR::Scripting::Application

data/lib/oar/scripting/config.rb

@@ -0,0 +1,33 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Wed Feb 29 15:31:57 +0100 2012
+#
+
+class OAR::Scripting::Config
+
+  @@config = Hash.new
+
+  @@config[:output]             = File.expand_path(ENV["PWD"])
+  @@config[:oar_conf_path]      = "/etc/oar"
+  @@config[:prologue_d_path]    = File.join @@config[:oar_conf_path], "prologue.d"
+  @@config[:epilogue_d_path]    = File.join @@config[:oar_conf_path], "epilogue.d"
+  @@config[:default_step_order] = 50
+  @@config[:log_path]           = "/var/log/oar"
+
+  def self.[](opt)
+    @@config[opt.to_sym]
+  end # def:: self.[](opt)
+
+  def self.[]=(opt, value)
+    @@config[opt.to_sym] = value
+  end # def:: self.[]=(opt, value)
+
+  def self.method_missing(method_symbol, *args)
+    @@config[method_symbol] = args[0]
+    @@config[method_symbol]
+  end # def:: self.method_missing(method_symbol), *args)
+
+  def self.inspect
+    @@config
+  end # def:: self.inspect
+
+end # class:: OAR::Scripting::Config

data/lib/oar/scripting/constants.rb

@@ -0,0 +1,11 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Wed Feb 22 17:38:51 +0100 2012
+#
+
+module OAR
+  module Scripting
+    GEM       = "oar-scripting"
+    VERSION   = "1.0.4"
+  end # module:: Scripting
+end # module:: OAR
+

data/lib/oar/scripting/epilogue.rb

@@ -0,0 +1,15 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Thu Feb 23 16:43:48 +0100 2012
+#
+
+require 'oar/scripting/script'
+
+class OAR::Scripting::Epilogue < OAR::Scripting::Script
+
+  def initialize
+    @@type = :epilogue
+    super
+  end # def:: initialize
+
+end # class:: OAR::Scripting::Epilogue < OAR::Scripting::Script
+

data/lib/oar/scripting/prologue.rb

@@ -0,0 +1,15 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Thu Feb 23 16:43:39 +0100 2012
+#
+
+require 'oar/scripting/script'
+
+class OAR::Scripting::Prologue < OAR::Scripting::Script
+
+  def initialize
+    @@type = :prologue
+    super
+  end # def:: initialize
+
+end # class:: OAR::Scripting::Prologue < OAR::Scripting::Script
+

data/lib/oar/scripting/script.rb

@@ -0,0 +1,103 @@
+# Author:: Pascal Morillon (<pascal.morillon@irisa.fr>)
+# Date:: Mon Feb 27 13:42:10 +0100 2012
+#
+
+require 'oar/scripting'
+require 'json'
+
+class OAR::Scripting::Script
+
+  def self.init(type)
+    @@resources = []
+    @@start_at = Time.new
+    @@job = getargs
+    @@type ||= type
+    @@logger ||= Logger.new(File.join(OAR::Scripting::Config[:log_path], "#{@@job[:id]}-#{@@type.to_s}-#{@@job[:user]}.log"))
+    @@logger.info "[begin]"
+    @@stats = { "job" => @@job, "steps" => [] }
+    @@steps = []
+    @@disabled_steps = []
+  end # def:: initialize
+
+  def self.load_steps
+    dir = OAR::Scripting::Config["#{@@type}_d_path".to_sym]
+    if File.exist? dir
+      Dir[File.join dir, "*.rb"].each do |file|
+        load file
+      end
+    end
+  end # def:: self.load_scripts
+
+  def self.getargs
+    job = { :id         => ARGV[0],
+            :user       => ARGV[1],
+            :nodesfile  => ARGV[2] }
+    begin
+      File.open(job[:nodesfile]).each { |line| @@resources << line.chomp }
+    rescue
+      # let @@resources empty
+    end
+    job[:resources_count] = @@resources.length
+    job[:host_count] = @@resources.uniq.length
+    job
+  end # def:: getargs
+
+  def self.logger
+    @@logger
+  end # def:: logger
+
+  def self.type
+    @@type
+  end # def:: type
+
+  def self.job
+    @@job
+  end # def:: self.job
+
+  def self.steps
+    @@steps
+  end # def:: self.steps
+
+  def self.oarstat
+    @@oarstat ||= JSON.parse(%x[oarstat -f -j @@job[:id] -J])[@@job[:id]]
+  end # def:: self.oarstat
+
+  def self.execute
+    @@steps.sort! { |a,b| a[:order] <=> b[:order] }
+    @@steps.each do |step|
+      @@logger.info "[begin_step]#{step[:name]}"
+      start = Time.new
+      begin
+        step[:proc].call
+      rescue Exception => e
+        @@logger.debug "[Error] step #{step[:name]} failed (describe in #{step[:file]}"
+        @@logger.debug e.message
+        @@logger.debug e.backtrace
+        raise unless step[:continue]
+      end
+      @@logger.info "[end_step]#{step[:name]}"
+      @@stats["steps"] << { "name" => step[:name].to_s, "duration" => (Time.now - start), "order" => step[:order] }
+    end
+    @@logger.info "[end]"
+    @@stats["duration"] = Time.now - @@start_at
+    @@logger.info "[stats]" + @@stats.to_json
+  end # def:: execute
+
+  def self.stats
+    @@stats
+  end # def:: self.stats
+
+  def self.disabled_steps
+    @@disabled_steps
+  end # def:: self.disabled_steps
+
+  def self.disable_steps(steps)
+    steps = [steps] unless steps.class == Array
+    @@disabled_steps += steps
+    steps2disable = Script.steps.select { |step| steps.include? step[:name] }
+    Script.logger.info "[disable_loaded_steps]#{steps2disable.inspect}"
+    @@steps -= steps2disable
+  end # def:: self.disable_steps(steps)
+
+end # class:: OAR::Scripting::Script
+

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: oar-scripting
+version: !ruby/object:Gem::Version
+  version: 1.0.4
+  prerelease: 
+platform: ruby
+authors:
+- Pascal Morillon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mixlib-cli
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: rubyvis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.5
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.2.1
+description: OAR helper library for prologue/epilogue scripting
+email: pascal.morillon@irisa.fr
+executables:
+- oar-scripting-graph
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+files:
+- README.rdoc
+- lib/oar/scripting/application/graph.rb
+- lib/oar/scripting/application.rb
+- lib/oar/scripting/config.rb
+- lib/oar/scripting/constants.rb
+- lib/oar/scripting/epilogue.rb
+- lib/oar/scripting/prologue.rb
+- lib/oar/scripting/script.rb
+- lib/oar/scripting.rb
+- bin/oar-scripting-graph
+homepage: https://github.com/pmorillon/oar-scripting
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- - lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -239453471554793347
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -239453471554793347
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: OAR helper library for prologue/epilogue scripting
+test_files: []