omf_ec 6.0.0.pre.3 → 6.0.0.pre.4

data/exp_repo/system/exp/eventlib.rb

@@ -1,96 +0,0 @@
-#
-# Copyright (c) 2006-2010 National ICT Australia (NICTA), Australia
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-#
-# = eventlib.rb 
-#
-# == Description
-#
-# This Ruby file contains various common Event declarations, which the EC will 
-# load before the user's experiment file
-#
-
-#
-# This provide some default Event definition that the user may use in his 
-# experiment, without having to worry about how to define them.
-# To start one of the event monitoring defined here, the user need to associate 
-# at least one block of tasks to it, using the onEvent() OEDL call.
-# See the OEDL documentation for more info and a tutorial
-#
-
-defEvent(:ALL_UP_AND_INSTALLED) do |event|
-  node_status = allGroups.state("status/@value")
-  app_status = allGroups.state("apps/app/status/@value")
-  if allEqual(node_status, "UP") && allEqual(app_status, "INSTALLED.OK")
-    event.fire 
-  end
-end
-
-defEvent(:ALL_UP) do |event|
-  node_status = allGroups.state("status/@value")
-  event.fire if allEqual(node_status, "UP")
-end
-
-defEvent(:ALL_INTERFACE_UP) do |event|
-  iface_status = allGroups.state("net/*/*/current/@status")
-  #info "TDEBUG - #{if_status.join(" ")}"
-  event.fire if allEqual(iface_status, "CONFIGURED.OK")
-end
-
-defEvent(:EXPERIMENT_DONE) do |event|
-  exp_status = Experiment.state("status/text()")
-  event.fire if allEqual(exp_status, "DONE")
-end
-
-onEvent(:EXPERIMENT_DONE, true) do |event|
-  Experiment.close
-end
-
-defEvent(:INTERRUPT, 1) do |event|
-  exp_status = Experiment.state("status/text()")
-  event.fire if allEqual(exp_status, "INTERRUPTED")
-end
-
-onEvent(:INTERRUPT) do |event|
-  MObject.info(:INTERRUPT, "\n\nUser issued an Interruption. Stopping the experiment now! Please wait...\n")
-  Experiment.done
-end
-
-defEvent(:NO_USER_DEFINED_EVENTS) do |event|
-  if Experiment.running? && !Experiment.disconnection_allowed?
-    if Event.empty?(:ignore => [:EXPERIMENT_DONE, :NO_USER_DEFINED_EVENTS])
-      event.fire 
-    end
-  end
-end
-
-onEvent(:NO_USER_DEFINED_EVENTS) do |event|
-  warn " "
-  warn "Warning!!! Your experiment has no user-defined events!"
-  warn "It is likely that nothing will happen from now one..."
-  warn "Press CTRL-C only ONCE to stop your experiment.\n"
-  # An alternative... not sure what is the best here, for now use the above
-  # warn "Closing down your experiment now..."
-  # Experiment.done
-end
-
-
-

data/exp_repo/system/exp/imageNode.rb

@@ -1,283 +0,0 @@
-#
-# Copyright (c) 2006-2008 National ICT Australia (NICTA), Australia
-#
-# Copyright (c) 2004-2008 WINLAB, Rutgers University, USA
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-#
-# = imageNode.rb
-#
-# == Description
-#
-# This file describes the Experiment that is used to image the nodes within OMF.
-# (In OMF, loading a disk image on a node is treated as an 'experiment' itself)
-#
-
-# Define the experiment properties
-Experiment.name = "imageNode"
-Experiment.project = "Orbit::Admin"
-defProperty('nodes', 'system:topo:all', "Nodes to image")
-defProperty('image', 'baseline.ndz', "Image to load on nodes")
-defProperty('domain', "#{OConfig.domain}", "Domain of the nodes to image")
-defProperty('outpath', "/tmp", "Path where to place the topology files resulting from this image")
-defProperty('outprefix', "#{Experiment.ID}", "Prefix to use for the topology files resulting from this image")
-# The following value of 1200sec for timeout is based on trial imaging experiments 
-defProperty('timeout', 1200, "Stop the imaging process <timeout> sec after the last node has powered up")
-
-# Define some constants
-MSG_CHECKINFAILED = <<TEXT
-# This creates a Topology with the resources which did not check in to the load experiment.
-# These nodes may have failed to boot into the PXE image or start the RC from within.
-#
-TEXT
-MSG_IMAGEFAILED = <<TEXT
-# This creates a Topology with the resources for which the image loading failed.
-# On these resources, the 'frisbee' client may have raised an error during its execution.
-# Please check the EC log file for the 'frisbee' client error message.
-#
-TEXT
-MSG_TIMEOUT = <<TEXT
-# This creates a Topology with the resources for which the image loading timed out.
-# These nodes did not finish imaging before the timeout limit.
-# Most of the time this is caused by some disk or network problems.
-#
-TEXT
-MSG_SUCCESS = <<TEXT
-# This creates a Topology with the resources which have successfully been imaged.
-#
-TEXT
-MESSAGES = {:checkinfailed => MSG_CHECKINFAILED, :imagefailed => MSG_IMAGEFAILED, :timeout => MSG_TIMEOUT, :success => MSG_SUCCESS}
-
-
-#
-# First of all, do some checks...
-# - check if the requested image really exists on the Repository
-#
-#url = "#{OConfig[:ec_config][:frisbee][:url]}/checkImage?img=#{prop.image.value}&domain=#{prop.domain.value}"
-#response = NodeHandler.service_call(url, "Image does not exist")
-response = OMF::Services.frisbee.checkImage(:img => "#{prop.image.value}", :domain => "#{prop.domain.value}")
-if response.elements[1].name != "OK"
-  MObject.error("Frisbee Service Call", response.root.text)
-  Experiment.done
-  exit
-end
-# - check if timeout value from command line is really an integer
-if (prop.timeout.value.to_i == 0)
-  MObject.error("The timeout value '#{prop.timeout.value}' is not an integer!")
-  MObject.error("Check command line syntax.")
-  Experiment.done
-  exit -1
-end
-
-@allNodes = []
-
-#
-# Define the group of node to image and set them into PXE boot 
-#
-defGroup('image', prop.nodes) {|ns|
-   ns.image = "pxe-5.4"
-   ns.each { |n| @allNodes << n }
-}
-
-OMF::Services.pxe.setBootImageNS(:ns => "#{@allNodes.map{|n| n.to_s }.join(',')}", :domain => "#{prop.domain.value}")
-
-def clearPXE
-  OMF::Services.pxe.clearBootImageNS(:ns => "#{@allNodes.map{|n| n.to_s }.join(',')}", :domain => "#{prop.domain.value}")
-end
-
-def outputTopologyFile(type, nset)
-  begin
-    filename = "#{prop.outpath.value}/#{prop.outprefix.value}-topo-#{type}.rb"
-    toponame = "#{prop.outprefix.value}-topo-#{type}"
-    
-    # we need to put the array back into the original order
-    # since nodes were added in the order they signed in
-    sortedNodes = Array.new(@allNodes)
-    sortedNodes.delete_if {|n| !nset.include?(n) }
-    sortedNodes.map!{|n| n.to_s }
-    File.open(filename, "w") do |f|
-      f.puts("# Topology name: #{toponame}", "# ")
-      f.puts(MESSAGES[type])
-      f.print("defTopology('#{toponame}', '")
-      f.print(sortedNodes.join(","))
-      f.puts "')"
-    end
-    return filename
-  rescue Exception => err
-    MObject.warn("exp", "Could not write result topology file: '#{filename}' (#{err})")
-    MObject.warn("exp", "(Most probably imaging was OK, but result file could not be created)")
-  end
-  return nil
-end
-
-#
-# Every 10s check the state of the imaging process and report accordingly
-#
-everyNS('image', 10) { |ns|
-  nodesUp = 0
-  nodesDone = 0
-  nodesWithError = 0
-  nodesWithErrorList = []
-  nodesWithSuccessList = []
-  nodesPendingList = []
-  nodeCnt = 0
-  progMax = 0
-  progMin = 100
-  nodeMin = nil
-  progSum = 0
-  startupDelayMax = 0
-  report = true
-  notDone = true
-  lastUpTime = 0
-
-  ns.each { |n|
-    nodeCnt += 1
-    if n.isUp
-#      nodesUp += 1
-      prog = n.match('apps/*/*/progress/text()')[-1].to_s.to_i
-#puts n
-#puts n.match('apps/*/*/progress')
-      progSum += prog
-      progMax = prog if prog > progMax
-      if prog < progMin
-        progMin = prog
-        nodeMin = n
-      end
-      startupDelay = n.checkedInAt.to_i - n.poweredAt.to_i
-      startupDelayMax = startupDelay if startupDelay > startupDelayMax
-      if n.poweredAt.to_i > lastUpTime
-        lastUpTime = n.poweredAt.to_i
-      end	
-
-      status = n.match('apps/*/status/')[0].to_s
-      nodesDone += 1 if status =~ /DONE/
-      if status =~ /DONE.ERR/
-        nodesWithError += 1
-        nodesWithErrorList << n
-      elsif status =~ /DONE.OK/
-        nodesWithSuccessList << n
-      else
-        nodesPendingList << n
-      end
-    else
-      # wait with reporting until everybody is up
-      report = false
-    end
-  }
-  if report
-    progAvg = nodeCnt > 0 ? progSum / nodeCnt : 0
-    stats = "#{progMin}/#{progAvg}/#{progMax}"
-    prog = "#{nodesDone}/#{nodesWithError}/#{nodeCnt}"
-    timeLeft = lastUpTime+prop.timeout.value-Time.now.to_i
-    info "Progress(#{prog}): #{stats} min(#{nodeMin})/avg/max (#{startupDelayMax}) - Timeout: #{timeLeft} sec."
-
-    if (nodesDone >= nodeCnt) || ((lastUpTime+prop.timeout.value) < Time.now.to_i)
-      # we are done
-      info " ----------------------------- "
-      info " Imaging Process Done " 
-      nodesWhichNeverCheckedIn = @allNodes - nodesWithErrorList - nodesPendingList - nodesWithSuccessList
-      if (l = nodesWhichNeverCheckedIn.length) > 0
-        f = outputTopologyFile(:checkinfailed, nodesWhichNeverCheckedIn)
-        info " #{l} node#{"s" if l>1} failed to check in - Topology saved in '#{f}'"
-      end
-      if (l = nodesWithErrorList.length) > 0
-        f = outputTopologyFile(:imagefailed, nodesWithErrorList)
-        info " #{l} node#{"s" if l>1} failed to image the disk - Topology saved in '#{f}'"
-      end
-      if (l = nodesPendingList.length) > 0
-        f = outputTopologyFile(:timeout, nodesPendingList)
-        info " #{l} node#{"s" if l>1} timed out - Topology saved in '#{f}'"
-      end
-      if (l = nodesWithSuccessList.length) > 0
-        f = outputTopologyFile(:success, nodesWithSuccessList)
-        info " #{l} node#{"s" if l>1} successfully imaged - Topology saved in '#{f}'"
-      end
-      info " ----------------------------- "
-      ns.stopImageServer("#{prop.image.value}", "#{prop.domain.value}")
-      Experiment.done
-      notDone = false
-    end
-  end
-  notDone
-}
-
-onEvent(:INTERRUPT) {
-  clearPXE
-}
-
-#
-# When all the nodes in the above group are Up, then start loading the image on them
-#
-onEvent(:ALL_UP) {
-  clearPXE
-  # Only execute imaging if node set is not empty!
-  # (e.g. in rare occasions no node managed to come up and register to EC, when this
-  # happens, we need to exit quietly from this 'onEvent(:ALL_UP)')
-  nodeCount = 0
-  group('image').each { |n|
-    nodeCount += 1
-  }
-  if (nodeCount != 0)
-    group('image').loadImage(Experiment.property('image'), "#{prop.domain.value}")
-  end
-}
-
-
-##defURL('/progress') {|req, res|
-#OMF::Common::Web.mapProc('/progress') {|req, res|
-#  body = []
-#  body << %q{
-#<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
-#<html>
-#  <head>
-#    <meta http-equiv="refresh" content="10">
-#    <title>Imaging Progress :: Orbit</title>
-#    <link href="resource/stylesheet/grid.css" type="text/css" rel="stylesheet"/>
-#  </head>
-#
-#  <body>
-#      <h1>Imaging Progress</h1>
-#      <table class="grid">
-#}
-#  # TODO: port this to OMF 5.4
-#  # (1 .. OConfig[:tb_config][:default][:y_max]).each { |y|
-#  #   body << "<tr class='row'>"
-#  #   (1 .. OConfig[:tb_config][:default][:x_max]).each { |x|
-#  #     n = Node[x,y]
-#  #     if (n == nil)
-#  #       body << "<td class='cell'></td>"
-#  #     elsif (n.isUp)
-#  #       body << "<td class='cell cell-up'>"
-#  #       progress = n.match('apps/builtin[1]/properties/progress/text()').to_s
-#  #       if (progress != nil)
-#  #         body << "<div class='cell-progress' style='width: #{progress}%'></div>"
-#  #       end
-#  #       body << "</td>"
-#  #     else
-#  #       body << "<td class='cell cell-down'></td>"
-#  #     end
-#  #   }
-#  #   body << "</tr>"
-#  # }
-#  body << "</table></body></html>"
-#  res.body = body.to_s
-#  res['Content-Type'] = "text/html"
-#}
-#

data/exp_repo/system/exp/reset.rb

@@ -1,21 +0,0 @@
-#
-# This experiment sends a reset to all the resources which are associated to it
-# Typically, one would run this experiment with a specific Experiment ID, with 
-# the following command:
-#   omf-5.4 exec -e your_experiment_id system:exp:reset
-#
-property.resetDelay = 1
-property.resetTries = 0
-info "---------------"
-info "  "
-info "  This management experiment will send a RESET to all resources"
-info "  in the experiment with the ID: #{Experiment.ID}"
-info "  (Typically one would run this experiment with the following"
-info "   command: omf-5.4 exec -e your_experiment_id system:exp:reset)"
-info "  "
-info "  Please wait a few second now..."
-info "  "
-info "---------------"
-defGroup("reset", "")
-ECCommunicator.instance.send_reset
-Experiment.done

data/exp_repo/system/exp/saveNode.rb

@@ -1,99 +0,0 @@
-#
-# Copyright (c) 2006-2009 National ICT Australia (NICTA), Australia
-#
-# Copyright (c) 2004-2009 WINLAB, Rutgers University, USA
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-#
-# = saveNode.rb
-#
-# == Description
-#
-# This file describes the Experiment that is used to save the disk image of a node within OMF.
-# (In OMF, saving the disk image of a node is treated as an 'experiment' itself)
-#
-#
-
-Experiment.name = "imageNode"
-Experiment.project = "Orbit::Admin"
-
-defProperty('node', 'omf.nicta.node1', "Node to save image of")
-defProperty('pxe', '1.1.6', "PXE version to use")
-defProperty('domain', "#{OConfig.domain}", "Domain of the node to save")
-defProperty('started', 'false', "internal flag")
-
-OMF::Services.pxe.setBootImageNS(:ns => "#{prop.node.value}", :domain => "#{prop.domain.value}")
-
-def clearPXE
-  OMF::Services.pxe.clearBootImageNS(:ns => "#{prop.node.value}", :domain => "#{prop.domain.value}")
-end
-
-#
-# Define nodes used in experiment
-#
-defGroup('save', Experiment.property('node')) {|n|
-  #n.pxeImage("#{prop.domain.value}", setPXE=true)
-  n.image = "pxe-5.4"
-}
-
-everyNS('save', 10) { |ns|
-  notDone = true
-  ns.each { |n|
-    status = n.match('apps/*/status/')[0].to_s
-    if status =~ /DONE/
-      notDone = false
-      if status =~ /DONE.ERR/
-        info("- Saving disk image of '#{n}' finished with ERRORS!")
-        info("  Check the log file (probably disk read error on the node)")
-      else
-        info("- Saving disk image of '#{n}' finished with success.")
-      end
-      info("- Saving process completed at: #{Time.now}")
-      info " "
-    end
-  }
-  Experiment.done if notDone == false
-  notDone
-}
-
-everyNS('save', 10) { |ns|
-  ns.each { |n|
-    status = n.match('apps/*/status/')[0].to_s
-    if status =~ /STARTED/
-      if prop.started.value == "false"
-        prop.started = "true"
-        info " "
-        info "- Saving process started at: #{Time.now}"
-        info "  (this may take a while depending on the size of your image)"
-      end
-    end
-  }
-}
-
-onEvent(:INTERRUPT) {
-  clearPXE
-}
-
-onEvent(:ALL_UP) {
-  clearPXE
-  group('save').each { |n|
-    n.saveImage
-  }
-}

data/exp_repo/system/exp/stat.rb

@@ -1,49 +0,0 @@
-# This experiment prints the CMC power state of each node in the given topology
-
-# empty group to stop OMF from complaining
-# & prevent it from powering on nodes
-defGroup("stat","")
-
-defProperty('nodes', 'system:topo:all', 'Nodes to query')
-defProperty('summary', false, 'Show a summary instead of details')
-
-topo = nil
-begin
-  # if 'nodes' is a topology file, try to load it
-  topo = Topology["#{prop.nodes}"]
-rescue
-  # if not create a new topology here
-  topo = Topology.create(nil, "#{prop.nodes}")
-end
-
-tuples = []
-
-$stderr.print " Talking to the CMC service, please wait"
-
-topo.eachNode {|n|
-  tuples << ["#{n.to_s}", "#{OMF::Services.cmc.status(n.to_s, OConfig.domain).
-    first_element("NODE_STATUS/detail/node").attributes['state']}"]
-  $stderr.print "."
-}
-
-puts
-puts "-----------------------------------------------"
-puts " Domain: #{OConfig.domain}"
-if property.summary.value
-  on = off = unknown = 0
-  tuples.each {|t|
-    on += 1 if t[1] == "POWERON"
-    off += 1 if t[1] == "POWEROFF"
-    unknown += 1 if t[1] == "UNKNOWN"
-  }
-  puts " Number of nodes in 'Power ON' state:\t#{on}"
-  puts " Number of nodes in 'Power OFF' state:\t#{off}"
-  puts " Number of nodes in 'Unknown' state:\t#{unknown}"
-else
-  tuples.each {|t|
-    puts " Node: #{t[0]}   \t State: #{t[1]}"
-  }
-end
-puts "-----------------------------------------------"
-
-Experiment.done

data/exp_repo/system/exp/stdlib.rb

@@ -1,122 +0,0 @@
-#
-# Copyright (c) 2006-2009 National ICT Australia (NICTA), Australia
-# Copyright (c) 2004-2009 WINLAB, Rutgers University, USA
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-#
-# = stdlib.rb 
-#
-# == Description
-#
-# This Ruby file contains various general declarations, which the EC will load 
-# before the user's experiment file
-#
-# These declarations implement the process that loop, check and wait that all 
-# the required nodes are UP, before proceeding with the remaining of the 
-# experiment
-#
-
-# Define some properties that we will use in this 
-defProperty('resetDelay', 210,
-            "Time to wait before assuming that node didn't boot")
-defProperty('resetTries', 1, 
-            "Number of reset tries before declaring node dead")
-# keeps track of how many times a node has been reset
-ResetCount = {}
-
-# 
-# This declaration calls the 'everyNS' loop defined in handlerCommand.rb
-# This declared bloc will be executed for all the existing node sets ('*') 
-# every 10sec. This loop will stop when the bloc returns 'false', which will 
-# happen when all the nodes are UP
-#
-everyNS('*', 10) { |ns|
-
-  # First check if the experiment has not been interrupted
-  #exp_status = Experiment.state("status/text()")
-  #return true if allEqual(exp_status, "INTERRUPTED")
-
-
-  # Now check if we are done with adding node in that experiment
-  # If not, we skip the checks and loop again in 10sec
-  if NodeSet.frozen?
-    # Yes, we are done adding nodes...
-    nodesDown = []
-    nodeCnt = 0
-    # For each node in this Node Set, check if it is UP
-    # Check that for 'resetDelay' time, if no sucess, reset the node
-    # Do only 'resetTries' number of resets before giving up on a node
-    ns.eachNode { |n|
-      nodeCnt += 1
-      if ! n.isUp
-        nodesDown << n
-        poweredAt = n.poweredAt
-        if (poweredAt.kind_of?(Time))
-          startupDelay = Time.now - poweredAt
-          if (startupDelay > Experiment.property('resetDelay').value)
-            count = ResetCount[n] = (ResetCount[n] || 0) + 1
-            if (count <= prop.resetTries.value)
-              MObject.info('stdlib', "Resetting node ", n)
-              n.reset()
-            else
-              MObject.warn('stdlib', "Giving up on node ", n)
-              if NodeHandler.ALLOWMISSING
-                Topology.removeNode(n)
-                if Topology.empty?
-                  MObject.info("stdlib", " ")
-                  MObject.info("stdlib", "No resources available for this "+
-                               "experiment. Closing the experiment now!" )
-                  MObject.info("stdlib", "Please wait and ignore remaining "+
-                               "messages...")
-                  MObject.info("stdlib", " ")
-                  Experiment.close
-                end
-              else
-                MObject.error('stdlib', "One or more nodes failed to check in " +
-                  "to your experiment. To ignore and continue, use the '-a'"+
-                " flag. Exiting now.")
-                Experiment.close
-              end
-            end
-          end
-        end
-      end
-    }
-    # Check the number of nodes still not UP...
-    nodesDownCnt = nodesDown.length
-    if nodesDownCnt > 0
-      MObject.info('stdlib', "Waiting for nodes (Up/Down/Total): "+
-        "#{nodeCnt-nodesDownCnt}/#{nodesDownCnt}/#{nodeCnt}",
-        " - (still down: ", nodesDown[0..2].join(','),")")
-    end
-    # Check if the experiment is not interrupted
-    exp_status = Experiment.state("status/text()")
-    if allEqual(exp_status, "INTERRUPTED")
-      false 
-    else 
-      # The experiment is running, stop looping if all the nodes are UP!
-      nodesDownCnt > 0
-    end
-  else
-    # We have not finished adding nodes to this experiment, 
-    # loop and check again in 10sec
-    true
-  end
-}