xp5k 0.0.5 → 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 29af43dd9e56a5562a4f327a6f54be5591ef9f98
+  data.tar.gz: b2003bfefbaad6a7779dd396a9cc03ce85a4bc79
+SHA512:
+  metadata.gz: d8b9f40bf89d6739869ca6633f813b8cb5ac0fc6f64dfd80a903de6cc0c91abb7111c5a7b0f3a7721f254270a25f59e59a888a540a52fe40bf8c9cc27804b1db
+  data.tar.gz: 521647dc1f924275374e140371ce75fa8e93dcfc4d5ece3f9141076ffa531da6d436aa99f3b9425d40685fb9e4b98ebea83b46b4fe1d56504a876e4a1a80028c

data/lib/xp5k/version.rb

@@ -1,3 +1,3 @@
 module XP5K
-  VERSION='0.0.5'
+  VERSION='0.0.6'
 end

data/lib/xp5k/xp.rb

@@ -19,6 +19,7 @@ module XP5K
       @roles              = []
       @links_deployments  = {"jobs" => {}, "roles" => {}}
       @deployed_nodes     = {"jobs" => {}, "roles" => {}}
+      @retries            = options[:retries] || 3
       @starttime          = Time.now
       @logger             = options[:logger] || Logger.new(STDOUT)
 
@@ -44,41 +45,33 @@ module XP5K
       self.todeploy << deployment_hash
     end
 
-    def deploy
+    def deploy()
+      # prepare assigned_nodes, goals and retries value
       self.todeploy.each do |x|
-        x[:nodes] ||= []
-        # Get assigned resources to deploy
+        x[:assigned_nodes] = []
         x[:jobs].each do |jobname|
           job = self.job_with_name(jobname)
-          x[:nodes] += job["assigned_nodes"]
+          self.deployed_nodes["jobs"][jobname] = []
+          x[:assigned_nodes] += job["assigned_nodes"]
         end
         x[:roles].each do |rolename|
-          x[:nodes] += role_with_name(rolename).servers
+          role = role_with_name(rolename)
+          self.deployed_nodes["roles"][rolename] = []
+          x[:assigned_nodes] += role.servers
         end
-        site = x[:site]
-        x.delete(:site)
-        x.delete(:roles)
-        x.delete(:jobs)
-        deployment = @connection.root.sites[site.to_sym].deployments.submit(x)
-        self.deployments << deployment
-        # update links_deployments
-        update_links_deployments(deployment["uid"], x)
-      end
-      logger.info "Waiting for all the deployments to be terminated..."
-      finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
-      while (!finished)
-        sleep 10
-        print "."
-        self.deployments.each do |deployment|
-          deployment.reload
-        end
-        finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
+        # initially all nodes have to be deployed
+        x[:nodes] = x[:assigned_nodes]
+        # set goal
+        x[:goal] = set_goal(x[:goal], x[:assigned_nodes].length)
+        # set retries
+        x[:retry] ||= false
+        x[:retries] ||= x[:retry]?@retries:1
       end
-      update_deployed_nodes()
-      update_cache()
-      print(" [#{green("OK")}]\n")
+      internal_deploy(@retries)
+      print_deploy_summary
     end
 
+
     def define_job(job_hash)
       self.jobs2submit << job_hash
 
@@ -86,7 +79,7 @@ module XP5K
         datas = JSON.parse(File.read(".xp_cache"))
         uid = datas["jobs"].select { |x| x["name"] == job_hash[:name] }.first["uid"]
         unless uid.nil?
-          job = @connection.root.sites[job_hash[:site].to_sym].jobs["#{uid}".to_sym]
+          job = @connection.root.sites[job_hash[:site].to_sym].jobs(:query => { :user => @connection.config.options[:username] })["#{uid}".to_sym]
           if (not job.nil? or job["state"] == "running")
             j = job.reload
             self.jobs << j
@@ -94,7 +87,7 @@ module XP5K
           end
         end
         # reload last deployed nodes
-        self.deployed_nodes = datas["deployed_nodes"] 
+        self.deployed_nodes = datas["deployed_nodes"]
       end
 
     end
@@ -104,19 +97,35 @@ module XP5K
         job = self.job_with_name(job2submit[:name])
         if job.nil?
           job = @connection.root.sites[job2submit[:site].to_sym].jobs.submit(job2submit)
-          #self.jobs << { :uid => job.properties['uid'], :name => job.properties['name'] }
           update_cache
-          logger.info "Waiting for the job #{job["name"]} ##{job['uid']} to be running at #{job2submit[:site]}..."
-          while job.reload["state"] != "running"
-            print(".")
-            sleep 3
-          end
-          self.jobs << job 
-          create_roles(job, job2submit) unless job2submit[:roles].nil?
-          print(" [#{green("OK")}]\n")
+          logger.info "Job \"#{job['name']}\" submitted with id ##{job['uid']} at #{job2submit[:site]}"
+          self.jobs << job
         else
-          logger.info "Job #{job["name"]} already submitted ##{job["uid"]}"
+          logger.info "Job \"#{job["name"]}\" already submitted ##{job["uid"]}"
+        end
+      end
+      update_cache()
+    end
+
+    def wait_for_jobs
+      logger.info "Waiting for running state"
+      ready = false
+      jobs_status = []
+      until ready
+        self.jobs.each.with_index do |job, id|
+          jobs_status[id] = job.reload["state"]
+          case jobs_status[id]
+          when "running"
+            create_roles(job, jobs2submit[id]) unless jobs2submit[id][:roles].nil?
+            logger.info "Job #{job['uid']} is running"
+          when /terminated|error/
+            logger.info "Job #{job['uid']} is terminated"
+          else
+            logger.info "Job #{job['uid']} will be scheduled at #{Time.at(job['scheduled_at'].to_i).to_datetime}"
+          end
         end
+        ready = true if jobs_status.uniq == ["running"]
+        sleep 3
       end
       update_cache()
     end
@@ -133,6 +142,7 @@ module XP5K
         role.servers = available_nodes[0..(role.size - 1)]
         available_nodes -= role.servers
         role.jobid = job['uid']
+        next if not self.roles.select { |x| x.name == role.name }.empty?
         self.roles << role
       end
     end
@@ -146,26 +156,24 @@ module XP5K
     end
 
     def get_deployed_nodes(job_or_role_name)
-      if deployed_nodes["jobs"].has_key?(job_or_role_name)
-        deployed_nodes["jobs"][job_or_role_name]
+      if self.deployed_nodes["jobs"].has_key?(job_or_role_name)
+        return self.deployed_nodes["jobs"][job_or_role_name]
       end
-      if deployed_nodes["roles"].has_key?(job_or_role_name)
-        deployed_nodes["roles"][job_or_role_name]
+      if self.deployed_nodes["roles"].has_key?(job_or_role_name)
+        return self.deployed_nodes["roles"][job_or_role_name]
       end
     end
 
     def status
       self.jobs.each do |job|
-        logger.info "Job #{job["name"]} ##{job["uid"]} status : #{job["state"]}"
+        logger.info "Job \"#{job["name"]}\" ##{job["uid"]} status : #{job["state"]}"
       end
     end
 
     def clean
       self.jobs.each do |job|
-        if job.reload["state"] == "running"
-          job.delete
-          logger.info "Job ##{job["uid"]} deleted !"
-        end
+        job.delete if (job['state'] =~ /running|waiting/)
+        logger.info "Job ##{job["uid"]} deleted !"
       end
       FileUtils.rm(".xp_cache")
     end
@@ -175,48 +183,56 @@ module XP5K
     def logger
       @logger
     end
-    
+
     def update_links_deployments (duid, todeploy)
       unless todeploy[:jobs].nil?
         todeploy[:jobs].each do |job|
-          @links_deployments["jobs"][job] = duid  
+          @links_deployments["jobs"][job] ||= []
+          @links_deployments["jobs"][job] << duid
         end
       end
 
       unless todeploy[:roles].nil?
         todeploy[:roles].each do |role|
-          @links_deployments["roles"][role] = duid  
+          @links_deployments["roles"][role] ||= []
+          @links_deployments["roles"][role] << duid
         end
       end
     end
 
     def update_deployed_nodes
-      self.deployments.each do |deployment|
-        duid = deployment["uid"]
-        self.links_deployments["jobs"].select{|k,v| v == duid }.keys.each do |job_name|
-          job = job_with_name(job_name)
-          deployed_nodes["jobs"][job["name"]] = intersect_nodes_job(job, deployment)
-        end
-        self.links_deployments["roles"].select{|k,v| v == duid }.keys.each do |role_name|
-          role = role_with_name(role_name)
-          deployed_nodes["roles"][role.name] = intersect_nodes_role(role, deployment)
+      self.links_deployments["jobs"].each do |jobname,v|
+        job = job_with_name(jobname)
+        deployed_nodes["jobs"][jobname]=[]
+        v.each do |duid|
+          deployment = self.deployments.select{ |d| d["uid"] == duid}.first
+          deployed_nodes["jobs"][jobname] += intersect_nodes_job(job, deployment)
         end
+      end
 
+      self.links_deployments["roles"].each do |rolename,v|
+        role = role_with_name(rolename)
+        deployed_nodes["roles"][rolename]=[]
+        v.each do |duid|
+          deployment = self.deployments.select{ |d| d["uid"] == duid}.first
+          deployed_nodes["roles"][rolename] += intersect_nodes_role(role, deployment)
+        end
       end
+
     end
-    
+
     def intersect_nodes_job (job, deployment)
       nodes_deployed = deployment["result"].select{ |k,v| v["state"]=='OK'}.keys
-      job["assigned_nodes"] & nodes_deployed
+      return job["assigned_nodes"] & nodes_deployed
     end
 
     def intersect_nodes_role (role, deployment)
       nodes_deployed = deployment["result"].select{ |k,v| v["state"]=='OK'}.keys
-      role.servers & nodes_deployed
+      return role.servers & nodes_deployed
     end
-    
+
     def update_cache
-      cache = { 
+      cache = {
         :jobs               => self.jobs.collect { |x| x.properties },
         :roles              => self.roles.map{ |x| { :name => x.name, :size => x.size, :servers => x.servers }},
         :deployed_nodes     => self.deployed_nodes,
@@ -227,5 +243,116 @@ module XP5K
       end
     end
 
+    def internal_deploy(n)
+
+      if (n<=0)
+        return
+      end
+
+      # Fill with nodes to deployed
+      self.todeploy.each do |x|
+        x[:nodes] = x[:assigned_nodes]
+        x[:jobs].each do |jobname|
+          x[:nodes] = x[:nodes] - self.deployed_nodes["jobs"][jobname]
+        end
+        x[:roles].each do |rolename|
+          x[:nodes] = x[:nodes] - self.deployed_nodes["roles"][rolename]
+        end
+      end
+
+      # Clean todeploy
+      self.todeploy.delete_if{ |x|
+        x[:nodes].empty? ||
+        (x[:goal] >= 0) && ( x[:nodes].length < ((1-x[:goal])*(x[:assigned_nodes].length ))) ||
+        x[:retries] <= 0
+      }
+
+      if self.todeploy.empty?
+        return
+      end
+
+      if (n<@retries)
+        logger.info "Redeployment of undeployed nodes"
+      end
+
+      # Launch deployments
+      self.todeploy.each do |y|
+        x = y.clone
+        site = x[:site]
+        x.delete(:site)
+        x.delete(:roles)
+        x.delete(:jobs)
+        x.delete(:assigned_nodes)
+        x.delete(:goal)
+        x.delete(:retry)
+        x.delete(:retries)
+
+        deployment = @connection.root.sites[site.to_sym].deployments.submit(x)
+        self.deployments << deployment
+        # Update links_deployments
+        update_links_deployments(deployment["uid"], y)
+        y[:retries] = y[:retries] - 1
+      end
+
+      logger.info "Waiting for all the deployments to be terminated..."
+      finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
+      while (!finished)
+        sleep 10
+        print "."
+        self.deployments.each do |deployment|
+          deployment.reload
+        end
+        finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
+      end
+      print(" [#{green("OK")}]\n")
+
+      # Update deployed nodes
+      update_deployed_nodes()
+      update_cache()
+
+      internal_deploy(n - 1)
+
+    end
+
+    def set_goal(goal, total)
+      if goal.nil?
+        return -1.0
+      end
+
+      if goal.to_s.include? "%"
+          return goal.to_f/100
+      elsif goal.to_f < 1
+        return goal.to_f
+      elsif goal.to_f == 1.0
+        return goal.to_f/total
+      else
+        return goal.to_f/total
+      end
+    end
+
+    def print_deploy_summary
+      puts "Summary of the deployment"
+      puts "-" * 60
+      printf "%+20s", "Name"
+      printf "%+20s", "Deployed"
+      printf "%+20s", "Undeployed"
+      puts "\n"
+      puts "-" * 60
+
+      self.deployed_nodes["jobs"].each do |jobname, deployed_nodes|
+        puts "\n"
+        printf "%+20s",jobname
+        printf "%20d", deployed_nodes.length
+        printf "%20d", job_with_name(jobname)["assigned_nodes"].length - deployed_nodes.length
+        puts "\n"
+      end
+
+      self.deployed_nodes["roles"].each do |rolename, deployed_nodes|
+        printf "%+20s",rolename
+        printf "%20d", deployed_nodes.length
+        printf "%20d", role_with_name(rolename).servers.length - deployed_nodes.length
+        puts "\n"
+      end
+    end
   end
 end

metadata

@@ -1,78 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: xp5k
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease: 
+  version: 0.0.6
 platform: ruby
 authors:
 - Pascal Morillon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-13 00:00:00.000000000 Z
+date: 2014-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: restfully
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>'
+    - - '>'
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>'
+    - - '>'
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.7
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
 description: A small Grid'5000 helper to submit jobs and deploy environments via REST
@@ -98,28 +89,26 @@ files:
 homepage: https://github.com/pmorillon/xp5k
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.14
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A small Grid'5000 helper to submit jobs and deploy environments via REST
   API
 test_files: []
-has_rdoc: