vagrant-g5k 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4bb98de93c858a960bbdbc3918b688c7c053b642
-  data.tar.gz: 4f05183a56b167b8bb535e91646e386c4fd58463
+  metadata.gz: d1df4a32a68cbad5242e82101c86660aec6edf0b
+  data.tar.gz: 2978fdaf2984face9272df254040893082497a7d
 SHA512:
-  metadata.gz: 4d1dd22b192c8eff2c6a77991ce555bf877b1c6970bd9f43721e960dc473a0f68459a5f699f2456285edf50d3b9a9ca8b5d654c38de130bacf9f11283bfda8dd
-  data.tar.gz: f326dc3f70a5b63ced19f3d8562ae8e8b8c72e786e2b498d1e9f6ca8f5e75bb78b7572c11af0496c70976248c37fafc83be867b977d62cba3777738fc73372f7
+  metadata.gz: 24b08525b133f8f58a185315f8553322b57c1cf16b544634dd3df56695d496a632e073fb5c790cd4a1c0a11a724d5ff9371e40bcb659d0dd3294f35fb47b795a
+  data.tar.gz: 1229bd98b6478fb89a7d68cb486d6c089b88682a2493558d87d9519bd47a3cdb7e955402ac9d345634d97d53c7b1196b4af95fa641ad471216b738b8db65ae50

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.0.16
+
+  * Improve stdout (add information on physical node)
+  * Stop the polling if the job is terminated
+
 # 0.0.15
 
   * Allow parallel boot

data/README.md

@@ -45,5 +45,24 @@ to back the disk image of the virtual machines :
 * `vagrant halt`
 * `vagrant provision`
 * `vagrant ssh`
+* `vagrant ssh-config`
 * `vagrant status`
 * `vagrant up`
+
+## Use ceph as backing strategy
+
+Vagrant-g5k will look into `~/.ceph/config` on each frontend where VMs are started.
+You can read[1] for further information on how to configure ceph on grid'5000.
+
+[1] : https://www.grid5000.fr/mediawiki/index.php/Ceph
+
+## Developping
+
+* clone the repository
+* use `$ bundle` to install all the dependencies (this may take some time)
+* then test your code against the provided (or modified) Vagrantfile using :
+```
+VAGRANT_LOG=debug VAGRANT_DEFAULT_PROVIDER=g5k bundle exec vagrant up
+```
+
+

data/Vagrantfile

@@ -3,12 +3,12 @@
 #
 # Testing purpose only
 #Vagrant.require_plugin "vagrant-g5k"
-NB_G5K=1
-NB_IGRIDA=1
+
+SITES=['rennes']
 
 Vagrant.configure(2) do |config|
-    (0..NB_G5K-1).each do |i|
-      config.vm.define "vm-g5k-#{i}" do |my|
+    SITES.each do |site|
+      config.vm.define "vm-#{site}" do |my|
         my.vm.box = "dummy"
 
         my.ssh.username = "root"
@@ -16,37 +16,24 @@ Vagrant.configure(2) do |config|
 
         my.vm.provider "g5k" do |g5k|
           g5k.project_id = "vagrant-g5k"
-          g5k.site = "nancy"
+          g5k.site = "#{site}"
           g5k.gateway = "access.grid5000.fr" 
-          g5k.image = { 
-             "pool"     => "msimonin_rbds",
-             "rbd"      => "bases/alpine_docker",
-             "snapshot" => "parent",
-             "id"       => "$USER",
-             "conf"     => "$HOME/.ceph/config",
-             "backing"  => "snapshot"
-           }
-          g5k.ports = ['2222-:22']
-        end #g5k
-      end #vm
-    end # each
-
-
-    (0..NB_IGRIDA-1).each do |i|
-      config.vm.define "vm-igrida-#{i}" do |my|
-        my.vm.box = "dummy"
-
-        my.vm.provider "g5k" do |g5k|
-          g5k.project_id = "vagrant-g5k"
-          g5k.site = "igrida-oar-frontend"
-          g5k.gateway = "transit.irisa.fr" 
-          g5k.image = { 
-            "path" => "/udd/msimonin/precise.qcow2",
+#          g5k.image = {
+#             "pool"     => "msimonin_rbds",
+#             "rbd"      => "bases/alpine_docker",
+#             "snapshot" => "parent",
+#             "id"       => "$USER",
+#            "conf"      => "$HOME/.ceph/config",
+#            "backing"   => "snapshot"
+#           }
+          g5k.image = {
+            "path"    => "/grid5000/virt-images/alpine_docker.qcow2",
             "backing" => "snapshot"
-           }
+          }
           g5k.ports = ['2222-:22']
+          g5k.oar = "virtual != 'none'"
         end #g5k
-      end # vm
+      end #vm
     end # each
 
 end

data/Vagrantfile.multisite

@@ -0,0 +1,37 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+#
+# Testing purpose only
+#Vagrant.require_plugin "vagrant-g5k"
+
+SITES=['rennes', 'nancy', 'nantes', 'lille', 'luxembourg', 'grenoble', 'lyon']
+
+Vagrant.configure(2) do |config|
+    SITES.each do |site|
+      config.vm.define "vm-#{site}" do |my|
+        my.vm.box = "dummy"
+
+        my.ssh.username = "root"
+        my.ssh.password = ""
+
+        my.vm.provider "g5k" do |g5k|
+          g5k.project_id = "vagrant-g5k"
+          g5k.site = "#{site}"
+          g5k.gateway = "access.grid5000.fr" 
+          g5k.image = { 
+             "pool"     => "msimonin_rbds",
+             "rbd"      => "bases/alpine_docker",
+             "snapshot" => "parent",
+             "id"       => "$USER",
+             "conf"     => "$HOME/.ceph/config",
+             "backing"  => "snapshot"
+           }
+          g5k.ports = ['2222-:22']
+          g5k.oar = "virtual != 'none'"
+        end #g5k
+      end #vm
+    end # each
+
+end
+
+

data/lib/vagrant-g5k/errors.rb

@@ -8,15 +8,20 @@ module VagrantPlugins
       end
 
       class TimeoutOnJobSubmissionError < VagrantG5KError
-        error_key("tired of waiting")
+        error_key("tired_of_waiting")
       end
 
       class JobNotRunning < VagrantG5KError
-        error_key("tired of waiting")
+        error_key("tired_of_waiting")
       end
 
+      class JobError < VagrantG5KError
+        error_key("job_error")
+      end
+
+
       class CommandError < VagrantG5KError
-        error_key("remote command error")
+        error_key("remote_command_error")
       end
 
 

data/lib/vagrant-g5k/util/g5k_utils.rb

@@ -18,7 +18,7 @@ module VagrantPlugins
     class Connection
       include Vagrant::Util::Retryable
 
-      attr_accessor :session
+      attr_accessor :driver
 
       attr_accessor :username
 
@@ -74,19 +74,24 @@ module VagrantPlugins
 
 
       def check_job(job_id)
-        oarstat = exec("oarstat --json")
-        oarstat = JSON.load(oarstat)
-        @logger.debug("Looking for the job id #{job_id} and username #{@username}")
-        r = oarstat.select!{ |k,v| k == job_id and v["owner"] == @username }.values.first
-        @logger.debug(r.inspect)
-        # update the assigned hostname
-        # this will be used to reach the vm 
+        oarstat = exec("oarstat -j #{job_id} --json")
+        # json is 
+        # { "job_id" : {description}}
+        r = JSON.load(oarstat)["#{job_id}"]
         if !r.nil?
           @node = r["assigned_network_address"].first
         end
         return r
       end
 
+      def process_errors(job_id)
+        job = check_job(job_id)
+        stderr_file = job["stderr_file"]
+        stderr = exec("cat #{stderr_file}")
+        @ui.error("#{stderr_file}:  #{stderr}")
+        raise VagrantPlugins::G5K::Errors::JobError
+      end
+
       def delete_job(job_id)
         @ui.info("Deleting the associated job")
         exec("oardel -c -s 12 #{job_id}")
@@ -145,10 +150,13 @@ module VagrantPlugins
         job_id = exec("oarsub --json -t allow_classic_ssh -l \"#{@oar}nodes=1,walltime=#{@walltime}\" --name #{env[:machine].name} --checkpoint 60 --signal 12  \"#{launcher_remote_path} #{args}\" | grep \"job_id\"| cut -d':' -f2").gsub(/"/,"").strip
 
         begin
-          retryable(:on => VagrantPlugins::G5K::Errors::JobNotRunning, :tries => 100, :sleep => 2) do
-            @ui.info("Waiting for the job to be running")
+          retryable(:on => VagrantPlugins::G5K::Errors::JobNotRunning, :tries => 100, :sleep => 1) do
             job = check_job(job_id)
-            if job.nil? or job["state"] != "Running"
+            if !job.nil? and ["Error", "Terminated"].include?(job["state"])
+              process_errors(job_id)
+            end
+            if job.nil? or (!job.nil? and job["state"] != "Running")
+              @ui.info("Waiting for the job to be running")
               raise VagrantPlugins::G5K::Errors::JobNotRunning 
             end
             # saving the id
@@ -159,7 +167,7 @@ module VagrantPlugins
           @ui.error("Tired of waiting")
           raise VagrantPlugins::G5K::Errors::JobNotRunning
         end
-        @ui.info("VM booted @#{@site} on #{@node}")
+        @ui.info("booted @#{@site} on #{@node}")
 
       end
 
@@ -229,7 +237,7 @@ module VagrantPlugins
           file = _rbd_clone_or_copy_image(env, clone = false)
         end
         # encapsulate the file to a qemu ready disk description
-        file = "rbd:#{file}:id=#{@image["id"]}:conf=#{@image["conf"]}"
+        file = "rbd:#{file}:id=#{@image["id"]}:conf=#{@image["conf"]}:rbd_cache=true,cache=writeback"
         @logger.debug("Generated drive string : #{file}")
         return file
       end

data/lib/vagrant-g5k/version.rb

@@ -1,5 +1,5 @@
 module VagrantPlugins
   module G5K
-    VERSION = '0.0.15'
+    VERSION = '0.0.16'
   end
 end

data/locales/en.yml

@@ -80,7 +80,11 @@ en:
         One or more of the needed AWS credentials are missing. No environment variables
         are set nor profile '%{profile}' exists at '%{location}'
 
-    errors:
+    errors: 
+      job_error: Job error
+      tired_of_waiting: Tired of waiting
+      remote_command_error: Remote command error
+
       fog_error: |-
         There was an error talking to AWS. The error message is shown
         below:

data/results/.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+*.retry

data/results/README.md

@@ -0,0 +1,43 @@
+# Post-Mortem analysis
+
+## What's inside the VMs ?
+
+Each virtual machine (VM) offer a toolbox to analyse various datas from the experimentation made with kolla-g5k. The datas themselves are also located inside the VM during the creation of the VM.
+
+You'll find :
+
+* Nginx exposed on port 8000. It allows you to browse Rally reports, confs and logs.
+* Grafana exposed on port 3000. It gives access of all the metrics collected during the experimentation
+* Kibana exposed on port 5601. It let you explores the logs.
+
+
+## List of provided experimentation
+
+* idle : *todo wiki url*
+* load-ded : *todo wiki url*
+* load-default : *todo wiki url*
+* [load-clust-rabbit](https://github.com/BeyondTheClouds/kolla-g5k/wiki/load-clust-rabbitmq)
+* concurrency : *todo wiki url*
+
+## Accessing the results
+
+Start a specific virtual machine :
+
+```
+vagrant up <experimentation>
+```
+
+Shutdown the virtual machine
+```
+vagrant halt <experimentation>
+```
+
+## Creating new results
+
+*todo*
+
+## Known limitation
+
+* The current implementation is tight to Grid'5000.
+* When creating a new set of result, indexing all the logs in elasticsearch will take some time.
+* From time to time VMs root filesystem goes read-only. Manually fsck the root partition and reboot may solve the issue `fsck /dev/vg0/lv_root`.

data/results/Vagrantfile

@@ -0,0 +1,126 @@
+# coding: utf-8
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+
+# This Vagrantfile makes use of the plugin vagrant-g5k
+# https://github.com/msimonin/vagrant-g5k
+#
+# version 0.0.13
+
+# The list of experimentation. There will be one VM per
+# experimentation. You can access it thought eg, `vagrant ssh idle`.
+XPS =[
+  {
+    :name  => "idle",
+    :confs => [ "cpt20-nfk05", "cpt20-nfk10", "cpt20-nfk25", "cpt20-nfk50" ]
+  },
+  {
+    # dedicated 1 node  for each mariadb, haproxy, conductor, rabbitmq, memcached
+    # with rally benchmark
+    :name  => "load-ded",
+    :confs => [ "cpt20-nfk05", "cpt20-nfk10", "cpt20-nfk25", "cpt20-nfk50-stopped"]
+  },
+  {
+    # default topology
+    # with rally benchmark
+    :name  => "load-default",
+    :confs => [ "cpt20-nfk05", "cpt20-nfk10", 'cpt20-nfk25']
+  },
+  {
+    :name  => "concurrency",
+    :confs => [ "ccy0001-0015-cpt20-nfk50",
+                "ccy0025-0045-cpt20-nfk50",
+                "ccy0100-1000-cpt20-nfk05" ]
+  },
+  {
+    # load test with a clustered rabbitmq
+    :name  => "load-clust-rabbit",
+    :confs => [ "cpt20-nfk50",
+                "cpt20-nfk50-tuned-report-sync-intervals",
+                "cpt20-nfk50-tuned-report-sync-intervals-handshake-timeout",
+                "cpt20-nfk50-cond10-tuned-handshake-timeout"]
+  }
+ # Add another experimentation
+ # ,{ :name  => "vanilla",
+ #    :confs => [ "cpt20-nfk05", "cpt20-nfk10", "cpt20-nfk25", "cpt20-nfk50" ]}
+]
+
+Vagrant.configure(2) do |config|
+    # user to log with inside the vm
+    # config.ssh.username = "root"
+    # password to use to log inside the vm
+ 
+    config.vm.provider "g5k" do |g5k|
+      # The project id.
+      # It is used to generate uniq remote storage for images
+      # It must be uniq accros all project managed by vagrant.
+      g5k.project_id = "vagrant-g5k"
+
+      # user name used to connect to g5k
+      g5k.username = "msimonin"
+
+      # private key
+      # g5k.private_key = File.join(ENV['HOME'], ".ssh/id_rsa_discovery")
+
+      # site to use
+      g5k.site = "rennes"
+      g5k.gateway = "access.grid5000.fr"
+
+      # walltime to use
+      g5k.walltime = "03:00:00"
+
+      # image location
+      # g5k.image = {
+      #  "path" => "$HOME/public/ubuntu1404.qcow2",
+      #  "backing" => "copy"
+      #}
+
+      # it could be backed by the ceph
+      g5k.image = {
+        "pool"     => "msimonin_rbds",
+        "rbd"      => "bases/bocutter-ubuntu1404_home_ceph",
+        "id"       => "msimonin",
+        "conf"     => "/home/msimonin/.ceph/config",
+        "backing"  => "copy"
+      }
+
+      # ports to expose (at least ssh has to be forwarded)
+      g5k.ports = ['2222-:22','3000-:3000', '8000-:80', '5601-:5601']
+    end
+
+    XPS.each do |xp|
+      config.vm.define xp[:name] do |my|
+        # box isn't used
+        my.vm.box = "dummy"
+
+        # From `boilerplate.yml`: this playbook relies on an `xps` variable.
+        # The `xps` variable is a list that contains the name of all
+        # experimentation. For instance, this list is as following for the
+        # idle experimentation:
+        # - idle-cpt20-nfk05
+        # - idle-cpt20-nfk10
+        # - idle-cpt20-nfk25
+        # - idle-cpt20-nfk50
+        #
+        # This ruby method computes this list and gives it to
+        # `ansible-playbook`. The construction of the list is based on the
+        # name of the experimentation `xp[:name]` and the list of
+        # experimentation done `xp[:confs]`
+        xps = {:xps => xp[:confs].map { |cfg| xp[:name] + "-" + cfg } }
+
+        # For the provision to run a dedicated proxy command seems necessary
+        # in your ssh config (the provisionner seems to not take into account
+        # the ssh-config of vagrant)
+        # Host *.grid5000.fr
+        #   User <login>
+        #   StrictHostKeyChecking no
+        #   ProxyCommand ssh <login>@194.254.60.4 "nc %h %p"
+        my.vm.provision :ansible do |ansible|
+          ansible.playbook = "boilerplate.yml"
+          ansible.extra_vars = xps
+          ansible.verbose = "-vvvv"
+        end
+      end
+    end
+end