vagrant-g5k 0.9.4 → 0.9.5

data/examples/gitlab/Vagrantfile

@@ -0,0 +1,43 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+  config.vm.provider "g5k" do |g5k, override|
+    override.nfs.functional = false
+    g5k.project_id = "vagrant-g5k-example-gitlab"
+    g5k.site = "rennes"
+    g5k.username = "msimonin"
+    g5k.gateway = "access.grid5000.fr"
+    g5k.walltime = "02:00:00"
+    g5k.image = {
+      :path    => "/home/msimonin/public/debian-8.7-amd64-bento.qcow2",
+      :backing => "snapshot"
+    }
+    g5k.net = {
+      :type => "bridge",
+    }
+    g5k.oar = "virtual != 'None'"
+  end #g5k
+  config.ssh.insert_key = false
+
+  # server machine
+  config.vm.define "server" do |server|
+    server.vm.box = "debian/contrib-jessie64"
+    server.vm.hostname = "server"
+    server.vm.network "private_network", ip: "192.168.42.3"
+    server.vm.synced_folder ".", "/vagrant", type: "rsync", disabled: true
+
+    server.vm.provider "virtualbox" do |vb|
+      # Customize the amount of memory on the VM:
+      vb.memory = "1024"
+      vb.cpus = 1
+    end
+
+    server.vm.provision :ansible do |ansible|
+      # Disable default limit to connect to all the machines
+      ansible.playbook = "main.yml"
+      ansible.raw_arguments = ["-vvv"]
+    end
+
+  end
+end

data/examples/gitlab/main.yml

@@ -0,0 +1,59 @@
+- name: Install gitlab
+  hosts: all
+  vars:
+    ansible_become: yes
+  tasks:
+    - name: Installing dependencies
+      apt: name={{ item }} state=present update_cache=yes
+      with_items:
+        - apt-transport-https
+        - python-setuptools
+
+    - name: Install pip
+      easy_install:
+        name: pip
+        state: latest
+
+    # NOTE(msimonin):
+    # # freezing the version is due to
+    # # see https://github.com/ansible/ansible/issues/17495
+    - name: Install docker-py
+      pip:
+        name: docker-py
+        version: 1.7.0
+
+    - name: Install gitlab module
+      pip:
+        name: pyapi-gitlab
+
+    - name: Adding Docker apt key
+      apt_key: keyserver=hkp://p80.pool.sks-keyservers.net:80 id=58118E89F3A912897C070ADBF76221572C52609D
+
+    - name: Adding Docker apt repository
+      apt_repository: repo='deb https://apt.dockerproject.org/repo debian-jessie main' state=present
+
+    - name: Installing dependencies
+      apt: name={{ item }} state=present update_cache=yes
+      with_items:
+        - docker-engine
+
+
+    - name: Install gitlab docker container
+      docker_container:
+        name: gitlab
+        ports:
+          - "443:443"
+          - "80:80"
+          - "2121:22"
+        restart_policy: always
+        volumes:
+          - "/srv/gitlab/config:/etc/gitlab"
+          - "/srv/gitlab/logs:/var/log/gitlab"
+          - "/srv/gitlab/data:/var/opt/gitlab"
+        image: gitlab/gitlab-ce:latest
+
+    - name: Waiting for gitlab to be ready
+      wait_for:
+        host: localhost
+        port: 80
+

data/examples/htb-netem/README.md

@@ -0,0 +1,62 @@
+POC : Emulation of latencies and bandwidth constraints on G5K
+
+```
+vagrant up --provider=g5k
+# generate the inventory, and constraint matrix
+./setup.rb
+# enforce the constraints
+ansible-playbook -i inventory.generated setup.yml -e @constraints.yml
+# validate the constraints
+ansible-playbook -i inventory.generated test.yml
+```
+
+`delays` subdirectory contains the validation results.
+
+```
+─$ ls -l delays
+total 88
+-rw-r--r--  1 msimonin  staff  791  7 déc 23:49 netem0
+-rw-r--r--  1 msimonin  staff  829  7 déc 23:49 netem1
+-rw-r--r--  1 msimonin  staff  829  7 déc 23:49 netem2
+-rw-r--r--  1 msimonin  staff  824  7 déc 23:49 netem3
+-rw-r--r--  1 msimonin  staff  814  7 déc 23:49 netem4
+-rw-r--r--  1 msimonin  staff  808  7 déc 23:49 netem5
+-rw-r--r--  1 msimonin  staff  818  7 déc 23:49 netem6
+-rw-r--r--  1 msimonin  staff  818  7 déc 23:49 netem7
+-rw-r--r--  1 msimonin  staff  817  7 déc 23:49 netem8
+-rw-r--r--  1 msimonin  staff  817  7 déc 23:49 netem9
+-rw-r--r--  1 msimonin  staff   15  7 déc 23:41 readme
+```
+
+```
+─$ cat delays/netem5
+10.158.4.213 : 60.35 60.23 60.21 60.22 60.21
+10.158.4.211 : 160.29 160.22 160.20 160.21 160.19
+10.158.4.208 : 260.27 260.21 260.20 260.23 260.20
+10.158.4.204 : 360.29 360.25 360.25 360.21 360.24
+10.158.4.206 : 460.25 460.21 460.20 460.23 460.17
+10.158.4.207 : 0.02 0.02 0.02 0.02 0.00
+10.158.4.210 : 660.21 660.21 660.21 660.22 -
+10.158.4.212 : 760.24 760.20 760.18 760.19 -
+10.158.4.205 : 860.26 860.24 860.24 860.24 -
+10.158.4.209 : 960.19 960.18 960.18 960.18 -
+
+      10 targets
+      10 alive
+       0 unreachable
+       0 unknown addresses
+
+       4 timeouts (waiting for response)
+      50 ICMP Echos sent
+      46 ICMP Echo Replies received
+       0 other ICMP received
+
+ 160 ms (min round trip time)
+ 423 ms (avg round trip time)
+ 960 ms (max round trip time)
+        4.751 sec (elapsed real time)
+```
+
+Example of latencies between the nodes.
+
+![latencies](example_latencies.png)

data/examples/htb-netem/Vagrantfile

@@ -0,0 +1,40 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+
+NB=3
+Vagrant.configure(2) do |config|
+  (0..NB-1).each do |i|
+    config.vm.define "netem#{i}" do |my|
+      my.vm.box = "dummy"
+      my.vm.synced_folder ".", "/vagrant", type: "rsync", disabled: true
+      my.ssh.insert_key = false
+
+      my.vm.provider "g5k" do |g5k, override|
+        override.nfs.functional = false
+        g5k.project_id = "poc-netem"
+        g5k.site = "rennes"
+        g5k.username = ENV['USER']
+        g5k.gateway = "access.grid5000.fr"
+        g5k.walltime = "02:00:00"
+
+        g5k.image = {
+          :path    => "public/ubuntu1404-9p.qcow2",
+          :backing => "snapshot"
+        }
+
+        g5k.net = {
+          :type => "bridge"
+        }
+
+        g5k.oar = "virtual != 'None'"
+        g5k.resources = {
+          :cpu => 2,
+          :mem => 2048
+        }
+      end #g5k
+
+    end
+
+  end
+end

data/examples/htb-netem/ansible.cfg

@@ -0,0 +1,2 @@
+[defaults]
+host_key_checking = False

data/examples/htb-netem/heat.py

@@ -0,0 +1,57 @@
+import matplotlib
+matplotlib.use('tkagg')
+
+"""
+Demonstrates similarities between pcolor, pcolormesh, imshow and pcolorfast
+for drawing quadrilateral grids.
+
+"""
+import matplotlib.pyplot as plt
+import numpy as np
+
+# make these smaller to increase the resolution
+dx, dy = 0.15, 0.05
+
+# generate 2 2d grids for the x & y bounds
+y, x = np.mgrid[slice(-3, 3 + dy, dy),
+                slice(-3, 3 + dx, dx)]
+z = (1 - x / 2. + x ** 5 + y ** 3) * np.exp(-x ** 2 - y ** 2)
+
+# x and y are bounds, so z should be the value *inside* those bounds.
+# Therefore, remove the last value from the z array.
+print(z)
+z = z[:-1, :-1]
+z_min, z_max = -np.abs(z).max(), np.abs(z).max()
+
+
+plt.subplot(2, 2, 1)
+plt.pcolor(x, y, z, cmap='RdBu', vmin=z_min, vmax=z_max)
+plt.title('pcolor')
+# set the limits of the plot to the limits of the data
+plt.axis([x.min(), x.max(), y.min(), y.max()])
+plt.colorbar()
+
+
+plt.subplot(2, 2, 2)
+plt.pcolormesh(x, y, z, cmap='RdBu', vmin=z_min, vmax=z_max)
+plt.title('pcolormesh')
+# set the limits of the plot to the limits of the data
+plt.axis([x.min(), x.max(), y.min(), y.max()])
+plt.colorbar()
+
+
+plt.subplot(2, 2, 3)
+plt.imshow(z, cmap='RdBu', vmin=z_min, vmax=z_max,
+           extent=[x.min(), x.max(), y.min(), y.max()],
+           interpolation='nearest', origin='lower')
+plt.title('image (interp. nearest)')
+plt.colorbar()
+
+
+ax = plt.subplot(2, 2, 4)
+ax.pcolorfast(x, y, z, cmap='RdBu', vmin=z_min, vmax=z_max)
+plt.title('pcolorfast')
+plt.colorbar()
+
+
+plt.show()

data/examples/htb-netem/hosts.j2

@@ -0,0 +1,3 @@
+{% for host in groups['all'] %}
+ {{ hostvars[host]['ansible_eth0']['ipv4']['address'] }}
+{% endfor %}

data/examples/htb-netem/setup.rb

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'json'
+
+ENABLE=true
+# available modes 
+# 'bl' bandwidth + latency constraints
+# 'l'
+# 'b'
+MODE='bl'
+
+
+# getting all ips
+ips=%x[vagrant ssh-config | grep HostName | awk '{print $2}']
+ips = ips.split("\n")
+
+constraints = []
+ips.each_with_index do |ip1, index1|
+  puts ip1
+  ips.each_with_index do |ip2, index2|
+    if ip1 == ip2
+      next
+    end
+    constraints << {
+      :source => ip1,
+      :target => ip2,
+      :rate =>"1000Mbit",
+#      :delay => "#{index1 * index2 * 10 + 10}ms"
+      :delay => "10ms"
+    }
+  end
+end
+constraints = {
+  :tc => {
+    :constraints => constraints,
+    :enable => ENABLE,
+    :mode => MODE
+  }
+}
+
+File.open('constraints.yml', 'w') {|f| f.write JSON.pretty_generate(constraints) } 
+
+# generate the inventory
+# we by pass vagrant support because it's too slow (sequential)
+inventory = ["[test]"]
+ips.each_with_index do |ip, index|
+  inventory << "netem#{index} ansible_host=#{ip} ansible_user=vagrant"
+end
+inventory << "[test:vars]"
+inventory << "ansible_ssh_common_args='-o IdentityFile=\"/Users/msimonin/.vagrant.d/insecure_private_key\" -o ProxyCommand=\"ssh -W %h:%p msimonin@access.grid5000.fr\"'"
+
+File.open('inventory.generated', 'w') {|f| f.write inventory.join("\n")}
+
+

data/examples/htb-netem/setup.yml

@@ -0,0 +1,50 @@
+---
+- hosts: test
+  become: true
+  tasks:
+  - name: Removing root qdisc
+    shell: "tc qdisc del dev eth0 root || true"
+    # we always restart
+
+  - name: Preparing htb
+    shell: "tc qdisc add dev eth0 root handle 1: htb"
+    when: 
+      - "{{ tc.enable }}"
+      - "'b' in tc.mode"
+
+  - name: Add rate limit
+    command: "tc class add dev eth0 parent 1: classid 1:{{ (item.0 + 1) * 10  }} htb rate {{ item.1.rate }}"
+    with_indexed_items: "{{ tc.constraints }}"
+    when:
+      - "{{ tc.enable }}"
+      - item[1].source == ansible_eth0["ipv4"]["address"]
+      - "'b' in tc.mode"
+
+
+  - name: Adding delay
+    command: "tc qdisc add dev eth0 parent 1:{{ (item.0 + 1) * 10 }} handle {{ (item.0 + 1)*10 }}: netem delay {{ item.1.delay }}"
+    with_indexed_items: "{{ tc.constraints }}"
+    when:
+      - "{{ tc.enable }}"
+      - item[1].source == ansible_eth0["ipv4"]["address"]
+      - "'b' in tc.mode"
+      - "'l' in tc.mode"
+
+  - name: Adding delay
+    command: "tc qdisc add dev eth0 parent 1: handle {{ (item.0 + 1)*10 }}: netem delay {{ item.1.delay }}"
+    with_indexed_items: "{{ tc.constraints }}"
+    when:
+      - "{{ tc.enable }}"
+      - item[1].source == ansible_eth0["ipv4"]["address"]
+      - "'b' not in tc.mode" 
+      - "'l' in tc.mode"
+
+
+  - name: Adding filter
+    command: "tc filter add dev eth0  parent 1: protocol ip  u32 match ip dst {{ item.1.target }} flowid 1:{{ (item.0 + 1) * 10 }}"
+    with_indexed_items: "{{ tc.constraints }}"
+    when:
+      - "{{ tc.enable }}"
+      - item[1].source == ansible_eth0["ipv4"]["address"]
+      - "'bl' in tc.mode"
+

data/examples/htb-netem/test.yml

@@ -0,0 +1,53 @@
+---
+- hosts: test
+  become: true
+  vars:
+    output_dir: result   
+  tasks:
+   
+    - name: Adding flent repository
+      apt_repository:
+        repo: deb http://download.opensuse.org/repositories/home:/tohojo:/flent/xUbuntu_14.04/ /
+        state: present
+
+    - name: Installing flent
+      apt:
+        name: flent
+        allow_unauthenticated: yes
+        update_cache: true
+
+    - name: Installing fping
+      apt: 
+        name: fping
+
+    - name: Uploading the host list
+      template: 
+        src: hosts.j2
+        dest: /tmp/hosts
+    
+    - name: Get the latencies between all the nodes
+      shell: "fping -C 5 -q -s -e -f /tmp/hosts 2>/tmp/result"
+
+    - name: Fetching the results
+      fetch:
+        src: /tmp/result
+        dest: "{{output_dir}}/{{ ansible_eth0['ipv4']['address'] }}.out"
+        flat: yes
+
+    - name: Running flent (tcp_upload)
+      shell: "flent tcp_upload -l 5 -H {{ hostvars[item]['ansible_eth0']['ipv4']['address']   }} -f stats -o /tmp/tcp_upload_{{ inventory_hostname }}_{{ item  }}.stats || true 2>/dev/null"
+      when: inventory_hostname == groups.test[0]
+      with_items: "{{ groups.test }}"
+
+    - name: debug
+      debug: var=groups.test
+
+    - name: Fetching the results
+      fetch:
+        src: "/tmp/tcp_upload_{{ inventory_hostname }}_{{ item  }}.stats"
+        dest: "{{output_dir}}/tcp_upload_{{ inventory_hostname }}_{{ item  }}.stats"
+        flat: yes
+      when: inventory_hostname == groups.test[0]
+      with_items: "{{ groups.test }}"
+
+

data/lib/vagrant-g5k/command.rb

@@ -1,5 +1,5 @@
 require 'net/ssh/multi'
-require 'vagrant-g5k/util/g5k_connection'
+require 'vagrant-g5k/g5k_connection'
 include Process
 
 module VagrantPlugins
@@ -12,21 +12,9 @@ module VagrantPlugins
       end
 
       def execute
-        # TODO
-        options = {}
-        opts = OptionParser.new do |o|
-          o.banner = 'Usage: vagrant g5k [vm-name]'
-          o.separator ''
-          o.version = VagrantPlugins::G5K::VERSION
-          o.program_name = 'vagrant g5k'
-        end
-        argv = parse_options(opts)
-        with_target_vms(argv, options) do |machine|
-          puts machine.config.vm.networks
-        end
-        puts "sleeping"
-        wait
+        puts 'Nothing implemented yet ;)'
       end
+
     end
   end
 end