vagrant-g5k 0.9.4 → 0.9.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,43 @@
1
+ # -*- mode: ruby -*-
2
+ # vi: set ft=ruby :
3
+
4
+ Vagrant.configure(2) do |config|
5
+ config.vm.provider "g5k" do |g5k, override|
6
+ override.nfs.functional = false
7
+ g5k.project_id = "vagrant-g5k-example-gitlab"
8
+ g5k.site = "rennes"
9
+ g5k.username = "msimonin"
10
+ g5k.gateway = "access.grid5000.fr"
11
+ g5k.walltime = "02:00:00"
12
+ g5k.image = {
13
+ :path => "/home/msimonin/public/debian-8.7-amd64-bento.qcow2",
14
+ :backing => "snapshot"
15
+ }
16
+ g5k.net = {
17
+ :type => "bridge",
18
+ }
19
+ g5k.oar = "virtual != 'None'"
20
+ end #g5k
21
+ config.ssh.insert_key = false
22
+
23
+ # server machine
24
+ config.vm.define "server" do |server|
25
+ server.vm.box = "debian/contrib-jessie64"
26
+ server.vm.hostname = "server"
27
+ server.vm.network "private_network", ip: "192.168.42.3"
28
+ server.vm.synced_folder ".", "/vagrant", type: "rsync", disabled: true
29
+
30
+ server.vm.provider "virtualbox" do |vb|
31
+ # Customize the amount of memory on the VM:
32
+ vb.memory = "1024"
33
+ vb.cpus = 1
34
+ end
35
+
36
+ server.vm.provision :ansible do |ansible|
37
+ # Disable default limit to connect to all the machines
38
+ ansible.playbook = "main.yml"
39
+ ansible.raw_arguments = ["-vvv"]
40
+ end
41
+
42
+ end
43
+ end
@@ -0,0 +1,59 @@
1
+ - name: Install gitlab
2
+ hosts: all
3
+ vars:
4
+ ansible_become: yes
5
+ tasks:
6
+ - name: Installing dependencies
7
+ apt: name={{ item }} state=present update_cache=yes
8
+ with_items:
9
+ - apt-transport-https
10
+ - python-setuptools
11
+
12
+ - name: Install pip
13
+ easy_install:
14
+ name: pip
15
+ state: latest
16
+
17
+ # NOTE(msimonin):
18
+ # # freezing the version is due to
19
+ # # see https://github.com/ansible/ansible/issues/17495
20
+ - name: Install docker-py
21
+ pip:
22
+ name: docker-py
23
+ version: 1.7.0
24
+
25
+ - name: Install gitlab module
26
+ pip:
27
+ name: pyapi-gitlab
28
+
29
+ - name: Adding Docker apt key
30
+ apt_key: keyserver=hkp://p80.pool.sks-keyservers.net:80 id=58118E89F3A912897C070ADBF76221572C52609D
31
+
32
+ - name: Adding Docker apt repository
33
+ apt_repository: repo='deb https://apt.dockerproject.org/repo debian-jessie main' state=present
34
+
35
+ - name: Installing dependencies
36
+ apt: name={{ item }} state=present update_cache=yes
37
+ with_items:
38
+ - docker-engine
39
+
40
+
41
+ - name: Install gitlab docker container
42
+ docker_container:
43
+ name: gitlab
44
+ ports:
45
+ - "443:443"
46
+ - "80:80"
47
+ - "2121:22"
48
+ restart_policy: always
49
+ volumes:
50
+ - "/srv/gitlab/config:/etc/gitlab"
51
+ - "/srv/gitlab/logs:/var/log/gitlab"
52
+ - "/srv/gitlab/data:/var/opt/gitlab"
53
+ image: gitlab/gitlab-ce:latest
54
+
55
+ - name: Waiting for gitlab to be ready
56
+ wait_for:
57
+ host: localhost
58
+ port: 80
59
+
@@ -0,0 +1,62 @@
1
+ POC : Emulation of latencies and bandwidth constraints on G5K
2
+
3
+ ```
4
+ vagrant up --provider=g5k
5
+ # generate the inventory, and constraint matrix
6
+ ./setup.rb
7
+ # enforce the constraints
8
+ ansible-playbook -i inventory.generated setup.yml -e @constraints.yml
9
+ # validate the constraints
10
+ ansible-playbook -i inventory.generated test.yml
11
+ ```
12
+
13
+ `delays` subdirectory contains the validation results.
14
+
15
+ ```
16
+ ─$ ls -l delays
17
+ total 88
18
+ -rw-r--r-- 1 msimonin staff 791 7 déc 23:49 netem0
19
+ -rw-r--r-- 1 msimonin staff 829 7 déc 23:49 netem1
20
+ -rw-r--r-- 1 msimonin staff 829 7 déc 23:49 netem2
21
+ -rw-r--r-- 1 msimonin staff 824 7 déc 23:49 netem3
22
+ -rw-r--r-- 1 msimonin staff 814 7 déc 23:49 netem4
23
+ -rw-r--r-- 1 msimonin staff 808 7 déc 23:49 netem5
24
+ -rw-r--r-- 1 msimonin staff 818 7 déc 23:49 netem6
25
+ -rw-r--r-- 1 msimonin staff 818 7 déc 23:49 netem7
26
+ -rw-r--r-- 1 msimonin staff 817 7 déc 23:49 netem8
27
+ -rw-r--r-- 1 msimonin staff 817 7 déc 23:49 netem9
28
+ -rw-r--r-- 1 msimonin staff 15 7 déc 23:41 readme
29
+ ```
30
+
31
+ ```
32
+ ─$ cat delays/netem5
33
+ 10.158.4.213 : 60.35 60.23 60.21 60.22 60.21
34
+ 10.158.4.211 : 160.29 160.22 160.20 160.21 160.19
35
+ 10.158.4.208 : 260.27 260.21 260.20 260.23 260.20
36
+ 10.158.4.204 : 360.29 360.25 360.25 360.21 360.24
37
+ 10.158.4.206 : 460.25 460.21 460.20 460.23 460.17
38
+ 10.158.4.207 : 0.02 0.02 0.02 0.02 0.00
39
+ 10.158.4.210 : 660.21 660.21 660.21 660.22 -
40
+ 10.158.4.212 : 760.24 760.20 760.18 760.19 -
41
+ 10.158.4.205 : 860.26 860.24 860.24 860.24 -
42
+ 10.158.4.209 : 960.19 960.18 960.18 960.18 -
43
+
44
+ 10 targets
45
+ 10 alive
46
+ 0 unreachable
47
+ 0 unknown addresses
48
+
49
+ 4 timeouts (waiting for response)
50
+ 50 ICMP Echos sent
51
+ 46 ICMP Echo Replies received
52
+ 0 other ICMP received
53
+
54
+ 160 ms (min round trip time)
55
+ 423 ms (avg round trip time)
56
+ 960 ms (max round trip time)
57
+ 4.751 sec (elapsed real time)
58
+ ```
59
+
60
+ Example of latencies between the nodes.
61
+
62
+ ![latencies](example_latencies.png)
@@ -0,0 +1,40 @@
1
+ # -*- mode: ruby -*-
2
+ # vi: set ft=ruby :
3
+
4
+
5
+ NB=3
6
+ Vagrant.configure(2) do |config|
7
+ (0..NB-1).each do |i|
8
+ config.vm.define "netem#{i}" do |my|
9
+ my.vm.box = "dummy"
10
+ my.vm.synced_folder ".", "/vagrant", type: "rsync", disabled: true
11
+ my.ssh.insert_key = false
12
+
13
+ my.vm.provider "g5k" do |g5k, override|
14
+ override.nfs.functional = false
15
+ g5k.project_id = "poc-netem"
16
+ g5k.site = "rennes"
17
+ g5k.username = ENV['USER']
18
+ g5k.gateway = "access.grid5000.fr"
19
+ g5k.walltime = "02:00:00"
20
+
21
+ g5k.image = {
22
+ :path => "public/ubuntu1404-9p.qcow2",
23
+ :backing => "snapshot"
24
+ }
25
+
26
+ g5k.net = {
27
+ :type => "bridge"
28
+ }
29
+
30
+ g5k.oar = "virtual != 'None'"
31
+ g5k.resources = {
32
+ :cpu => 2,
33
+ :mem => 2048
34
+ }
35
+ end #g5k
36
+
37
+ end
38
+
39
+ end
40
+ end
@@ -0,0 +1,2 @@
1
+ [defaults]
2
+ host_key_checking = False
@@ -0,0 +1,57 @@
1
+ import matplotlib
2
+ matplotlib.use('tkagg')
3
+
4
+ """
5
+ Demonstrates similarities between pcolor, pcolormesh, imshow and pcolorfast
6
+ for drawing quadrilateral grids.
7
+
8
+ """
9
+ import matplotlib.pyplot as plt
10
+ import numpy as np
11
+
12
+ # make these smaller to increase the resolution
13
+ dx, dy = 0.15, 0.05
14
+
15
+ # generate 2 2d grids for the x & y bounds
16
+ y, x = np.mgrid[slice(-3, 3 + dy, dy),
17
+ slice(-3, 3 + dx, dx)]
18
+ z = (1 - x / 2. + x ** 5 + y ** 3) * np.exp(-x ** 2 - y ** 2)
19
+
20
+ # x and y are bounds, so z should be the value *inside* those bounds.
21
+ # Therefore, remove the last value from the z array.
22
+ print(z)
23
+ z = z[:-1, :-1]
24
+ z_min, z_max = -np.abs(z).max(), np.abs(z).max()
25
+
26
+
27
+ plt.subplot(2, 2, 1)
28
+ plt.pcolor(x, y, z, cmap='RdBu', vmin=z_min, vmax=z_max)
29
+ plt.title('pcolor')
30
+ # set the limits of the plot to the limits of the data
31
+ plt.axis([x.min(), x.max(), y.min(), y.max()])
32
+ plt.colorbar()
33
+
34
+
35
+ plt.subplot(2, 2, 2)
36
+ plt.pcolormesh(x, y, z, cmap='RdBu', vmin=z_min, vmax=z_max)
37
+ plt.title('pcolormesh')
38
+ # set the limits of the plot to the limits of the data
39
+ plt.axis([x.min(), x.max(), y.min(), y.max()])
40
+ plt.colorbar()
41
+
42
+
43
+ plt.subplot(2, 2, 3)
44
+ plt.imshow(z, cmap='RdBu', vmin=z_min, vmax=z_max,
45
+ extent=[x.min(), x.max(), y.min(), y.max()],
46
+ interpolation='nearest', origin='lower')
47
+ plt.title('image (interp. nearest)')
48
+ plt.colorbar()
49
+
50
+
51
+ ax = plt.subplot(2, 2, 4)
52
+ ax.pcolorfast(x, y, z, cmap='RdBu', vmin=z_min, vmax=z_max)
53
+ plt.title('pcolorfast')
54
+ plt.colorbar()
55
+
56
+
57
+ plt.show()
@@ -0,0 +1,3 @@
1
+ {% for host in groups['all'] %}
2
+ {{ hostvars[host]['ansible_eth0']['ipv4']['address'] }}
3
+ {% endfor %}
@@ -0,0 +1,54 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'json'
4
+
5
+ ENABLE=true
6
+ # available modes
7
+ # 'bl' bandwidth + latency constraints
8
+ # 'l'
9
+ # 'b'
10
+ MODE='bl'
11
+
12
+
13
+ # getting all ips
14
+ ips=%x[vagrant ssh-config | grep HostName | awk '{print $2}']
15
+ ips = ips.split("\n")
16
+
17
+ constraints = []
18
+ ips.each_with_index do |ip1, index1|
19
+ puts ip1
20
+ ips.each_with_index do |ip2, index2|
21
+ if ip1 == ip2
22
+ next
23
+ end
24
+ constraints << {
25
+ :source => ip1,
26
+ :target => ip2,
27
+ :rate =>"1000Mbit",
28
+ # :delay => "#{index1 * index2 * 10 + 10}ms"
29
+ :delay => "10ms"
30
+ }
31
+ end
32
+ end
33
+ constraints = {
34
+ :tc => {
35
+ :constraints => constraints,
36
+ :enable => ENABLE,
37
+ :mode => MODE
38
+ }
39
+ }
40
+
41
+ File.open('constraints.yml', 'w') {|f| f.write JSON.pretty_generate(constraints) }
42
+
43
+ # generate the inventory
44
+ # we by pass vagrant support because it's too slow (sequential)
45
+ inventory = ["[test]"]
46
+ ips.each_with_index do |ip, index|
47
+ inventory << "netem#{index} ansible_host=#{ip} ansible_user=vagrant"
48
+ end
49
+ inventory << "[test:vars]"
50
+ inventory << "ansible_ssh_common_args='-o IdentityFile=\"/Users/msimonin/.vagrant.d/insecure_private_key\" -o ProxyCommand=\"ssh -W %h:%p msimonin@access.grid5000.fr\"'"
51
+
52
+ File.open('inventory.generated', 'w') {|f| f.write inventory.join("\n")}
53
+
54
+
@@ -0,0 +1,50 @@
1
+ ---
2
+ - hosts: test
3
+ become: true
4
+ tasks:
5
+ - name: Removing root qdisc
6
+ shell: "tc qdisc del dev eth0 root || true"
7
+ # we always restart
8
+
9
+ - name: Preparing htb
10
+ shell: "tc qdisc add dev eth0 root handle 1: htb"
11
+ when:
12
+ - "{{ tc.enable }}"
13
+ - "'b' in tc.mode"
14
+
15
+ - name: Add rate limit
16
+ command: "tc class add dev eth0 parent 1: classid 1:{{ (item.0 + 1) * 10 }} htb rate {{ item.1.rate }}"
17
+ with_indexed_items: "{{ tc.constraints }}"
18
+ when:
19
+ - "{{ tc.enable }}"
20
+ - item[1].source == ansible_eth0["ipv4"]["address"]
21
+ - "'b' in tc.mode"
22
+
23
+
24
+ - name: Adding delay
25
+ command: "tc qdisc add dev eth0 parent 1:{{ (item.0 + 1) * 10 }} handle {{ (item.0 + 1)*10 }}: netem delay {{ item.1.delay }}"
26
+ with_indexed_items: "{{ tc.constraints }}"
27
+ when:
28
+ - "{{ tc.enable }}"
29
+ - item[1].source == ansible_eth0["ipv4"]["address"]
30
+ - "'b' in tc.mode"
31
+ - "'l' in tc.mode"
32
+
33
+ - name: Adding delay
34
+ command: "tc qdisc add dev eth0 parent 1: handle {{ (item.0 + 1)*10 }}: netem delay {{ item.1.delay }}"
35
+ with_indexed_items: "{{ tc.constraints }}"
36
+ when:
37
+ - "{{ tc.enable }}"
38
+ - item[1].source == ansible_eth0["ipv4"]["address"]
39
+ - "'b' not in tc.mode"
40
+ - "'l' in tc.mode"
41
+
42
+
43
+ - name: Adding filter
44
+ command: "tc filter add dev eth0 parent 1: protocol ip u32 match ip dst {{ item.1.target }} flowid 1:{{ (item.0 + 1) * 10 }}"
45
+ with_indexed_items: "{{ tc.constraints }}"
46
+ when:
47
+ - "{{ tc.enable }}"
48
+ - item[1].source == ansible_eth0["ipv4"]["address"]
49
+ - "'bl' in tc.mode"
50
+
@@ -0,0 +1,53 @@
1
+ ---
2
+ - hosts: test
3
+ become: true
4
+ vars:
5
+ output_dir: result
6
+ tasks:
7
+
8
+ - name: Adding flent repository
9
+ apt_repository:
10
+ repo: deb http://download.opensuse.org/repositories/home:/tohojo:/flent/xUbuntu_14.04/ /
11
+ state: present
12
+
13
+ - name: Installing flent
14
+ apt:
15
+ name: flent
16
+ allow_unauthenticated: yes
17
+ update_cache: true
18
+
19
+ - name: Installing fping
20
+ apt:
21
+ name: fping
22
+
23
+ - name: Uploading the host list
24
+ template:
25
+ src: hosts.j2
26
+ dest: /tmp/hosts
27
+
28
+ - name: Get the latencies between all the nodes
29
+ shell: "fping -C 5 -q -s -e -f /tmp/hosts 2>/tmp/result"
30
+
31
+ - name: Fetching the results
32
+ fetch:
33
+ src: /tmp/result
34
+ dest: "{{output_dir}}/{{ ansible_eth0['ipv4']['address'] }}.out"
35
+ flat: yes
36
+
37
+ - name: Running flent (tcp_upload)
38
+ shell: "flent tcp_upload -l 5 -H {{ hostvars[item]['ansible_eth0']['ipv4']['address'] }} -f stats -o /tmp/tcp_upload_{{ inventory_hostname }}_{{ item }}.stats || true 2>/dev/null"
39
+ when: inventory_hostname == groups.test[0]
40
+ with_items: "{{ groups.test }}"
41
+
42
+ - name: debug
43
+ debug: var=groups.test
44
+
45
+ - name: Fetching the results
46
+ fetch:
47
+ src: "/tmp/tcp_upload_{{ inventory_hostname }}_{{ item }}.stats"
48
+ dest: "{{output_dir}}/tcp_upload_{{ inventory_hostname }}_{{ item }}.stats"
49
+ flat: yes
50
+ when: inventory_hostname == groups.test[0]
51
+ with_items: "{{ groups.test }}"
52
+
53
+
@@ -1,5 +1,5 @@
1
1
  require 'net/ssh/multi'
2
- require 'vagrant-g5k/util/g5k_connection'
2
+ require 'vagrant-g5k/g5k_connection'
3
3
  include Process
4
4
 
5
5
  module VagrantPlugins
@@ -12,21 +12,9 @@ module VagrantPlugins
12
12
  end
13
13
 
14
14
  def execute
15
- # TODO
16
- options = {}
17
- opts = OptionParser.new do |o|
18
- o.banner = 'Usage: vagrant g5k [vm-name]'
19
- o.separator ''
20
- o.version = VagrantPlugins::G5K::VERSION
21
- o.program_name = 'vagrant g5k'
22
- end
23
- argv = parse_options(opts)
24
- with_target_vms(argv, options) do |machine|
25
- puts machine.config.vm.networks
26
- end
27
- puts "sleeping"
28
- wait
15
+ puts 'Nothing implemented yet ;)'
29
16
  end
17
+
30
18
  end
31
19
  end
32
20
  end