bfire 0.2.0

data/README.md

@@ -0,0 +1,90 @@
+# bfire
+A powerful DSL to launch experiments on BonFIRE.
+
+What this does for you:
+
+* Nice DSL to declare the resources you want;
+* Groups compute resources into... groups;
+* Supports dependencies between groups, and builds the dependency graph to launch groups in the right order;
+* Provision software on compute resources using [Puppet](http://www.puppetlabs.com/);
+* Provides hooks after each deployment step so that you can launch your own commands;
+* Abstracts SSH connections, including connections going through gateways;
+* Registers metrics into Zabbix;
+* Scale up or scale down groups based on any condition you want, including metric values.
+
+This is very much a work in progress, and a proof of concept. 
+The code is definitely not something you want to look at.
+
+## Usage
+
+    $ bfire my-experiment.rb
+
+Or, if you are developing in the project's directory:
+
+    $ git clone git://github.com/crohr/bfire.git
+    $ cd bfire/
+    $ ruby -I lib/ bin/bfire my-experiment.rb
+
+Content of `my-experiment.rb`:
+
+    set :name, "Simple Experiment using bfire"
+    set :walltime, 3600
+    set :gateway, "ssh.bonfire.grid5000.fr"
+    set :user, ENV['USER']
+    set :logging, INFO
+
+    set :squeeze, "BonFIRE Debian Squeeze 2G v1"
+    set :zabbix, "BonFIRE Zabbix Aggregator v2"
+    set :wan, "BonFIRE WAN"
+
+    group :monitor do
+      at "uk-epcc"
+      instance_type "small"
+      deploy conf[:zabbix]
+      connect_to conf[:wan]
+    end
+
+    group :servers do
+      at "fr-inria"
+      instance_type "small"
+      deploy conf[:squeeze]
+      connect_to conf[:wan]
+
+      # This is not a runtime dependency, it starts right after the resources in
+      # the monitor group have been _created_ (they're not necessarily _running_).
+      depends_on :monitor do |group|
+        {:aggregator_ip => group.take(:first)['nic'][0]['ip']}
+      end
+    end
+
+    group :clients do
+      at "fr-inria"
+      at "de-hlrs"
+      instance_type "small"
+      deploy conf[:squeeze]
+      connect_to conf[:wan]
+
+      depends_on :monitor do |group|
+        {:aggregator_ip => group.take(:first)['nic'][0]['ip']}
+      end
+      depends_on :servers do |group|
+        {:server_ips => group.map{|vm| vm['nic'][0]['ip']}}
+      end
+
+      on :launched do
+        puts "Yeah, our resources have been launched!"
+      end
+
+      # The ready event is generated once the group resources are launched AND
+      # ssh accessible.
+      on :ready do |group|
+        group.each{|vm|
+          puts "#{group.banner}#{vm['name']} - #{vm['nic'][0]['ip']}"
+        }
+      end
+    end
+
+See the `examples` directory for up to date examples.
+
+## Authors
+* Cyril Rohr <cyril.rohr@inria.fr>

data/bin/bfire

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+require 'bfire'
+require 'optparse'
+require 'logger'
+require 'pp'
+
+logger = Logger.new(STDERR)
+logger.level = Logger.const_get(ENV['DEBUG'] || "WARN")
+
+@options = {
+  :logger => logger,
+  :restfully_config => File.expand_path(
+    "~/.restfully/api.bonfire-project.eu"
+  )
+}
+  
+filename = ARGV[0]
+if filename.nil?
+  dir = File.dirname(Dir.pwd)
+  dsl = ""
+  while line = gets do
+    dsl << line
+  end
+else
+  dir = File.dirname(filename)
+  dsl = File.read(filename)
+end
+
+engine = Bfire::Engine.new(:root => dir)
+engine.instance_eval(dsl)
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = <<BANNER
+* Description
+  bfire: Launch experiments on BonFIRE.
+
+* Usage
+  $ bfire engine-file
+
+* Options
+BANNER
+  
+  opts.on("-a=", "--authorized-keys=", "Specify an authorized_keys file to use instead of your public key [default=#{engine.conf[:authorized_keys]}].") do |v|
+    authorized_keys = File.expand_path(v)
+    if File.file?(authorized_keys) && File.readable?(authorized_keys)
+      @options[:authorized_keys] = authorized_keys
+    else
+      fail "Cannot find your authorized_keys file at #{authorized_keys.inspect}"
+    end
+  end
+  opts.on("-c=", "--config=", "Pass the Restfully YAML configuration file. See <https://github.com/crohr/restfully>.") do |v|
+    @options[:restfully_config] = File.expand_path(v)
+  end
+  opts.on("-u=", "--user=", "Specify your BonFIRE username [default=#{engine.conf[:user]}].") do |v|
+    @options[:user] = v
+  end
+  opts.on("-g=", "--gateway=", "Issues SSH commands via the specified gateway [default=#{engine.conf[:gateway]}].") do |v|
+    @options[:gateway] = v
+  end
+  opts.on("-w=", "--walltime=", "Specify the walltime for the experiment (in seconds) [default=#{engine.conf[:walltime]}].") do |v|
+    @options[:walltime] = v.to_i
+  end
+  opts.on("-k=", "--key=", "Specify the private key to use [default=#{engine.conf[:key]}].") do |v|
+    public_key = File.expand_path(v+".pub")
+    if File.file?(public_key) && File.readable?(public_key)
+      @options[:key] = v
+      # public_key can already be set by --authorized-keys-file, 
+      # don't override:
+      @options[:authorized_keys] ||= public_key 
+    else
+      fail "Cannot find the public part of your SSH key at #{public_key.inspect}"
+    end
+  end
+  {
+    :no_cleanup => "Do not delete experiment at the end.",
+    :no_cancel => "Do not delete experiment at the end, even if an error occurs.",
+  }.each do |flag, description|
+    opts.on("--#{flag.to_s.gsub('_','-')}", description) do |v|
+      @options[flag] = true
+    end
+  end
+  opts.on("--dev", "Attempts to reuse an existing experiment with the same name, in the running state.") do |v|
+    @options[:dev] = true
+  end
+  opts.on("--name=", "Name for your experiment [default=#{engine.conf[:name]}].") do |v|
+    @options[:name] = v
+  end
+  # opts.on("--log=", "Outputs log messages to the given file. Defaults to STDERR.") do |v|
+  #   original_logger_level = logger.level
+  #   logger = Logger.new(File.expand_path(v))
+  #   logger.level = original_logger_level
+  #   @options[:logger] = logger
+  # end
+  opts.on("--debug", "Set the logging level (DEBUG,INFO,WARN,ERROR,UNKNOWN) [default=#{engine.conf[:logging]}].") do |v|
+    @options[:logging] = Logger.const_get(v.to_s.upcase)
+  end
+  opts.on("--version", "Display the version.") do |v|
+    puts Bfire::VERSION
+    exit(0)
+  end
+  opts.on_tail("-h", "--help", "Show this message.") do
+    puts opts
+    exit(0)
+  end
+
+end
+
+
+begin
+  option_parser.parse!
+  @options.each do |k,v|
+    engine.set(k, v)
+  end
+
+  engine.run!
+rescue OptionParser::ParseError => e
+  STDERR.puts "Error when parsing the options: #{e.message} (#{e.class.name})"
+  exit(1)
+end
+

data/examples/benchmark.rb

@@ -0,0 +1,18 @@
+# $ bfire simple.rb
+
+set :name, "Benchmark using bfire"
+set :walltime, 3600
+set :gateway, "ssh.bonfire.grid5000.fr"
+set :user, ENV['USER']
+set :logging, INFO
+
+set :squeeze, "BonFIRE Debian Squeeze 2G v2"
+set :wan, "BonFIRE WAN"
+
+group :servers do
+  at "fr-inria"
+  instance_type "lite"
+  deploy conf[:squeeze]
+  connect_to conf[:wan]
+  scale 1..100, :initial => 2
+end

data/examples/dag.rb

@@ -0,0 +1,26 @@
+
+group :c do
+  depends_on :a
+  depends_on :b
+end
+
+group :f do
+  depends_on :c
+end
+
+group :a do
+  
+end
+
+group :b do
+  
+end
+
+
+group :d do
+  depends_on :c
+end
+
+group :e do
+  depends_on :d
+end

data/examples/elasticity.rb

@@ -0,0 +1,105 @@
+# $ bfire elasticity.rb
+
+# Define global properties
+set :name, "BonFIRE elasticity experiment"
+set :key, "~/.ssh/id_rsa"
+set :authorized_keys, "~/.ssh/authorized_keys"
+set :walltime, 3600
+set :gateway, "ssh.bonfire.grid5000.fr"
+set :user, ENV['USER']
+set :logging, INFO
+
+set :squeeze, "BonFIRE Debian Squeeze 2G v2"
+set :zabbix, "BonFIRE Zabbix Aggregator v4"
+set :wan, "BonFIRE WAN"
+
+
+# Monitoring
+group :eye, :tag => "BonFIRE-monitor" do
+  at "fr-inria"
+  instance_type "small"
+  deploy conf[:zabbix]
+  connect_to conf[:wan]
+end
+
+# HTTP Routing
+group :web do
+  at "fr-inria"
+  instance_type 'small'
+  deploy conf[:squeeze]
+  # Two interfaces for the publicly facing server
+  connect_to conf[:wan]
+  connect_to :public
+
+  provider :puppet,
+    :classes => ['common', 'web'],
+    :modules => "./modules"
+
+  depends_on :eye do |g|
+    {:aggregator_ip => g.take(:first)['nic'][0]['ip']}
+  end
+
+  # Register custom metrics
+  register 'connection_waiting_time',
+    :command => "/usr/bin/tail -n 1 /var/log/haproxy.log | cut -d ' ' -f 10 | cut -d '/' -f 2",
+    :type => :numeric
+end
+
+# App servers
+group :app do
+  at "fr-inria"
+  at "de-hlrs"
+  instance_type "small"
+  connect_to conf[:wan]
+  deploy conf[:squeeze]
+  provider :puppet,
+    :classes => ['common', 'app'],
+    :modules => "./modules"
+
+  depends_on :eye do |g|
+    {:aggregator_ip => g.take(:first)['nic'][0]['ip']}
+  end
+
+  depends_on :web do |g|
+    {:router_ip => g.take(:first)['nic'][0]['ip']}
+  end
+
+  # Scaling
+  scale 1..10, {
+    :initial => 2,
+    :up => lambda {|engine|
+      values = engine.metric("connection_waiting_time",
+        :hosts => engine.group(:web).take(:first),
+        :type => :numeric
+      ).values[0..3]
+      puts "Metric values: #{values.inspect}, avg=#{values.avg.inspect}"
+      values.avg >= 600
+    },
+    :down => lambda {|engine|
+      engine.metric("connection_waiting_time",
+        :hosts => engine.group(:web).take(:first),
+        :type => :numeric
+      ).values[0..5].avg < 200
+    },
+    :period => 90,
+    :placement => :round_robin
+  }
+end
+
+# All groups are "ready", launch an HTTP benchmarking tool against web's first
+# resource on public interface:
+on :ready do
+  sleep 20
+  cmd = "ab -r -c 8 -n 10000 http://#{group(:web).first['nic'].find{|n| n['ip'] =~ /^131/}['ip']}/delay?delay=0.3"
+  puts "***********"
+  puts cmd
+  system cmd
+end
+
+# Define your networks
+network :public do |name, location|
+  case location['name']
+  when "fr-inria"
+    location.networks.find{|network| network['name'] =~ /Public Network/i}
+  end
+end

data/examples/ibbt.rb

@@ -0,0 +1,125 @@
+# $ bfire elasticity.rb
+
+# Define global properties
+set :name, "BonFIRE elasticity experiment"
+set :key, "~/.ssh/demo"
+set :authorized_keys, "~/.ssh/demo.pub"
+set :walltime, 4*3600
+set :gateway, "ssh.bonfire.grid5000.fr"
+set :user, ENV['USER']
+set :logging, DEBUG
+
+set :squeeze, "BonFIRE Debian Squeeze 2G v2"
+set :zabbix, "BonFIRE Zabbix Aggregator v4"
+set :wan, "BonFIRE WAN"
+
+
+# Monitoring
+group :eye, :tag => "BonFIRE-monitor" do
+  at "fr-inria"
+  instance_type "small"
+  deploy conf[:zabbix]
+  connect_to conf[:wan]
+end
+
+# HTTP Routing
+# group :web do
+#   at "fr-inria"
+#   instance_type 'small'
+#   deploy conf[:squeeze]
+#   # Two interfaces for the publicly facing server
+#   connect_to conf[:wan]
+#   connect_to :public
+# 
+#   provider :puppet,
+#     :classes => ['common', 'web'],
+#     :modules => "./modules"
+# 
+#   depends_on :eye do |g|
+#     {:aggregator_ip => g.take(:first)['nic'][0]['ip']}
+#   end
+# 
+#   # Register custom metrics
+#   register 'connection_waiting_time',
+#     :command => "/usr/bin/tail -n 1 /var/log/haproxy.log | cut -d ' ' -f 10 | cut -d '/' -f 2",
+#     :type => :numeric
+# end
+
+group :servers do
+  at "be-ibbt"
+  connect_to conf[:wan]
+  connect_to :private
+  deploy "BonFIRE Debian Squeeze v1"
+  instance_type "small"
+end
+
+# App servers
+group :clients do
+  at "be-ibbt"
+  instance_type "small"
+  connect_to :private
+  deploy "BonFIRE Debian Squeeze v1"
+  # provider :puppet,
+  #   :classes => ['common', 'app'],
+  #   :modules => "./modules"
+
+  depends_on :eye do |g|
+    {:aggregator_ip => g.take(:first)['nic'][0]['ip']}
+  end
+
+  depends_on :servers do |g|
+    {:iperf_server => g.take(:first)['nic'][0]['ip']}
+  end
+
+  # Scaling
+  scale 1..10, {
+    :initial => 2,
+    # :up => lambda {|engine|
+    #   values = engine.metric("connection_waiting_time",
+    #     :hosts => engine.group(:web).take(:first),
+    #     :type => :numeric
+    #   ).values[0..3]
+    #   puts "Metric values: #{values.inspect}, avg=#{values.avg.inspect}"
+    #   values.avg >= 600
+    # },
+    # :down => lambda {|engine|
+    #   engine.metric("connection_waiting_time",
+    #     :hosts => engine.group(:web).take(:first),
+    #     :type => :numeric
+    #   ).values[0..5].avg < 200
+    # },
+    # :period => 90,
+    :placement => :round_robin
+  }
+end
+
+# All groups are "ready", launch an HTTP benchmarking tool against web's first
+# resource on public interface:
+# on :ready do
+#   sleep 20
+#   cmd = "ab -r -c 8 -n 10000 http://#{group(:web).first['nic'].find{|n| n['ip'] =~ /^131/}['ip']}/delay?delay=0.3"
+#   puts "***********"
+#   puts cmd
+#   system cmd
+# end
+
+network :private do |name, location, experiment|
+  experiment.networks.submit(
+    :location => location,
+    :name => "network-experiment#{experiment['id']}",
+    :bandwidth => 10,
+    :latency => 0,
+    :size => 24,
+    :lossrate => 0,
+    # You MUST specify the address:
+    :address => "192.168.0.1"
+  )
+end
+
+# Define your networks
+# network :public do |name, location|
+#   case location['name']
+#   when "fr-inria"
+#     location.networks.find{|network| network['name'] =~ /Public Network/i}
+#   end
+# end

data/examples/mine.rb

@@ -0,0 +1,40 @@
+use Bfire::API::Bonfire, {
+  :username => "crohr",
+  :password => "toto",
+  :gateway => "ssh.bonfire.grid5000.fr"
+}
+
+set :debug, INFO
+
+set :walltime, 3600
+set :logging, INFO
+
+set :squeeze, "BonFIRE Debian Squeeze 2G v1"
+set :zabbix, "BonFIRE Zabbix Aggregator v2"
+set :wan, "BonFIRE WAN"
+
+
+group :app do
+  attach :db4g
+  
+  at "fr-inria" do
+    type "lite"
+    connect "BonFIRE WAN"
+    connect :private
+  end
+  
+  at "us-eastc1" do
+    type "t1.micro"
+  end
+end
+
+
+network :private do
+  visibility :private
+  cidr "192.168.0.0/24"
+end
+
+storage :db4g do
+  size 4.GB
+  fstype "ext3"
+end

data/examples/modules/apache2/manifests/init.pp

@@ -0,0 +1,44 @@
+class apache2 {
+  include "apache2::${operatingsystem}"
+}
+
+# Class: apache2::debian
+#
+#
+class apache2::debian {
+  package { "apache2":
+    ensure  => installed,
+  }
+  
+  service { "apache2":
+    ensure  => running,
+    enable  => true,
+    require => Package["apache2"];
+  }
+}
+
+# Class: apache2::ubuntu
+#
+#
+class apache2::ubuntu {
+  include apache2::debian
+}
+
+# Class: apache2::centos
+#
+#
+class apache2::centos {
+  package { "httpd":
+    ensure  => installed,
+  }
+  
+  service { "httpd":
+    ensure  => running,
+    enable  => true,
+    require => Package["httpd"];
+  }
+}
+
+
+
+

data/examples/modules/app/files/app/app.rb

@@ -0,0 +1,29 @@
+require 'sinatra'
+
+def registered?
+  cmd = '. /etc/default/bonfire && curl -f http://$ROUTER_IP:8000/hosts -X POST -d "ip=$WAN_IP"'
+  system(cmd)
+  $?.exitstatus == 0
+rescue Exception => e
+  puts "Received #{e.class.name}: #{e.message}"
+  false
+end
+
+Thread.new{
+  # Register with the server on launch
+  # Not efficient but works
+  until registered? do
+    puts "ROUTER not ready yet."
+    sleep 3
+  end
+}
+
+get '/' do
+  "UP"
+end
+
+get '/delay' do
+  delay = (params[:delay] || 0).to_f
+  sleep delay
+  "Slept for #{delay}s."
+end

data/examples/modules/app/files/app/config.ru

@@ -0,0 +1,2 @@
+require 'app'
+run Sinatra::Application

data/examples/modules/app/files/app.phtml

@@ -0,0 +1,4 @@
+<?
+  sleep($_GET['sleep']);
+  print("Hello");
+?>

data/examples/modules/app/manifests/init.pp

@@ -0,0 +1,19 @@
+class app {
+  include sinatra
+  
+  file{"/tmp/app":
+    recurse => true,
+    source => "puppet:///modules/app/app",
+    notify => Service["myapp"]
+  }
+
+  service{"myapp":
+    ensure => running,
+    start => "/usr/bin/thin -d -l /var/log/thin.log -p 80 -R config.ru -c /tmp/app --tag myapp start",
+    stop => "/usr/bin/thin -d -l /var/log/thin.log -p 80 -R config.ru -c /tmp/app --tag myapp stop",
+    require => [
+      Package["libsinatra-ruby"],
+      File["/tmp/app"]
+    ]
+  }
+}

data/examples/modules/common/manifests/init.pp

@@ -0,0 +1,8 @@
+class common {
+  package{"curl":
+    ensure => installed
+  }
+  package{"vim":
+    ensure => installed
+  }
+}

data/examples/modules/haproxy/files/default

@@ -0,0 +1,4 @@
+# Set ENABLED to 1 if you want the init script to start haproxy.
+ENABLED=1
+# Add extra flags here.
+#EXTRAOPTS="-de -m 16"

data/examples/modules/haproxy/files/haproxy.rsyslog.conf

@@ -0,0 +1,2 @@
+local0.* /var/log/haproxy.log
+# local0.* -/var/log/syslog

data/examples/modules/haproxy/manifests/init.pp

@@ -0,0 +1,21 @@
+class haproxy {
+  include rsyslog
+
+  package{"haproxy":
+    ensure => installed
+  }
+  service{"haproxy":
+    ensure => running,
+    require => Package["haproxy"]
+  }
+  file{"/etc/default/haproxy":
+    source => "puppet:///modules/haproxy/default",
+    require => Package["haproxy"],
+    notify => Service["haproxy"]
+  }
+  file{"/etc/rsyslog.d/haproxy.conf":
+    source => "puppet:///modules/haproxy/haproxy.rsyslog.conf",
+    require => [Package["rsyslog"], Package["haproxy"]],
+    notify => Service["rsyslog"]
+  }
+}