bfire 0.2.0

data/lib/bfire/group.rb

@@ -0,0 +1,241 @@
+require 'bfire/template'
+require 'bfire/rule'
+
+module Bfire
+  class Group
+    include Enumerable
+    include PubSub::Publisher
+
+    attr_reader :engine
+    attr_reader :name
+    attr_reader :dependencies
+    attr_reader :templates
+    # A free-form text tag to add to every compute name of this group.
+    attr_reader :tag
+
+    def initialize(engine, name, options = {})
+      @engine = engine
+      @name = name
+      @tag = options.delete(:tag)
+      raise Error, "Tag name can't contain two or more consecutive dashes" if @tag && @tag =~ /-{2,}/
+      @options = options
+      @listeners = {}
+      @dependencies = []
+
+      @templates = []
+      @default_template = Template.new(self, :default)
+      @current_template = @default_template
+
+      raise Error, "Group name can only contain [a-zA-Z0-9] characters" if name !~ /[a-z0-9]+/i
+
+      on(:error) {|group| Thread.current.group.list.each{|t|
+          t[:ko] = true
+          t.kill
+        }
+      }
+      on(:ready) {|group|
+        group.engine.logger.info "#{group.banner}All VMs are now READY: #{computes.map{|vm|
+          [vm['name'], (vm['nic'] || []).map{|n| n['ip']}.inspect].join("=")
+        }.join("; ")}"
+      }
+    end
+
+    def launch_initial_resources
+      merge_templates!
+      engine.logger.debug "#{banner}Merged templates=#{templates.inspect}"
+      check!
+      if rule.launch_initial_resources
+        trigger :launched
+        true
+      else
+        trigger :error
+        false
+      end
+    rescue Exception => e
+      engine.logger.error "#{banner}#{e.class.name}: #{e.message}"
+      engine.logger.debug e.backtrace.join("; ")
+      trigger :error
+    end
+
+    def monitor
+      rule.manage(computes)
+      rule.monitor
+    rescue Exception => e
+      engine.logger.error "#{banner}#{e.class.name}: #{e.message}"
+      engine.logger.debug e.backtrace.join("; ")
+      trigger :error
+    end
+
+    def provision!(vms)
+      return true if provider.nil?
+      engine.logger.info "#{banner}Provisioning..."
+      vms.all?{|vm|
+        provisioned = false
+        ip = vm['nic'][0]['ip']
+        engine.ssh(ip, 'root') {|s|
+          provisioned = unless provider.install(s)
+            engine.logger.error "Failed to install provider on #{vm.inspect} (IP=#{ip})."
+            false
+          else
+            result = provider.run(s) do |stream|
+              engine.logger.info "#{banner}[#{ip}] #{stream}"
+            end
+          end
+        }
+        provisioned
+      }
+    end
+
+    # Delegates every unknown method to the current Template, except #conf.
+    def method_missing(method, *args, &block)
+      if method == :conf
+        engine.send(method, *args, &block)
+      else
+        @current_template.send(method, *args, &block)
+      end
+    end
+
+    # ======================
+    # = Group-only methods =
+    # ======================
+
+    def rule
+      @rule ||= Rule.new(self, :initial => 1, :range => 1..1)
+    end
+
+    # Defines the scaling rule for this group
+    def scale(range, options = {})
+      @rule = Rule.new(self, options.merge(:range => range))
+    end
+
+    def at(location, &block)
+      t = template(location)
+      @current_template = t
+      instance_eval(&block) unless block.nil?
+      @current_template = @default_template
+    end
+
+    def depends_on(group_name, &block)
+      @dependencies.push [group_name, block]
+    end
+
+    # Define the provider to use to provision the compute resources
+    # (Puppet, Chef...).
+    # If <tt>selected_provider</tt> is nil, returns the current provider.
+    def provider(selected_provider = nil, options = {})
+      return @provider if selected_provider.nil?
+      options[:modules] = engine.path_to(options[:modules]) if options[:modules]
+      @provider = Provider::Puppet.new(options)
+    end
+
+
+    # ===========
+    # = Helpers =
+    # ===========
+
+    def banner
+      "[#{name}] "
+    end
+
+    # Iterates over the collection of compute resources.
+    # Required for the Enumerable module.
+    def each(*args, &block)
+      computes.each(*args, &block)
+    end
+
+    def computes
+      templates.map{|t| t.instances}.flatten
+    end
+
+    # Return the first <tt>how_many</tt> compute resources of the group.
+    def take(how_many = :all)
+      case how_many
+      when :all
+        computes
+      when :first
+        computes[0]
+      else
+        raise ArgumentError, "You must pass :all, :first, or a Fixnum" unless how_many.kind_of?(Fixnum)
+        computes.take(how_many)
+      end
+    end
+
+    def inspect
+      s = "#<#{self.class.name}:0x#{object_id.to_s(16)}"
+      s << " #{banner}" if banner
+      s << "VMs: "
+      s << computes.map{|vm|
+        [vm['name'].inspect, (vm['nic'] || []).map{|n|
+          n['ip']
+        }.inspect].join("=")
+      }.join("; ")
+      s << ">"
+    end
+
+    def reload
+      each(&:reload)
+    end
+
+    def ssh_accessible?(vms)
+      vms.all?{|compute|
+        begin
+          ip = compute['nic'][0]['ip']
+          Timeout.timeout(30) do
+            engine.ssh(ip, 'root', :log => false) {|s|
+              s.exec!("hostname")
+            }
+          end
+          true
+        rescue Exception => e
+          engine.logger.debug "#{banner}Can't SSH yet to #{compute.signature} at IP=#{ip.inspect}. Reason: #{e.class.name}, #{e.message}. Will retry later."
+          false
+        end
+      }
+    end
+
+    def template(location)
+      t = @templates.find{|t| t.name == location}
+      if t.nil?
+        t = Template.new(
+          self,
+          location
+        )
+        @templates.push(t)
+      end
+      t
+    end
+
+    def check!
+      check_templates!
+      if provider && !provider.valid?
+        raise Error, "#{banner}#{provider.errors.map(&:inspect).join(", ")}"
+      end
+    end
+
+    def merge_templates!
+      default = @default_template
+      if engine.conf[:authorized_keys]
+        default.context :authorized_keys => File.read(
+          File.expand_path(engine.conf[:authorized_keys])
+        )
+      end
+      if @templates.empty?
+        @templates.push template(:any)
+      end
+      templates.each{|t|
+        t.merge_defaults!(default).resolve!
+      }
+    end # def merge_templates!
+
+    protected
+
+    def check_templates!
+      errors = []
+      templates.each do |t|
+        t.valid? || errors.push({t.name => t.errors})
+      end
+      raise Error, "#{banner}#{errors.map(&:inspect).join(", ")}" unless errors.empty?
+    end # def check_templates!
+
+  end
+end

data/lib/bfire/metric.rb

@@ -0,0 +1,36 @@
+class Array
+  def sum
+    inject(:+)
+  end
+  
+  def avg
+    sum.to_f / size
+  end
+  
+  def median
+    sorted = sort
+    (sorted[size/2] + sorted[(size+1)/2]) / 2
+  end
+end
+
+module Bfire
+  class Metric
+
+    def initialize(name, results, opts = {})
+      @name = name
+      @results = results
+      @opts = opts
+    end
+
+    def values
+      @results.map{|r| 
+        case @opts[:type]
+        when :numeric
+          r['value'].to_f
+        else
+          r['value']
+        end
+      }.reverse
+    end
+  end
+end

data/lib/bfire/provider/puppet.rb

@@ -0,0 +1,58 @@
+module Bfire
+  module Provider
+    class Puppet
+      attr_reader :classes
+      attr_reader :modules
+      attr_reader :options
+      attr_reader :errors
+
+      def initialize(opts = {})
+        @classes = opts.delete(:classes) || opts.delete("classes")
+        @modules = opts.delete(:modules) || opts.delete("modules")
+        @options = opts
+        @errors = []
+      end
+
+      def install(ssh_session)
+        res = ssh_session.exec!("apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install curl puppet -y")
+        res = ssh_session.exec!("which puppet")
+        !res.nil? && !res.empty?
+      end
+      
+      def run(ssh_session)
+        ssh_session.exec!("rm -rf /tmp/puppet && mkdir -p /tmp/puppet")
+        ssh_session.scp.upload!(
+          StringIO.new(manifest("vm")),
+          "/tmp/puppet/manifest.pp"
+        )
+        ssh_session.sftp.upload!(modules, "/tmp/puppet/modules")
+        ssh_session.exec!(
+          "puppet --modulepath /tmp/puppet/modules /tmp/puppet/manifest.pp"
+        ) do |ch, stream, data|
+          yield "[#{stream.to_s.upcase}] #{data.chomp}"
+        end
+        true
+      end
+
+      def manifest(name)
+        content = <<MANIFEST
+class #{name} {
+  #{classes.map{|klass| "include #{klass}"}.join("\n")}
+}
+
+include #{name}
+MANIFEST
+      end
+
+      def valid?
+        @errors = []
+        if modules.nil?
+          @errors.push("You must pass a :modules option to `provider`")
+        elsif !File.directory?(modules)
+          @errors.push("#{modules} is not a valid directory")
+        end
+        @errors.empty?
+      end
+    end
+  end
+end

data/lib/bfire/pub_sub/publisher.rb

@@ -0,0 +1,40 @@
+module Bfire
+  module PubSub
+    module Publisher
+      # Notify all group listeners when event <tt>event</tt> occurs.
+      def trigger(event)
+        triggered_events.push(event)
+        engine.logger.info "#{banner}Triggering #{event.inspect} event..."
+        (hooks[event] || []).each{|block| 
+          if block.arity == 1
+            block.call(self)
+          else
+            engine.instance_eval(&block) 
+          end
+        }
+      end
+
+      # Defines a procedure (hook) to launch when event <tt>event</tt> occurs.
+      def on(event, &block)
+        hooks[event.to_sym] ||= []
+        hooks[event.to_sym] << block
+      end
+
+      def error?
+        triggered_events.include?(:error)
+      end
+      
+      def hooks
+        @hooks ||= {}
+      end
+      
+      def triggered_events
+        @triggered_events ||= []
+      end
+      
+      def self.included(mod)
+      end
+
+    end
+  end
+end

data/lib/bfire/rule.rb

@@ -0,0 +1,110 @@
+module Bfire
+  class Rule
+    attr_reader :group
+    attr_reader :opts
+
+    include PubSub::Publisher
+
+    def initialize(group, opts = {})
+      @group = group
+      @opts = {:period => 5*60, :initial => 1, :range => 1..1}.merge(opts)
+    end
+
+    # we only support round-robin placement for now
+    def monitor
+      loop do
+        sleep opts[:period]
+        group.engine.logger.info "#{group.banner}Monitoring group elasticity rule..."
+        # this is blocking because we don't want the rule to be triggered
+        # too many times.
+        if scale_up?
+          group.engine.logger.info "#{group.banner}Scaling up!"
+          manage(scale(:up))
+        elsif scale_down?
+          group.engine.logger.info "#{group.banner}Scaling down!"
+          manage(scale(:down))
+        else
+          group.engine.logger.info "#{group.banner}..."
+        end
+      end
+    end
+
+    def launch_initial_resources
+      scale(:up, opts[:initial])
+    end
+
+    def scale(up_or_down, count = 1)
+      new_computes = []
+      count.times do |i|
+        sorted_templates = group.templates.sort_by{|t| t.instances.length}
+        if up_or_down == :down
+          vm_to_delete = sorted_templates.last.instances[0]
+          if vm_to_delete.nil?
+            group.engine.logger.warn "#{group.banner}No resource to delete!"
+          else
+            group.engine.logger.info "#{group.banner}Removing compute #{vm_to_delete.signature}..."
+            if vm_to_delete.delete
+              sorted_templates.last.instances.delete vm_to_delete
+              group.trigger :scaled_down
+            end
+          end
+        else
+          template = sorted_templates.first
+          computes = group.engine.launch_compute(template)
+          template.instances.push(*computes)
+          new_computes.push(*computes)
+        end
+      end
+      new_computes
+    end
+
+    def manage(vms)
+      return true if vms.empty?
+      group.engine.logger.info "#{group.banner}Monitoring VMs... IPs: #{vms.map{|vm| [vm['name'], (vm['nic'] || []).map{|n| n['ip']}.inspect].join("=")}.join("; ")}."
+      vms.each(&:reload)
+      if failed = vms.find{|compute| compute['state'] == 'FAILED'}
+        group.engine.logger.warn "#{group.banner}Compute #{failed.signature} is in a FAILED state."
+        if group.triggered_events.include?(:ready)
+          group.trigger :error
+        else
+          group.trigger :scale_error
+        end
+      elsif vms.all?{|compute| compute['state'] == 'ACTIVE'}
+        group.engine.logger.info "#{group.banner}All compute resources are ACTIVE"
+        if group.ssh_accessible?(vms)
+          group.engine.logger.info "#{group.banner}All compute resources are SSH-able"
+          provisioned = group.provision!(vms)
+          if group.triggered_events.include?(:ready)
+            if provisioned
+              group.trigger :scaled_up
+            else
+              group.trigger :scale_error
+            end
+          else
+            if provisioned
+              group.trigger :ready
+            else
+              group.trigger :error
+            end
+          end
+          monitor
+        else
+          sleep 20
+          manage(vms)
+        end
+      else
+        group.engine.logger.info "#{group.banner}Some compute resources are still PENDING"
+        sleep 10
+        manage(vms)
+      end
+    end
+
+    def scale_up?
+      opts[:up] && group.computes.length < opts[:range].end && opts[:up].call(group.engine)
+    end
+
+    def scale_down?
+      opts[:down] && group.computes.length > opts[:range].begin && opts[:down].call(group.engine)
+    end
+  end
+end

data/lib/bfire/template.rb

@@ -0,0 +1,142 @@
+module Bfire
+  class Template
+    # Return the list of nics defined.
+    attr_reader :nics
+    # Return the list of disks defined.
+    attr_reader :disks
+    # Return the template name (i.e. the location name).
+    attr_reader :name
+    # Return the properties defined for this template (instance_type, etc.).
+    attr_reader :properties
+    # Return the list of metrics defined.
+    attr_reader :metrics
+    attr_reader :context
+    attr_reader :instances
+
+    # Return an Array of error messages in case this template is not valid.
+    attr_reader :errors
+    # Return the group this template belongs to.
+    attr_reader :group
+
+    def initialize(group, location_name = nil)
+      @group = group
+      @location_name = location_name
+      @name = location_name
+      @nics = []
+      @disks = []
+      @errors = []
+      @metrics = []
+      @properties = {}
+      @context = {}
+      @instances = []
+    end
+
+    def location
+      @location ||= if @location_name == :default
+        # noop
+      else
+        group.engine.fetch_location(@location_name)
+      end
+    end
+
+    def context(opts = {})
+      if opts.empty?
+        @context
+      else
+        @context.merge!(opts)
+      end
+    end
+
+    # Define the instance type to use.
+    def instance_type(instance_type)
+      @properties[:instance_type] = instance_type.to_s
+    end
+
+    # Define the image to deploy on the compute resources.
+    def deploy(storage, options = {})
+      props = options.merge(
+        :storage => storage
+      )
+      if @disks.empty?
+        @disks.push props
+      else
+        @disks[0] = props
+      end
+    end
+
+    def connect_to(network, options = {})
+      @nics.push options.merge(
+        :network => network
+      )
+    end
+
+    # Merge this template with another one.
+    # nics, disks, and metrics will be added, while other properties will be
+    # merged.
+    def merge_defaults!(template)
+      @properties = template.properties.merge(@properties)
+      @context = template.context.merge(@context)
+      template.nics.each do |nic|
+        @nics.unshift nic.clone
+      end
+      template.disks.each do |disk|
+        @disks.unshift disk.clone
+      end
+      template.metrics.each do |metric|
+        @metrics.unshift metric.clone
+      end
+      self
+    end
+
+    # Returns true if valid, false otherwise
+    def valid?
+      @errors = []
+      @errors.push("You must specify an instance_type") unless properties[:instance_type]
+      @errors.push("You must specify at least one disk image") if @disks.empty?
+      @errors.push("You must specify at least one network attachment") if @nics.empty?
+      @errors.empty?
+    end
+
+    # Resolve the networks and storages required for the template to be valid.
+    def resolve!
+      nics.each{|nic|
+        nic[:network] = group.engine.fetch_network(
+          nic[:network],
+          location
+        ) || raise(Error, "Can't find network #{nic[:network].inspect} at #{location["name"].inspect}")
+      }
+      disks.each{|disk|
+        disk[:storage] = group.engine.fetch_storage(
+          disk[:storage],
+          location
+        ) || raise(Error, "Can't find storage #{disk[:storage].inspect} at #{location["name"].inspect}")
+      }
+      self
+    end
+
+    # Register a metric on the compute resources.
+    def register(metric_name, options = {})
+      @metrics.push options.merge(:name => metric_name)
+    end
+
+    # Exports the template to a ruby Hash, which conforms to what is expected
+    # by Restfully to submit a resource.
+    def to_h
+      h = {}
+      h.merge!(@properties)
+      h['name'] = "#{group.name}--#{name}--#{SecureRandom.hex(4)}"
+      h['name'] << "-#{group.tag}" if group.tag
+      h['nic'] = nics
+      h['disk'] = disks
+      h['location'] = location
+      h['context'] = @context
+      h['context']['metrics'] = XML::Node.new_cdata(metrics.map{|m|
+        "<metric>"+[m[:name], m[:command]].join(",")+"</metric>"
+      }.join("")) unless metrics.empty?
+      group.dependencies.each{|gname,block|
+        h['context'].merge!(block.call(group.engine.group(gname)))
+      }
+      h
+    end
+  end # class Template
+end # module Bup

data/lib/bfire/version.rb

@@ -0,0 +1,3 @@
+module Bfire
+  VERSION = "0.2.0"
+end

data/lib/bfire.rb

@@ -0,0 +1,10 @@
+require 'uuidtools'
+
+require 'bfire/pub_sub/publisher'
+require 'bfire/provider/puppet'
+require 'bfire/version'
+require 'bfire/engine'
+
+module Bfire
+  class Error < StandardError; end
+end