ant_hill 0.3.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ant_hill.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1 @@
+load 'lib/tasks/ant_hill.rake'

data/ant_hill.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "ant_hill/version"
+
+Gem::Specification.new do |s|
+  s.name        = "ant_hill"
+  s.version     = AntHill::VERSION
+  s.authors     = ["Ivan Neverov"]
+  s.email       = ["ineverov@sphereconsultinginc.com"]
+  s.homepage    = ""
+  s.summary     = %q{Run tests in grid}
+  s.description = %q{Application for running stuff with same purpose on several nodes
+It find "best matching job" (based on setup time) for particular node and setup node
+Then it run job on this node
+
+Originally it was desined for CI.}
+
+  s.rubyforge_project = "ant_hill"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  #s.add_development_dependency "ruby-debug19"
+  s.add_runtime_dependency "net-ssh"
+  s.add_development_dependency "rspec"
+end

data/bin/spawn_queen

@@ -0,0 +1,14 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'ant_hill'
+#AntHill::Configuration.config
+queen = nil
+if ARGV.size == 1
+  queen = AntHill::Queen.queen
+elsif ARGV.size == 2
+  queen = AntHill::Queen.queen
+  filename = ARGV[1]
+  queen.restore_queen(filename)
+end
+queen.service
+

data/lib/ant_hill.rb

@@ -0,0 +1,36 @@
+require 'yaml'
+require 'logger'
+require 'fileutils'
+require 'drb'
+
+# Require file from ant_hill directory
+def require_ant_hill(file)
+  require "ant_hill/#{file}"
+end
+private :require_ant_hill
+
+# Main module
+module AntHill
+end
+
+# Instance of job
+require_ant_hill 'ant'
+# Configuration
+require_ant_hill 'configuration'
+# Node
+require_ant_hill 'creep'
+# Main object 
+require_ant_hill 'queen'
+# Set of jobs with same logic
+require_ant_hill 'ant_colony'
+# "Colony" specific logic for setting up and running job
+require_ant_hill 'creep_modifier'
+# Base connection class
+require_ant_hill 'connection_pool'
+# SSH connection
+require_ant_hill 'connections/ssh_connection'
+# Gem version
+require_ant_hill 'version'
+# logger
+require_ant_hill 'log'
+

data/lib/ant_hill/ant.rb

@@ -0,0 +1,126 @@
+module AntHill
+  # Instance of job to process
+  class Ant
+    # Attribute readers
+    # +type+:: +AntColony+ type
+    # +colony+:: +AntColony+
+    # +status+:: ant status
+    # +config+:: configuration
+    # +params+:: +AntColony+ params_for_ant and ant specific params
+    attr_reader :type, :colony, :status, :config, :params
+    # Attribute accessors
+    # +execution_status+:: status of run in +CreepModifier+
+    # +runner+:: +Creep+ there ant is processing or was processed
+    # +prior+:: base priority of ant
+    # +output+:: return value of run method of +CreepModifier+
+    attr_accessor :execution_status, :runner, :prior, :output
+    include DRbUndumped
+
+    # Initialize method
+    # +params+:: +Ant+ specific params
+    # +colony+:: +AntColony+ which +Ant+ belongs to
+    # [+config+]:: configuration
+    def initialize(params, colony, config = Configuration.config)
+      @colony = colony
+      @config = config
+      @output = ''
+      # Ant params are colony params_for_ant + specific ant params
+      @params = @colony.params_for_ant.merge(params)
+
+      @status = :not_started
+      @execution_status = :queued
+      # Cache priorities for each creep for faster access
+      @cached_priorities = {}
+      # Ant type is colony type
+      @type = colony.type
+      # Set initial priority to queen start time - Time.now so ants created later will have lower priority
+      @prior = config.init_time - Time.now
+      # Add colony priority
+      @prior += colony.get_priority
+    end
+
+    # Cache of creeps priorities
+    def priority_cache(creep)
+      @cached_priorities[creep] ||= creep.priority(self)
+    end
+
+    # Delete priority cache for specified creep
+    # +creep+:: +Creep+ for which delete cache
+    def delete_cache_for_creep(creep)
+      @cached_priorities.delete(creep)
+    end
+
+    # Deelte all priprities cahce
+    def delete_cache
+      @cached_priorities = {}
+    end
+
+    # Create string representation of Ant
+    def to_s
+      @colony.ant_to_s(self)
+    end
+
+    # Show diff between ant params and ant colony params
+    def diff_with_colony
+      colony_params = colony.params
+      params.inject({}){ |res, kv|
+        res[kv[0]] = kv[1] if colony_params[kv[0]] != kv[1]
+        res
+      }
+    end
+
+    # Create Ant from hash
+    def from_hash(data)
+      @type = data[:type]
+      @status = data[:status]
+      @executeion_status = data[:executeion_status]
+      @prior = data[:prior]
+      @output = data[:output]
+    end
+
+    # Convert Ant to hash
+    def to_hash
+      {
+        :type => @type,
+        :params => diff_with_colony,
+        :status => @status,
+        :execution_status => @execution_status,
+        :prior => @prior,
+        :output => @output
+      }
+    end
+
+    # Re-process current ant
+    def return_to_queue(queen = Queen.queen)
+      queen.add_ants([self])
+    end
+
+    # Update status for ant
+    # +status+:: new status
+    def change_status(status)
+      @status = status
+    end
+
+    # return logger for ant_colony
+    def logger
+      colony.logger
+    end
+
+    # Start ant processing
+    def start
+      change_status(:started)
+      colony.colony_ant_started
+    end
+
+    # Finish ant processing
+    def finish
+      change_status(:finished)
+      colony.colony_ant_finished
+    end
+
+    # Check if ant had been finished
+    def finished?
+      status == :finished
+    end
+  end
+end

data/lib/ant_hill/ant_colony.rb

@@ -0,0 +1,212 @@
+module AntHill
+
+  # Object that find Ants and store them 
+  class AntColony
+    # Attribute accessors
+    # +params+:: +AntColony+ params
+    # +ants+:: array of +Ant+'s for this colony
+    attr_accessor :params, :ants
+    
+    # Attribute reader
+    # +logger+:: logger for AntColony
+    attr_reader :logger
+    
+    include DRbUndumped
+
+    # Initailize of +AntColony+
+    # +params+:: params for colony
+    # [+config+]:: configuration
+    def initialize(params={}, config = Configuration.config )
+      @params = params
+      @config = config
+      @logger = Log.logger_for(:ant_colony, config)
+      @created_at = Time.now
+      @ants = []
+      @started = false
+    end
+
+    # Create +AntColony+ from hash
+    def from_hash(hash = nil)
+      if hash
+        @started = hash[:started]
+        @params = hash[:params]
+        @ants = hash[:ants].collect{|ant_data|
+          ant = Ant.new(ant_data[:params], self)
+          ant.from_hash(ant_data)
+          ant
+        }
+      end
+    end
+
+    # Convert +AntColony+ into hash
+    # +include_finished+:: include in hash finished +Ant+'s (default: false)
+    def to_hash(include_finished = false)
+      _ants = @ants
+      _ants = @ants.select{|a| !a.finished?} unless include_finished
+      {
+        :id => object_id,
+        :started => @started,
+        :params => @params,
+        :ants => _ants.collect{|a| a.to_hash}
+      }
+    end
+
+    # Ger +CreepModifier+ class for +AntColony+ type
+    def creep_modifier_class
+      @creep_modifier_class ||= @config.creep_modifier_class(type)
+      return @creep_modifier_class if @creep_modifier_class
+      logger.error "Colony will die without creep modifier ;("
+    end
+
+    # Params will be inherited by +Ant+s
+    def params_for_ant
+      params.inject({}) do |hash,kv| 
+        if !inherited_params || inherited_params.include?(kv[0])
+          hash[kv[0]]=kv[1]
+        end
+        hash
+      end
+    end
+
+    # return true if no +CreepModifier+ found for colony
+    def spoiled?
+      !@creep_modifier
+    end
+
+    # Find ants for colony params
+    def get_ants
+      @ants = []
+      ant_larvas = search_ants(params)
+      @ants = ant_larvas.collect{|larva|
+        Ant.new(larva, self)
+      }
+      after_search
+      @ants
+    rescue => e
+      logger.error "Error while processing search ants for colony\n#{e}\n#{e.backtrace}"
+      # Retry 3 times, esle return []
+      retries ||= 0
+      retries += 1
+      retry if retries < 3
+      []
+    ensure
+      # FIXME: Trigger colony finished if no ants were found
+      colony_ant_finished
+    end
+
+    # Check if colony matches params
+    # +params+:: params to match
+    def is_it_me?(params)
+      params.all? do |key, value|
+        @params[key] == value
+      end
+    end
+
+    # Return list of not finished ants
+    def not_finished
+      ants.select{|ant|  ant.finished? }
+    end
+
+    # Check if colony had been finished
+    def finished?
+      ants.all?{ |a| a.finished? } || ants.empty?
+    end
+
+    # Return logger
+    def logger
+      Log.logger_for :ant_colony
+    end
+
+    # Trigger colony_started if not already started
+    def colony_ant_started
+      # FIXME: Dont require any arguments
+      unless @started
+        @started = true
+        begin 
+          colony_started
+        rescue => e
+          logger.error "There was an error processing colony_started method for #{self.class}: #{e}\n#{e.backtrace}"
+        end
+      end
+      @started ||= true
+    end
+
+    # Trigger colony_finished if all ants are finished
+    def colony_ant_finished
+      # FIXME: Dont require any arguments
+      if finished?
+        begin 
+          colony_finished
+        rescue => e
+          logger.error "There was an error processing colony_finished method for #{self.class}: #{e}\n#{e.backtrace}"
+        ensure 
+          Queen.queen.kill_colony(self)
+        end
+      end
+    end
+    
+    # Colony type
+    def type
+      @params['type']
+    end
+
+    # Calculate priority for colony
+    def get_priority
+      pr = 0
+      begin
+        pr = priority
+      rescue => e
+        logger.error "There was an error processing priority method for #{self.class}: #{e}\n#{e.backtrace}"
+      ensure
+        return pr
+      end
+    end
+
+    # Mark all unprocessed ants as finished
+    def kill
+      ants.each do |ant|
+        ant.change_status(:finished) if ant.status == :not_started
+      end
+    end
+
+    # Return array of params hashes for ants
+    # default: empty array
+    # Should be redefined in child class
+    def search_ants(params)
+      []
+    end
+
+    # Return AntColony priority
+    # default: -created_at.to_int
+    # Can be redefined in child class
+    def priority
+      -created_at.to_i
+    end
+
+    # Convert ant to string
+    # Can be redefined in child class
+    def ant_to_s(ant)
+      ant.params.inspect
+    end
+    
+    # Actions to perform if colony started 
+    # Can be redefined in child class
+    def colony_started
+    end
+
+    # Actions to perform if colony finished
+    # Can be redefined in child class
+    def colony_finished
+    end
+    
+    # Actions to perform after ants were found
+    # Can be redefined in child class
+    def after_search
+    end
+
+    # List of params Ants will inherit
+    # Can be redefined in child class
+    def inherited_params
+    end
+  end
+end

data/lib/ant_hill/configuration.rb

@@ -0,0 +1,154 @@
+module AntHill
+  # Configuration of AntHill
+  class Configuration
+    # Attribute readers
+    # +init_time+:: initialize time
+    attr_reader :init_time
+    include DRbUndumped
+    # Default output string lengths for console monitor
+    DEFAULT_MONITOR = { 'hostname_lenght' => 15, 'processed_lenght' => 7} 
+
+    # Initialize
+    def initialize
+      @init_time = Time.now
+      @config_file = ''
+      @configuration = {}
+    end
+   
+    # Convert config to hash
+    def to_hash
+      {:init_time => @init_time}
+    end
+
+    # Convert hash to config
+    def from_hash(hash)
+      if hash
+        @init_time = hash[:init_time]
+      end
+    end 
+
+    # Parse configuration file
+    # +filename+:: configuration file path
+    def parse_yaml(filename)
+      @config_file = filename
+      begin
+        @configuration = YAML::load_file(filename)
+      rescue => ex
+        STDERR.puts "Couldn't find config file #{filename}"
+        STDERR.puts ex
+        STDERR.puts ex.backtrace
+        exit(1)
+      end
+    end
+
+    # Require ant_hill implementations
+    def require_libs
+      basedir = @configuration['basedir']
+      lib_path = @configuration['lib_path']
+      $LOAD_PATH << basedir
+      require File.join(basedir,lib_path)
+    rescue LoadError => e
+      STDERR.puts "Configuration file is invalid! No such file exists #{File.join(basedir, lib_path)}\n#{e}\n#{e.backtrace}" 
+    end
+
+    # Validate minimum configuration is set
+    def validate
+      strict_attrs = ['basedir', 'lib_path', 'types', 'creeps', 'log_dir', 'log_level']
+      error = false
+      unless strict_attrs.all?{|a| @configuration[a]}
+        STDERR.puts "Configuration file is invalid! Pls. define #{strict_attrs.find{|a| !@configuration[a]}.inspect} keys in it"
+        exit(1)
+      end
+      if @configuration['types'].length == 0
+        STDERR.puts "Configuration file is invalid! Pls. define at least one colony type in types section"
+        exit(1)
+      end
+      if @configuration['creeps'].length == 0
+        STDERR.puts "Configuration file is invalid! Pls. define at least one creep type in creeps section"
+        exit(1)
+      end 
+    end
+
+    # return class for colony type
+    # +type+:: AntColony type, if nil defult_type key will be used
+    def ant_colony_class(type=nil)
+      get_class_by_type_and_object(type || default_type, 'ant_colony_class')
+    end
+
+    # Return +CreepModifier+ class for colony type
+    # +type+:: AntColony type, if nil defult_type key will be used
+    def creep_modifier_class(type=nil)
+      get_class_by_type_and_object(type || default_type, 'creep_modifier_class')
+    end
+
+    # Get class for given parameters
+    # +type+:: colony type
+    # +object+:: ant_colony_class or creep_modifier_class
+    def get_class_by_type_and_object(type, object)
+      if @configuration['types'][type] && klass = @configuration['types'][type][object]
+        return get_const_by_name(klass)
+      else
+        Log.logger_for(:configuration).error("No class configuration defined for #{object} and type #{type}")
+      end
+      return nil
+    end
+
+    # Get constant by name
+    # +name+:: constant name
+    def get_const_by_name(name)
+      consts = name.split("::")
+      obj = Object
+      begin
+        consts.each{|const| 
+          obj = obj.const_get(const)
+        }
+      rescue
+        Log.logger_for(:configuration).error("No such class defined: #{name}")
+      end
+      return obj
+    end
+
+    # Class used for accesing creep node
+    def get_connection_class
+      get_const_by_name(connection_class)
+    end
+
+    # monitor configuration
+    def monitor
+      return @monitor if @monitor
+      @monitor = DEFAULT_MONITOR
+      config = @configuration['monitor'] || {}
+      @monitor = @monitor.merge(config)
+    end
+
+    # get configuration value by name
+    def [](key)
+      @configuration[key.to_s]
+    end
+
+    # allow to use config.<key> syntax
+    # +key+:: return value for specified key
+    def method_missing(key, *args)
+      meth = key.to_s.gsub(/^get_/, "")
+      if @configuration.has_key?(meth)
+        @configuration[meth]
+      else
+        STDERR.puts "No key #{meth} defined in #{@config_file}"
+      end
+    end
+
+    # Singleton object
+    class << self
+      # Return configuration object
+      # [+filename+]:: config filename or first argument
+      def config(filename = ARGV[0])
+        return @@config if defined?(@@config)
+        @@config =  self.new
+        @@config.parse_yaml(filename)
+        @@config.validate
+        @@config.require_libs
+        @@config
+      end
+    end
+  end
+end