ant_hill 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in ant_hill.gemspec
4
+ gemspec
@@ -0,0 +1 @@
1
+ load 'lib/tasks/ant_hill.rake'
@@ -0,0 +1,29 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "ant_hill/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "ant_hill"
7
+ s.version = AntHill::VERSION
8
+ s.authors = ["Ivan Neverov"]
9
+ s.email = ["ineverov@sphereconsultinginc.com"]
10
+ s.homepage = ""
11
+ s.summary = %q{Run tests in grid}
12
+ s.description = %q{Application for running stuff with same purpose on several nodes
13
+ It find "best matching job" (based on setup time) for particular node and setup node
14
+ Then it run job on this node
15
+
16
+ Originally it was desined for CI.}
17
+
18
+ s.rubyforge_project = "ant_hill"
19
+
20
+ s.files = `git ls-files`.split("\n")
21
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
22
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
23
+ s.require_paths = ["lib"]
24
+
25
+ # specify any dependencies here; for example:
26
+ #s.add_development_dependency "ruby-debug19"
27
+ s.add_runtime_dependency "net-ssh"
28
+ s.add_development_dependency "rspec"
29
+ end
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/ruby
2
+ require 'rubygems'
3
+ require 'ant_hill'
4
+ #AntHill::Configuration.config
5
+ queen = nil
6
+ if ARGV.size == 1
7
+ queen = AntHill::Queen.queen
8
+ elsif ARGV.size == 2
9
+ queen = AntHill::Queen.queen
10
+ filename = ARGV[1]
11
+ queen.restore_queen(filename)
12
+ end
13
+ queen.service
14
+
@@ -0,0 +1,36 @@
1
+ require 'yaml'
2
+ require 'logger'
3
+ require 'fileutils'
4
+ require 'drb'
5
+
6
+ # Require file from ant_hill directory
7
+ def require_ant_hill(file)
8
+ require "ant_hill/#{file}"
9
+ end
10
+ private :require_ant_hill
11
+
12
+ # Main module
13
+ module AntHill
14
+ end
15
+
16
+ # Instance of job
17
+ require_ant_hill 'ant'
18
+ # Configuration
19
+ require_ant_hill 'configuration'
20
+ # Node
21
+ require_ant_hill 'creep'
22
+ # Main object
23
+ require_ant_hill 'queen'
24
+ # Set of jobs with same logic
25
+ require_ant_hill 'ant_colony'
26
+ # "Colony" specific logic for setting up and running job
27
+ require_ant_hill 'creep_modifier'
28
+ # Base connection class
29
+ require_ant_hill 'connection_pool'
30
+ # SSH connection
31
+ require_ant_hill 'connections/ssh_connection'
32
+ # Gem version
33
+ require_ant_hill 'version'
34
+ # logger
35
+ require_ant_hill 'log'
36
+
@@ -0,0 +1,126 @@
1
+ module AntHill
2
+ # Instance of job to process
3
+ class Ant
4
+ # Attribute readers
5
+ # +type+:: +AntColony+ type
6
+ # +colony+:: +AntColony+
7
+ # +status+:: ant status
8
+ # +config+:: configuration
9
+ # +params+:: +AntColony+ params_for_ant and ant specific params
10
+ attr_reader :type, :colony, :status, :config, :params
11
+ # Attribute accessors
12
+ # +execution_status+:: status of run in +CreepModifier+
13
+ # +runner+:: +Creep+ there ant is processing or was processed
14
+ # +prior+:: base priority of ant
15
+ # +output+:: return value of run method of +CreepModifier+
16
+ attr_accessor :execution_status, :runner, :prior, :output
17
+ include DRbUndumped
18
+
19
+ # Initialize method
20
+ # +params+:: +Ant+ specific params
21
+ # +colony+:: +AntColony+ which +Ant+ belongs to
22
+ # [+config+]:: configuration
23
+ def initialize(params, colony, config = Configuration.config)
24
+ @colony = colony
25
+ @config = config
26
+ @output = ''
27
+ # Ant params are colony params_for_ant + specific ant params
28
+ @params = @colony.params_for_ant.merge(params)
29
+
30
+ @status = :not_started
31
+ @execution_status = :queued
32
+ # Cache priorities for each creep for faster access
33
+ @cached_priorities = {}
34
+ # Ant type is colony type
35
+ @type = colony.type
36
+ # Set initial priority to queen start time - Time.now so ants created later will have lower priority
37
+ @prior = config.init_time - Time.now
38
+ # Add colony priority
39
+ @prior += colony.get_priority
40
+ end
41
+
42
+ # Cache of creeps priorities
43
+ def priority_cache(creep)
44
+ @cached_priorities[creep] ||= creep.priority(self)
45
+ end
46
+
47
+ # Delete priority cache for specified creep
48
+ # +creep+:: +Creep+ for which delete cache
49
+ def delete_cache_for_creep(creep)
50
+ @cached_priorities.delete(creep)
51
+ end
52
+
53
+ # Deelte all priprities cahce
54
+ def delete_cache
55
+ @cached_priorities = {}
56
+ end
57
+
58
+ # Create string representation of Ant
59
+ def to_s
60
+ @colony.ant_to_s(self)
61
+ end
62
+
63
+ # Show diff between ant params and ant colony params
64
+ def diff_with_colony
65
+ colony_params = colony.params
66
+ params.inject({}){ |res, kv|
67
+ res[kv[0]] = kv[1] if colony_params[kv[0]] != kv[1]
68
+ res
69
+ }
70
+ end
71
+
72
+ # Create Ant from hash
73
+ def from_hash(data)
74
+ @type = data[:type]
75
+ @status = data[:status]
76
+ @executeion_status = data[:executeion_status]
77
+ @prior = data[:prior]
78
+ @output = data[:output]
79
+ end
80
+
81
+ # Convert Ant to hash
82
+ def to_hash
83
+ {
84
+ :type => @type,
85
+ :params => diff_with_colony,
86
+ :status => @status,
87
+ :execution_status => @execution_status,
88
+ :prior => @prior,
89
+ :output => @output
90
+ }
91
+ end
92
+
93
+ # Re-process current ant
94
+ def return_to_queue(queen = Queen.queen)
95
+ queen.add_ants([self])
96
+ end
97
+
98
+ # Update status for ant
99
+ # +status+:: new status
100
+ def change_status(status)
101
+ @status = status
102
+ end
103
+
104
+ # return logger for ant_colony
105
+ def logger
106
+ colony.logger
107
+ end
108
+
109
+ # Start ant processing
110
+ def start
111
+ change_status(:started)
112
+ colony.colony_ant_started
113
+ end
114
+
115
+ # Finish ant processing
116
+ def finish
117
+ change_status(:finished)
118
+ colony.colony_ant_finished
119
+ end
120
+
121
+ # Check if ant had been finished
122
+ def finished?
123
+ status == :finished
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,212 @@
1
+ module AntHill
2
+
3
+ # Object that find Ants and store them
4
+ class AntColony
5
+ # Attribute accessors
6
+ # +params+:: +AntColony+ params
7
+ # +ants+:: array of +Ant+'s for this colony
8
+ attr_accessor :params, :ants
9
+
10
+ # Attribute reader
11
+ # +logger+:: logger for AntColony
12
+ attr_reader :logger
13
+
14
+ include DRbUndumped
15
+
16
+ # Initailize of +AntColony+
17
+ # +params+:: params for colony
18
+ # [+config+]:: configuration
19
+ def initialize(params={}, config = Configuration.config )
20
+ @params = params
21
+ @config = config
22
+ @logger = Log.logger_for(:ant_colony, config)
23
+ @created_at = Time.now
24
+ @ants = []
25
+ @started = false
26
+ end
27
+
28
+ # Create +AntColony+ from hash
29
+ def from_hash(hash = nil)
30
+ if hash
31
+ @started = hash[:started]
32
+ @params = hash[:params]
33
+ @ants = hash[:ants].collect{|ant_data|
34
+ ant = Ant.new(ant_data[:params], self)
35
+ ant.from_hash(ant_data)
36
+ ant
37
+ }
38
+ end
39
+ end
40
+
41
+ # Convert +AntColony+ into hash
42
+ # +include_finished+:: include in hash finished +Ant+'s (default: false)
43
+ def to_hash(include_finished = false)
44
+ _ants = @ants
45
+ _ants = @ants.select{|a| !a.finished?} unless include_finished
46
+ {
47
+ :id => object_id,
48
+ :started => @started,
49
+ :params => @params,
50
+ :ants => _ants.collect{|a| a.to_hash}
51
+ }
52
+ end
53
+
54
+ # Ger +CreepModifier+ class for +AntColony+ type
55
+ def creep_modifier_class
56
+ @creep_modifier_class ||= @config.creep_modifier_class(type)
57
+ return @creep_modifier_class if @creep_modifier_class
58
+ logger.error "Colony will die without creep modifier ;("
59
+ end
60
+
61
+ # Params will be inherited by +Ant+s
62
+ def params_for_ant
63
+ params.inject({}) do |hash,kv|
64
+ if !inherited_params || inherited_params.include?(kv[0])
65
+ hash[kv[0]]=kv[1]
66
+ end
67
+ hash
68
+ end
69
+ end
70
+
71
+ # return true if no +CreepModifier+ found for colony
72
+ def spoiled?
73
+ !@creep_modifier
74
+ end
75
+
76
+ # Find ants for colony params
77
+ def get_ants
78
+ @ants = []
79
+ ant_larvas = search_ants(params)
80
+ @ants = ant_larvas.collect{|larva|
81
+ Ant.new(larva, self)
82
+ }
83
+ after_search
84
+ @ants
85
+ rescue => e
86
+ logger.error "Error while processing search ants for colony\n#{e}\n#{e.backtrace}"
87
+ # Retry 3 times, esle return []
88
+ retries ||= 0
89
+ retries += 1
90
+ retry if retries < 3
91
+ []
92
+ ensure
93
+ # FIXME: Trigger colony finished if no ants were found
94
+ colony_ant_finished
95
+ end
96
+
97
+ # Check if colony matches params
98
+ # +params+:: params to match
99
+ def is_it_me?(params)
100
+ params.all? do |key, value|
101
+ @params[key] == value
102
+ end
103
+ end
104
+
105
+ # Return list of not finished ants
106
+ def not_finished
107
+ ants.select{|ant| ant.finished? }
108
+ end
109
+
110
+ # Check if colony had been finished
111
+ def finished?
112
+ ants.all?{ |a| a.finished? } || ants.empty?
113
+ end
114
+
115
+ # Return logger
116
+ def logger
117
+ Log.logger_for :ant_colony
118
+ end
119
+
120
+ # Trigger colony_started if not already started
121
+ def colony_ant_started
122
+ # FIXME: Dont require any arguments
123
+ unless @started
124
+ @started = true
125
+ begin
126
+ colony_started
127
+ rescue => e
128
+ logger.error "There was an error processing colony_started method for #{self.class}: #{e}\n#{e.backtrace}"
129
+ end
130
+ end
131
+ @started ||= true
132
+ end
133
+
134
+ # Trigger colony_finished if all ants are finished
135
+ def colony_ant_finished
136
+ # FIXME: Dont require any arguments
137
+ if finished?
138
+ begin
139
+ colony_finished
140
+ rescue => e
141
+ logger.error "There was an error processing colony_finished method for #{self.class}: #{e}\n#{e.backtrace}"
142
+ ensure
143
+ Queen.queen.kill_colony(self)
144
+ end
145
+ end
146
+ end
147
+
148
+ # Colony type
149
+ def type
150
+ @params['type']
151
+ end
152
+
153
+ # Calculate priority for colony
154
+ def get_priority
155
+ pr = 0
156
+ begin
157
+ pr = priority
158
+ rescue => e
159
+ logger.error "There was an error processing priority method for #{self.class}: #{e}\n#{e.backtrace}"
160
+ ensure
161
+ return pr
162
+ end
163
+ end
164
+
165
+ # Mark all unprocessed ants as finished
166
+ def kill
167
+ ants.each do |ant|
168
+ ant.change_status(:finished) if ant.status == :not_started
169
+ end
170
+ end
171
+
172
+ # Return array of params hashes for ants
173
+ # default: empty array
174
+ # Should be redefined in child class
175
+ def search_ants(params)
176
+ []
177
+ end
178
+
179
+ # Return AntColony priority
180
+ # default: -created_at.to_int
181
+ # Can be redefined in child class
182
+ def priority
183
+ -created_at.to_i
184
+ end
185
+
186
+ # Convert ant to string
187
+ # Can be redefined in child class
188
+ def ant_to_s(ant)
189
+ ant.params.inspect
190
+ end
191
+
192
+ # Actions to perform if colony started
193
+ # Can be redefined in child class
194
+ def colony_started
195
+ end
196
+
197
+ # Actions to perform if colony finished
198
+ # Can be redefined in child class
199
+ def colony_finished
200
+ end
201
+
202
+ # Actions to perform after ants were found
203
+ # Can be redefined in child class
204
+ def after_search
205
+ end
206
+
207
+ # List of params Ants will inherit
208
+ # Can be redefined in child class
209
+ def inherited_params
210
+ end
211
+ end
212
+ end
@@ -0,0 +1,154 @@
1
+ module AntHill
2
+ # Configuration of AntHill
3
+ class Configuration
4
+ # Attribute readers
5
+ # +init_time+:: initialize time
6
+ attr_reader :init_time
7
+ include DRbUndumped
8
+ # Default output string lengths for console monitor
9
+ DEFAULT_MONITOR = { 'hostname_lenght' => 15, 'processed_lenght' => 7}
10
+
11
+ # Initialize
12
+ def initialize
13
+ @init_time = Time.now
14
+ @config_file = ''
15
+ @configuration = {}
16
+ end
17
+
18
+ # Convert config to hash
19
+ def to_hash
20
+ {:init_time => @init_time}
21
+ end
22
+
23
+ # Convert hash to config
24
+ def from_hash(hash)
25
+ if hash
26
+ @init_time = hash[:init_time]
27
+ end
28
+ end
29
+
30
+ # Parse configuration file
31
+ # +filename+:: configuration file path
32
+ def parse_yaml(filename)
33
+ @config_file = filename
34
+ begin
35
+ @configuration = YAML::load_file(filename)
36
+ rescue => ex
37
+ STDERR.puts "Couldn't find config file #{filename}"
38
+ STDERR.puts ex
39
+ STDERR.puts ex.backtrace
40
+ exit(1)
41
+ end
42
+ end
43
+
44
+ # Require ant_hill implementations
45
+ def require_libs
46
+ basedir = @configuration['basedir']
47
+ lib_path = @configuration['lib_path']
48
+ $LOAD_PATH << basedir
49
+ require File.join(basedir,lib_path)
50
+ rescue LoadError => e
51
+ STDERR.puts "Configuration file is invalid! No such file exists #{File.join(basedir, lib_path)}\n#{e}\n#{e.backtrace}"
52
+ end
53
+
54
+ # Validate minimum configuration is set
55
+ def validate
56
+ strict_attrs = ['basedir', 'lib_path', 'types', 'creeps', 'log_dir', 'log_level']
57
+ error = false
58
+ unless strict_attrs.all?{|a| @configuration[a]}
59
+ STDERR.puts "Configuration file is invalid! Pls. define #{strict_attrs.find{|a| !@configuration[a]}.inspect} keys in it"
60
+ exit(1)
61
+ end
62
+ if @configuration['types'].length == 0
63
+ STDERR.puts "Configuration file is invalid! Pls. define at least one colony type in types section"
64
+ exit(1)
65
+ end
66
+ if @configuration['creeps'].length == 0
67
+ STDERR.puts "Configuration file is invalid! Pls. define at least one creep type in creeps section"
68
+ exit(1)
69
+ end
70
+ end
71
+
72
+ # return class for colony type
73
+ # +type+:: AntColony type, if nil defult_type key will be used
74
+ def ant_colony_class(type=nil)
75
+ get_class_by_type_and_object(type || default_type, 'ant_colony_class')
76
+ end
77
+
78
+ # Return +CreepModifier+ class for colony type
79
+ # +type+:: AntColony type, if nil defult_type key will be used
80
+ def creep_modifier_class(type=nil)
81
+ get_class_by_type_and_object(type || default_type, 'creep_modifier_class')
82
+ end
83
+
84
+ # Get class for given parameters
85
+ # +type+:: colony type
86
+ # +object+:: ant_colony_class or creep_modifier_class
87
+ def get_class_by_type_and_object(type, object)
88
+ if @configuration['types'][type] && klass = @configuration['types'][type][object]
89
+ return get_const_by_name(klass)
90
+ else
91
+ Log.logger_for(:configuration).error("No class configuration defined for #{object} and type #{type}")
92
+ end
93
+ return nil
94
+ end
95
+
96
+ # Get constant by name
97
+ # +name+:: constant name
98
+ def get_const_by_name(name)
99
+ consts = name.split("::")
100
+ obj = Object
101
+ begin
102
+ consts.each{|const|
103
+ obj = obj.const_get(const)
104
+ }
105
+ rescue
106
+ Log.logger_for(:configuration).error("No such class defined: #{name}")
107
+ end
108
+ return obj
109
+ end
110
+
111
+ # Class used for accesing creep node
112
+ def get_connection_class
113
+ get_const_by_name(connection_class)
114
+ end
115
+
116
+ # monitor configuration
117
+ def monitor
118
+ return @monitor if @monitor
119
+ @monitor = DEFAULT_MONITOR
120
+ config = @configuration['monitor'] || {}
121
+ @monitor = @monitor.merge(config)
122
+ end
123
+
124
+ # get configuration value by name
125
+ def [](key)
126
+ @configuration[key.to_s]
127
+ end
128
+
129
+ # allow to use config.<key> syntax
130
+ # +key+:: return value for specified key
131
+ def method_missing(key, *args)
132
+ meth = key.to_s.gsub(/^get_/, "")
133
+ if @configuration.has_key?(meth)
134
+ @configuration[meth]
135
+ else
136
+ STDERR.puts "No key #{meth} defined in #{@config_file}"
137
+ end
138
+ end
139
+
140
+ # Singleton object
141
+ class << self
142
+ # Return configuration object
143
+ # [+filename+]:: config filename or first argument
144
+ def config(filename = ARGV[0])
145
+ return @@config if defined?(@@config)
146
+ @@config = self.new
147
+ @@config.parse_yaml(filename)
148
+ @@config.validate
149
+ @@config.require_libs
150
+ @@config
151
+ end
152
+ end
153
+ end
154
+ end