bosh-director 1.5.0.pre.1623 → 1.5.0.pre.1633
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/bosh/director/compile_task.rb +0 -1
- data/lib/bosh/director/config.rb +1 -1
- data/lib/bosh/director/deployment_plan.rb +1 -0
- data/lib/bosh/director/deployment_plan/job.rb +0 -2
- data/lib/bosh/director/deployment_plan/multi_job_updater.rb +65 -0
- data/lib/bosh/director/deployment_plan/update_config.rb +11 -1
- data/lib/bosh/director/deployment_plan/updater.rb +23 -30
- data/lib/bosh/director/event_log.rb +100 -88
- data/lib/bosh/director/instance_deleter.rb +5 -7
- data/lib/bosh/director/instance_updater.rb +3 -3
- data/lib/bosh/director/job_runner.rb +1 -1
- data/lib/bosh/director/job_updater.rb +65 -63
- data/lib/bosh/director/jobs/base_job.rb +0 -2
- data/lib/bosh/director/jobs/update_deployment.rb +2 -4
- data/lib/bosh/director/resource_pool_updater.rb +0 -3
- data/lib/bosh/director/validation_helper.rb +1 -5
- data/lib/bosh/director/version.rb +1 -1
- metadata +18 -17
data/lib/bosh/director/config.rb
CHANGED
@@ -80,7 +80,7 @@ module Bosh::Director
|
|
80
80
|
|
81
81
|
# Event logger supposed to be overridden per task,
|
82
82
|
# the default one does nothing
|
83
|
-
@event_log = EventLog.new
|
83
|
+
@event_log = EventLog::Log.new
|
84
84
|
|
85
85
|
# by default keep only last 500 tasks in disk
|
86
86
|
@max_tasks = config.fetch("max_tasks", 500).to_i
|
@@ -13,6 +13,7 @@ require 'bosh/director/deployment_plan/compiled_package'
|
|
13
13
|
require 'bosh/director/deployment_plan/rendered_templates_archive'
|
14
14
|
require 'bosh/director/deployment_plan/preparer'
|
15
15
|
require 'bosh/director/deployment_plan/resource_pools'
|
16
|
+
require 'bosh/director/deployment_plan/multi_job_updater'
|
16
17
|
require 'bosh/director/deployment_plan/updater'
|
17
18
|
require 'bosh/director/deployment_plan/release_version'
|
18
19
|
require 'bosh/director/deployment_plan/resource_pool'
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Bosh::Director
|
2
|
+
module DeploymentPlan
|
3
|
+
class SerialMultiJobUpdater
|
4
|
+
def run(base_job, deployment_plan, jobs)
|
5
|
+
base_job.logger.info("Updating jobs serially: #{jobs.map(&:name).join(', ')}")
|
6
|
+
|
7
|
+
jobs.each do |j|
|
8
|
+
base_job.task_checkpoint
|
9
|
+
base_job.logger.info("Updating job: #{j.name}")
|
10
|
+
job_updater = JobUpdater.new(deployment_plan, j)
|
11
|
+
job_updater.update
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
class ParallelMultiJobUpdater
|
17
|
+
def run(base_job, deployment_plan, jobs)
|
18
|
+
base_job.logger.info("Updating jobs in parallel: #{jobs.map(&:name).join(', ')}")
|
19
|
+
base_job.task_checkpoint
|
20
|
+
|
21
|
+
ThreadPool.new(max_threads: jobs.size).wrap do |pool|
|
22
|
+
jobs.each do |j|
|
23
|
+
pool.process do
|
24
|
+
base_job.logger.info("Updating job: #{j.name}")
|
25
|
+
job_updater = JobUpdater.new(deployment_plan, j)
|
26
|
+
job_updater.update
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class BatchMultiJobUpdater
|
34
|
+
def run(base_job, deployment_plan, jobs)
|
35
|
+
serial_updater = SerialMultiJobUpdater.new
|
36
|
+
parallel_updater = ParallelMultiJobUpdater.new
|
37
|
+
|
38
|
+
partition_jobs_by_serial(jobs).each do |jp|
|
39
|
+
updater = jp.first.update.serial? ? serial_updater : parallel_updater
|
40
|
+
updater.run(base_job, deployment_plan, jp)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def partition_jobs_by_serial(jobs)
|
47
|
+
job_partitions = []
|
48
|
+
last_partition = []
|
49
|
+
|
50
|
+
jobs.each do |j|
|
51
|
+
lastj = last_partition.last
|
52
|
+
if !lastj || lastj.update.serial? == j.update.serial?
|
53
|
+
last_partition << j
|
54
|
+
else
|
55
|
+
job_partitions << last_partition
|
56
|
+
last_partition = [j]
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
job_partitions << last_partition if last_partition.any?
|
61
|
+
job_partitions
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -43,6 +43,12 @@ module Bosh::Director
|
|
43
43
|
parse_watch_times(update_watch_times)
|
44
44
|
end
|
45
45
|
|
46
|
+
@serial = safe_property(update_config, "serial", {
|
47
|
+
class: :boolean,
|
48
|
+
optional: true,
|
49
|
+
default: default_update_config ? default_update_config.serial? : true,
|
50
|
+
})
|
51
|
+
|
46
52
|
if optional
|
47
53
|
@canaries ||= default_update_config.canaries
|
48
54
|
|
@@ -75,6 +81,10 @@ module Bosh::Director
|
|
75
81
|
|
76
82
|
result
|
77
83
|
end
|
84
|
+
|
85
|
+
def serial?
|
86
|
+
!!@serial
|
87
|
+
end
|
78
88
|
end
|
79
89
|
end
|
80
|
-
end
|
90
|
+
end
|
@@ -1,55 +1,48 @@
|
|
1
1
|
module Bosh::Director
|
2
2
|
module DeploymentPlan
|
3
3
|
class Updater
|
4
|
-
def initialize(
|
5
|
-
@
|
6
|
-
@logger =
|
4
|
+
def initialize(base_job, event_log, resource_pools, assembler, deployment_plan, multi_job_updater)
|
5
|
+
@base_job = base_job
|
6
|
+
@logger = base_job.logger
|
7
7
|
@event_log = event_log
|
8
8
|
@resource_pools = resource_pools
|
9
9
|
@assembler = assembler
|
10
10
|
@deployment_plan = deployment_plan
|
11
|
+
@multi_job_updater = multi_job_updater
|
11
12
|
end
|
12
13
|
|
13
14
|
def update
|
14
|
-
event_log.begin_stage('Preparing DNS', 1)
|
15
|
-
|
15
|
+
@event_log.begin_stage('Preparing DNS', 1)
|
16
|
+
@base_job.track_and_log('Binding DNS') do
|
16
17
|
if Config.dns_enabled?
|
17
|
-
assembler.bind_dns
|
18
|
+
@assembler.bind_dns
|
18
19
|
end
|
19
20
|
end
|
20
21
|
|
21
|
-
logger.info('Updating resource pools')
|
22
|
-
resource_pools.update
|
23
|
-
|
22
|
+
@logger.info('Updating resource pools')
|
23
|
+
@resource_pools.update
|
24
|
+
@base_job.task_checkpoint
|
24
25
|
|
25
|
-
logger.info('Binding instance VMs')
|
26
|
-
assembler.bind_instance_vms
|
26
|
+
@logger.info('Binding instance VMs')
|
27
|
+
@assembler.bind_instance_vms
|
27
28
|
|
28
|
-
event_log.begin_stage('Preparing configuration', 1)
|
29
|
-
|
30
|
-
assembler.bind_configuration
|
29
|
+
@event_log.begin_stage('Preparing configuration', 1)
|
30
|
+
@base_job.track_and_log('Binding configuration') do
|
31
|
+
@assembler.bind_configuration
|
31
32
|
end
|
32
33
|
|
33
|
-
logger.info('Deleting no longer needed VMs')
|
34
|
-
assembler.delete_unneeded_vms
|
34
|
+
@logger.info('Deleting no longer needed VMs')
|
35
|
+
@assembler.delete_unneeded_vms
|
35
36
|
|
36
|
-
logger.info('Deleting no longer needed instances')
|
37
|
-
assembler.delete_unneeded_instances
|
37
|
+
@logger.info('Deleting no longer needed instances')
|
38
|
+
@assembler.delete_unneeded_instances
|
38
39
|
|
39
|
-
logger.info('Updating jobs')
|
40
|
-
deployment_plan.jobs
|
41
|
-
job.task_checkpoint
|
42
|
-
logger.info("Updating job: #{bosh_job.name}")
|
43
|
-
JobUpdater.new(deployment_plan, bosh_job).update
|
44
|
-
end
|
40
|
+
@logger.info('Updating jobs')
|
41
|
+
@multi_job_updater.run(@base_job, @deployment_plan, @deployment_plan.jobs)
|
45
42
|
|
46
|
-
logger.info('Refilling resource pools')
|
47
|
-
resource_pools.refill
|
43
|
+
@logger.info('Refilling resource pools')
|
44
|
+
@resource_pools.refill
|
48
45
|
end
|
49
|
-
|
50
|
-
private
|
51
|
-
|
52
|
-
attr_reader :job, :event_log, :resource_pools, :logger, :assembler, :deployment_plan
|
53
46
|
end
|
54
47
|
end
|
55
48
|
end
|
@@ -1,8 +1,5 @@
|
|
1
|
-
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
-
|
3
1
|
module Bosh::Director
|
4
|
-
|
5
|
-
|
2
|
+
module EventLog
|
6
3
|
# Event log conventions:
|
7
4
|
# All event log entries having same "stage" logically belong to the same
|
8
5
|
# event group. "tags" is an array of strings that supposed to act as hint
|
@@ -25,112 +22,127 @@ module Bosh::Director
|
|
25
22
|
# Job update (mysql_node):
|
26
23
|
# update |-------- | (2/4) 50%
|
27
24
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
@lock = Mutex.new
|
35
|
-
@counter = 0
|
36
|
-
end
|
25
|
+
class Log
|
26
|
+
def initialize(io = nil)
|
27
|
+
@logger = CustomLogger.new(io || StringIO.new)
|
28
|
+
@last_stage = Stage.new(self, 'unknown', [], 0)
|
29
|
+
@last_stage_lock = Mutex.new
|
30
|
+
end
|
37
31
|
|
38
|
-
|
39
|
-
|
40
|
-
@
|
41
|
-
@tags = tags
|
42
|
-
@counter = 0
|
43
|
-
@total = total
|
32
|
+
def begin_stage(stage_name, total = nil, tags = [])
|
33
|
+
stage = Stage.new(self, stage_name, tags, total)
|
34
|
+
@last_stage_lock.synchronize { @last_stage = stage }
|
44
35
|
end
|
45
|
-
end
|
46
36
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
@counter += 1
|
51
|
-
index = @counter
|
37
|
+
def track(task_name = nil, &blk)
|
38
|
+
last_stage = @last_stage_lock.synchronize { @last_stage }
|
39
|
+
last_stage.advance_and_track(task_name, &blk)
|
52
40
|
end
|
53
41
|
|
54
|
-
|
42
|
+
# Adds an error entry to the event log.
|
43
|
+
# @param [DirectorError] error Director error
|
44
|
+
# @return [void]
|
45
|
+
def log_error(error)
|
46
|
+
@logger.info(Yajl::Encoder.encode(
|
47
|
+
:time => Time.now.to_i,
|
48
|
+
:error => {
|
49
|
+
:code => error.error_code,
|
50
|
+
:message => error.message,
|
51
|
+
},
|
52
|
+
))
|
53
|
+
end
|
55
54
|
|
56
|
-
|
57
|
-
|
58
|
-
yield ticker if block_given?
|
59
|
-
rescue => e
|
60
|
-
task_failed(task, index, 100, e.to_s)
|
61
|
-
raise
|
55
|
+
def log_entry(entry)
|
56
|
+
@logger.info(Yajl::Encoder.encode(entry))
|
62
57
|
end
|
63
|
-
finish_task(task, index)
|
64
58
|
end
|
65
59
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
}
|
76
|
-
}
|
60
|
+
class Stage
|
61
|
+
def initialize(event_log, name, tags, total)
|
62
|
+
@event_log = event_log
|
63
|
+
@name = name
|
64
|
+
@tags = tags
|
65
|
+
@index = 0
|
66
|
+
@total = total
|
67
|
+
@index_lock = Mutex.new
|
68
|
+
end
|
77
69
|
|
78
|
-
|
79
|
-
|
70
|
+
def advance_and_track(task_name, &blk)
|
71
|
+
task = @index_lock.synchronize do
|
72
|
+
@index += 1
|
73
|
+
Task.new(self, task_name, @index)
|
74
|
+
end
|
75
|
+
|
76
|
+
task.start
|
77
|
+
begin
|
78
|
+
blk.call(task) if blk
|
79
|
+
rescue => e
|
80
|
+
task.failed(e.to_s)
|
81
|
+
raise
|
82
|
+
end
|
83
|
+
task.finish
|
84
|
+
end
|
80
85
|
|
81
|
-
|
82
|
-
|
86
|
+
def log_entry(entry)
|
87
|
+
@event_log.log_entry({
|
88
|
+
:time => Time.now.to_i,
|
89
|
+
:stage => @name,
|
90
|
+
:tags => @tags,
|
91
|
+
:total => @total,
|
92
|
+
}.merge(entry))
|
93
|
+
end
|
83
94
|
end
|
84
95
|
|
85
|
-
|
86
|
-
|
87
|
-
|
96
|
+
class Task
|
97
|
+
def initialize(stage, name, index)
|
98
|
+
@stage = stage
|
99
|
+
@name = name
|
100
|
+
@index = index
|
101
|
+
@state = 'in_progress'
|
102
|
+
@progress = 0
|
103
|
+
end
|
88
104
|
|
89
|
-
|
90
|
-
|
91
|
-
|
105
|
+
def advance(delta, data = {})
|
106
|
+
@state = 'in_progress'
|
107
|
+
@progress = [@progress + delta, 100].min
|
108
|
+
log_entry(data)
|
109
|
+
end
|
92
110
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
:stage => @stage,
|
97
|
-
:task => task,
|
98
|
-
:tags => @tags,
|
99
|
-
:index => index,
|
100
|
-
:total => @total,
|
101
|
-
:state => state,
|
102
|
-
:progress => progress,
|
103
|
-
}
|
104
|
-
|
105
|
-
if data.size > 0
|
106
|
-
entry[:data] = data
|
111
|
+
def start
|
112
|
+
@state = 'started'
|
113
|
+
log_entry
|
107
114
|
end
|
108
115
|
|
109
|
-
|
110
|
-
|
116
|
+
def finish
|
117
|
+
@state = 'finished'
|
118
|
+
@progress = 100
|
119
|
+
log_entry
|
120
|
+
end
|
111
121
|
|
112
|
-
|
122
|
+
def failed(error_msg = nil)
|
123
|
+
@state = 'failed'
|
124
|
+
@progress = 100
|
125
|
+
log_entry("error" => error_msg)
|
126
|
+
end
|
113
127
|
|
114
|
-
|
115
|
-
def format_message(level, time, progname, msg)
|
116
|
-
msg + "\n"
|
117
|
-
end
|
118
|
-
end
|
128
|
+
private
|
119
129
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
130
|
+
def log_entry(data = {})
|
131
|
+
task_entry = {
|
132
|
+
:task => @name,
|
133
|
+
:index => @index,
|
134
|
+
:state => @state,
|
135
|
+
:progress => @progress.to_i,
|
136
|
+
}
|
137
|
+
task_entry[:data] = data if data.size > 0
|
138
|
+
@stage.log_entry(task_entry)
|
139
|
+
end
|
129
140
|
end
|
130
141
|
|
131
|
-
|
132
|
-
|
133
|
-
|
142
|
+
class CustomLogger < ::Logger
|
143
|
+
def format_message(level, time, progname, msg)
|
144
|
+
msg + "\n"
|
145
|
+
end
|
134
146
|
end
|
135
147
|
end
|
136
148
|
end
|
@@ -7,7 +7,6 @@ module Bosh::Director
|
|
7
7
|
@deployment_plan = deployment_plan
|
8
8
|
@cloud = Config.cloud
|
9
9
|
@logger = Config.logger
|
10
|
-
@event_log = Config.event_log
|
11
10
|
@blobstore = App.instance.blobstores.blobstore
|
12
11
|
end
|
13
12
|
|
@@ -15,11 +14,11 @@ module Bosh::Director
|
|
15
14
|
# @param [Array<Models::Instance>] instances list of instances to delete
|
16
15
|
# @param [Hash] options optional list of options controlling concurrency
|
17
16
|
# @return [void]
|
18
|
-
def delete_instances(instances, options = {})
|
17
|
+
def delete_instances(instances, event_log_stage, options = {})
|
19
18
|
max_threads = options[:max_threads] || Config.max_threads
|
20
19
|
ThreadPool.new(:max_threads => max_threads).wrap do |pool|
|
21
20
|
instances.each do |instance|
|
22
|
-
pool.process { delete_instance(instance) }
|
21
|
+
pool.process { delete_instance(instance, event_log_stage) }
|
23
22
|
end
|
24
23
|
end
|
25
24
|
end
|
@@ -27,11 +26,11 @@ module Bosh::Director
|
|
27
26
|
# Deletes a single instance and attached persistent disks
|
28
27
|
# @param [Models::Instance] instance instance to delete
|
29
28
|
# @return [void]
|
30
|
-
def delete_instance(instance)
|
29
|
+
def delete_instance(instance, event_log_stage)
|
31
30
|
vm = instance.vm
|
32
|
-
@
|
33
|
-
@logger.info("Delete unneeded instance: #{vm.cid}")
|
31
|
+
@logger.info("Delete unneeded instance: #{vm.cid}")
|
34
32
|
|
33
|
+
event_log_stage.advance_and_track(vm.cid) do
|
35
34
|
drain(vm.agent_id)
|
36
35
|
@cloud.delete_vm(vm.cid)
|
37
36
|
delete_snapshots(instance)
|
@@ -105,6 +104,5 @@ module Bosh::Director
|
|
105
104
|
delete_dns_records(record_pattern, @deployment_plan.dns_domain.id)
|
106
105
|
end
|
107
106
|
end
|
108
|
-
|
109
107
|
end
|
110
108
|
end
|
@@ -12,10 +12,10 @@ module Bosh::Director
|
|
12
12
|
attr_reader :current_state
|
13
13
|
|
14
14
|
# @params [DeploymentPlan::Instance] instance
|
15
|
-
def initialize(instance,
|
15
|
+
def initialize(instance, event_log_task)
|
16
16
|
@cloud = Config.cloud
|
17
17
|
@logger = Config.logger
|
18
|
-
@
|
18
|
+
@event_log_task = event_log_task
|
19
19
|
@blobstore = App.instance.blobstores.blobstore
|
20
20
|
|
21
21
|
@instance = instance
|
@@ -42,7 +42,7 @@ module Bosh::Director
|
|
42
42
|
end
|
43
43
|
|
44
44
|
def report_progress
|
45
|
-
@
|
45
|
+
@event_log_task.advance(100.0 / update_steps())
|
46
46
|
end
|
47
47
|
|
48
48
|
def update_steps
|
@@ -58,7 +58,7 @@ module Bosh::Director
|
|
58
58
|
@debug_logger.level = Config.logger.level
|
59
59
|
@debug_logger.formatter = ThreadFormatter.new
|
60
60
|
|
61
|
-
Config.event_log = EventLog.new(event_log)
|
61
|
+
Config.event_log = EventLog::Log.new(event_log)
|
62
62
|
Config.result = TaskResultFile.new(result_log)
|
63
63
|
Config.logger = @debug_logger
|
64
64
|
|
@@ -10,20 +10,8 @@ module Bosh::Director
|
|
10
10
|
@event_log = Config.event_log
|
11
11
|
end
|
12
12
|
|
13
|
-
def delete_unneeded_instances
|
14
|
-
@logger.info("Deleting no longer needed instances")
|
15
|
-
unneeded_instances = @job.unneeded_instances
|
16
|
-
|
17
|
-
return if unneeded_instances.empty?
|
18
|
-
|
19
|
-
@event_log.begin_stage("Deleting unneeded instances", unneeded_instances.size, [@job.name])
|
20
|
-
deleter = InstanceDeleter.new(@deployment_plan)
|
21
|
-
deleter.delete_instances(unneeded_instances, max_threads: @job.update.max_in_flight)
|
22
|
-
|
23
|
-
@logger.info("Deleted no longer needed instances")
|
24
|
-
end
|
25
|
-
|
26
13
|
def update
|
14
|
+
@logger.info("Deleting no longer needed instances")
|
27
15
|
delete_unneeded_instances
|
28
16
|
|
29
17
|
instances = []
|
@@ -37,68 +25,27 @@ module Bosh::Director
|
|
37
25
|
end
|
38
26
|
|
39
27
|
@logger.info("Found #{instances.size} instances to update")
|
40
|
-
|
41
|
-
@event_log.begin_stage("Updating job", instances.size, [ @job.name ])
|
28
|
+
event_log_stage = @event_log.begin_stage("Updating job", instances.size, [ @job.name ])
|
42
29
|
|
43
30
|
ThreadPool.new(:max_threads => @job.update.max_in_flight).wrap do |pool|
|
44
31
|
num_canaries = [ @job.update.canaries, instances.size ].min
|
32
|
+
@logger.info("Starting canary update num_canaries=#{num_canaries}")
|
33
|
+
update_canaries(pool, instances, num_canaries, event_log_stage)
|
45
34
|
|
46
|
-
@logger.info("
|
47
|
-
# Canaries first
|
48
|
-
num_canaries.times do
|
49
|
-
instance = instances.shift
|
50
|
-
|
51
|
-
pool.process do
|
52
|
-
desc = "#{@job.name}/#{instance.index}"
|
53
|
-
@event_log.track("#{desc} (canary)") do |ticker|
|
54
|
-
with_thread_name("canary_update(#{desc})") do
|
55
|
-
unless @job.should_halt?
|
56
|
-
begin
|
57
|
-
InstanceUpdater.new(instance, ticker).
|
58
|
-
update(:canary => true)
|
59
|
-
rescue Exception => e
|
60
|
-
@logger.error("Error updating canary instance: #{e}\n" +
|
61
|
-
"#{e.backtrace.join("\n")}")
|
62
|
-
@job.record_update_error(e, :canary => true)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
35
|
+
@logger.info("Waiting for canaries to update")
|
70
36
|
pool.wait
|
71
|
-
@logger.info("Finished canary update")
|
72
37
|
|
38
|
+
@logger.info("Finished canary update")
|
73
39
|
if @job.should_halt?
|
74
40
|
@logger.warn("Halting deployment due to a canary failure")
|
75
41
|
halt
|
76
42
|
end
|
77
43
|
|
78
|
-
# Continue with the rest of the updates
|
79
44
|
@logger.info("Continuing the rest of the update")
|
80
|
-
instances
|
81
|
-
pool.process do
|
82
|
-
desc = "#{@job.name}/#{instance.index}"
|
83
|
-
@event_log.track(desc) do |ticker|
|
84
|
-
with_thread_name("instance_update(#{desc})") do
|
85
|
-
unless @job.should_halt?
|
86
|
-
begin
|
87
|
-
InstanceUpdater.new(instance, ticker).update
|
88
|
-
rescue Exception => e
|
89
|
-
@logger.error("Error updating instance: #{e}\n" +
|
90
|
-
"#{e.backtrace.join("\n")}")
|
91
|
-
@job.record_update_error(e)
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
45
|
+
update_instances(pool, instances, event_log_stage)
|
98
46
|
end
|
99
47
|
|
100
48
|
@logger.info("Finished the rest of the update")
|
101
|
-
|
102
49
|
if @job.should_halt?
|
103
50
|
@logger.warn("Halting deployment due to an update failure")
|
104
51
|
halt
|
@@ -106,10 +53,65 @@ module Bosh::Director
|
|
106
53
|
end
|
107
54
|
|
108
55
|
def halt
|
109
|
-
|
110
|
-
|
111
|
-
|
56
|
+
raise(@job.halt_exception || RuntimeError.new("Deployment has been halted"))
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def delete_unneeded_instances
|
62
|
+
unneeded_instances = @job.unneeded_instances
|
63
|
+
return if unneeded_instances.empty?
|
64
|
+
|
65
|
+
event_log_stage = @event_log.begin_stage("Deleting unneeded instances", unneeded_instances.size, [@job.name])
|
66
|
+
deleter = InstanceDeleter.new(@deployment_plan)
|
67
|
+
deleter.delete_instances(unneeded_instances, event_log_stage, max_threads: @job.update.max_in_flight)
|
68
|
+
|
69
|
+
@logger.info("Deleted no longer needed instances")
|
70
|
+
end
|
71
|
+
|
72
|
+
def update_canaries(pool, instances, num_canaries, event_log_stage)
|
73
|
+
num_canaries.times do
|
74
|
+
instance = instances.shift
|
75
|
+
pool.process { update_canary_instance(instance, event_log_stage) }
|
76
|
+
end
|
112
77
|
end
|
113
78
|
|
79
|
+
def update_canary_instance(instance, event_log_stage)
|
80
|
+
desc = "#{@job.name}/#{instance.index}"
|
81
|
+
event_log_stage.advance_and_track("#{desc} (canary)") do |ticker|
|
82
|
+
next if @job.should_halt?
|
83
|
+
|
84
|
+
with_thread_name("canary_update(#{desc})") do
|
85
|
+
begin
|
86
|
+
InstanceUpdater.new(instance, ticker).update(:canary => true)
|
87
|
+
rescue Exception => e
|
88
|
+
@logger.error("Error updating canary instance: #{e.inspect}\n#{e.backtrace.join("\n")}")
|
89
|
+
@job.record_update_error(e, :canary => true)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def update_instances(pool, instances, event_log_stage)
|
96
|
+
instances.each do |instance|
|
97
|
+
pool.process { update_instance(instance, event_log_stage) }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def update_instance(instance, event_log_stage)
|
102
|
+
desc = "#{@job.name}/#{instance.index}"
|
103
|
+
event_log_stage.advance_and_track(desc) do |ticker|
|
104
|
+
next if @job.should_halt?
|
105
|
+
|
106
|
+
with_thread_name("instance_update(#{desc})") do
|
107
|
+
begin
|
108
|
+
InstanceUpdater.new(instance, ticker).update
|
109
|
+
rescue Exception => e
|
110
|
+
@logger.error("Error updating instance: #{e.inspect}\n#{e.backtrace.join("\n")}")
|
111
|
+
@job.record_update_error(e)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
114
116
|
end
|
115
117
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
-
|
3
1
|
module Bosh::Director
|
4
2
|
module Jobs
|
5
3
|
class UpdateDeployment < BaseJob
|
@@ -52,8 +50,8 @@ module Bosh::Director
|
|
52
50
|
|
53
51
|
def update
|
54
52
|
resource_pools = DeploymentPlan::ResourcePools.new(event_log, @resource_pool_updaters)
|
55
|
-
|
56
|
-
updater = DeploymentPlan::Updater.new(self, event_log, resource_pools, @assembler, @deployment_plan)
|
53
|
+
multi_job_updater = DeploymentPlan::BatchMultiJobUpdater.new
|
54
|
+
updater = DeploymentPlan::Updater.new(self, event_log, resource_pools, @assembler, @deployment_plan, multi_job_updater)
|
57
55
|
updater.update
|
58
56
|
end
|
59
57
|
|
@@ -1,8 +1,5 @@
|
|
1
|
-
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
-
|
3
1
|
module Bosh::Director
|
4
2
|
module ValidationHelper
|
5
|
-
|
6
3
|
def safe_property(hash, property, options = {})
|
7
4
|
result = nil
|
8
5
|
|
@@ -48,8 +45,7 @@ module Bosh::Director
|
|
48
45
|
|
49
46
|
def invalid_type(property, klass, value)
|
50
47
|
raise ValidationInvalidType,
|
51
|
-
|
48
|
+
"Property `#{property}' (value #{value.inspect}) did not match the required type `#{klass}'"
|
52
49
|
end
|
53
|
-
|
54
50
|
end
|
55
51
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bosh-director
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.5.0.pre.
|
4
|
+
version: 1.5.0.pre.1633
|
5
5
|
prerelease: 6
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-12-
|
12
|
+
date: 2013-12-23 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bcrypt-ruby
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - ~>
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.5.0.pre.
|
37
|
+
version: 1.5.0.pre.1633
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,7 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - ~>
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.5.0.pre.
|
45
|
+
version: 1.5.0.pre.1633
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: bosh-core
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,7 +50,7 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - ~>
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 1.5.0.pre.
|
53
|
+
version: 1.5.0.pre.1633
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -58,7 +58,7 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - ~>
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.5.0.pre.
|
61
|
+
version: 1.5.0.pre.1633
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: bosh_common
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,7 +66,7 @@ dependencies:
|
|
66
66
|
requirements:
|
67
67
|
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: 1.5.0.pre.
|
69
|
+
version: 1.5.0.pre.1633
|
70
70
|
type: :runtime
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -74,7 +74,7 @@ dependencies:
|
|
74
74
|
requirements:
|
75
75
|
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version: 1.5.0.pre.
|
77
|
+
version: 1.5.0.pre.1633
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
79
|
name: bosh_cpi
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,7 +82,7 @@ dependencies:
|
|
82
82
|
requirements:
|
83
83
|
- - ~>
|
84
84
|
- !ruby/object:Gem::Version
|
85
|
-
version: 1.5.0.pre.
|
85
|
+
version: 1.5.0.pre.1633
|
86
86
|
type: :runtime
|
87
87
|
prerelease: false
|
88
88
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -90,7 +90,7 @@ dependencies:
|
|
90
90
|
requirements:
|
91
91
|
- - ~>
|
92
92
|
- !ruby/object:Gem::Version
|
93
|
-
version: 1.5.0.pre.
|
93
|
+
version: 1.5.0.pre.1633
|
94
94
|
- !ruby/object:Gem::Dependency
|
95
95
|
name: bosh_openstack_cpi
|
96
96
|
requirement: !ruby/object:Gem::Requirement
|
@@ -98,7 +98,7 @@ dependencies:
|
|
98
98
|
requirements:
|
99
99
|
- - ~>
|
100
100
|
- !ruby/object:Gem::Version
|
101
|
-
version: 1.5.0.pre.
|
101
|
+
version: 1.5.0.pre.1633
|
102
102
|
type: :runtime
|
103
103
|
prerelease: false
|
104
104
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -106,7 +106,7 @@ dependencies:
|
|
106
106
|
requirements:
|
107
107
|
- - ~>
|
108
108
|
- !ruby/object:Gem::Version
|
109
|
-
version: 1.5.0.pre.
|
109
|
+
version: 1.5.0.pre.1633
|
110
110
|
- !ruby/object:Gem::Dependency
|
111
111
|
name: bosh_aws_cpi
|
112
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -114,7 +114,7 @@ dependencies:
|
|
114
114
|
requirements:
|
115
115
|
- - ~>
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 1.5.0.pre.
|
117
|
+
version: 1.5.0.pre.1633
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -122,7 +122,7 @@ dependencies:
|
|
122
122
|
requirements:
|
123
123
|
- - ~>
|
124
124
|
- !ruby/object:Gem::Version
|
125
|
-
version: 1.5.0.pre.
|
125
|
+
version: 1.5.0.pre.1633
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: bosh_vsphere_cpi
|
128
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -130,7 +130,7 @@ dependencies:
|
|
130
130
|
requirements:
|
131
131
|
- - ~>
|
132
132
|
- !ruby/object:Gem::Version
|
133
|
-
version: 1.5.0.pre.
|
133
|
+
version: 1.5.0.pre.1633
|
134
134
|
type: :runtime
|
135
135
|
prerelease: false
|
136
136
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -138,7 +138,7 @@ dependencies:
|
|
138
138
|
requirements:
|
139
139
|
- - ~>
|
140
140
|
- !ruby/object:Gem::Version
|
141
|
-
version: 1.5.0.pre.
|
141
|
+
version: 1.5.0.pre.1633
|
142
142
|
- !ruby/object:Gem::Dependency
|
143
143
|
name: eventmachine
|
144
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -461,7 +461,7 @@ dependencies:
|
|
461
461
|
version: '0'
|
462
462
|
description: ! 'BOSH Director
|
463
463
|
|
464
|
-
|
464
|
+
72d86c'
|
465
465
|
email: support@cloudfoundry.com
|
466
466
|
executables:
|
467
467
|
- bosh-director
|
@@ -570,6 +570,7 @@ files:
|
|
570
570
|
- lib/bosh/director/deployment_plan/instance.rb
|
571
571
|
- lib/bosh/director/deployment_plan/job.rb
|
572
572
|
- lib/bosh/director/deployment_plan/manual_network.rb
|
573
|
+
- lib/bosh/director/deployment_plan/multi_job_updater.rb
|
573
574
|
- lib/bosh/director/deployment_plan/network.rb
|
574
575
|
- lib/bosh/director/deployment_plan/network_subnet.rb
|
575
576
|
- lib/bosh/director/deployment_plan/planner.rb
|