bosh-director 1.2513.0 → 1.2524.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/bosh-director-worker +3 -0
- data/lib/bosh/director.rb +3 -0
- data/lib/bosh/director/errand.rb +1 -0
- data/lib/bosh/director/errand/result.rb +55 -0
- data/lib/bosh/director/errand/runner.rb +24 -53
- data/lib/bosh/director/job_runner.rb +30 -31
- data/lib/bosh/director/jobs/fetch_logs.rb +5 -65
- data/lib/bosh/director/jobs/run_errand.rb +42 -45
- data/lib/bosh/director/log_bundles_cleaner.rb +34 -0
- data/lib/bosh/director/logs_fetcher.rb +40 -0
- data/lib/bosh/director/version.rb +1 -1
- metadata +41 -22
data/bin/bosh-director-worker
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'erb'
|
4
4
|
require 'bosh/director'
|
5
|
+
require 'resque/failure/backtrace'
|
5
6
|
|
6
7
|
config_file = nil
|
7
8
|
index = nil
|
@@ -71,6 +72,8 @@ Resque.after_fork do
|
|
71
72
|
Thread.new { EM.run }
|
72
73
|
end
|
73
74
|
|
75
|
+
Resque::Failure.backend = Resque::Failure::Backtrace
|
76
|
+
|
74
77
|
worker.log "Starting worker #{worker}"
|
75
78
|
|
76
79
|
worker.work(ENV['INTERVAL'] || 5) # interval, will block
|
data/lib/bosh/director.rb
CHANGED
@@ -86,6 +86,9 @@ require 'bosh/director/resource_pool_updater'
|
|
86
86
|
require 'bosh/director/sequel'
|
87
87
|
require 'common/thread_pool'
|
88
88
|
|
89
|
+
require 'bosh/director/log_bundles_cleaner'
|
90
|
+
require 'bosh/director/logs_fetcher'
|
91
|
+
|
89
92
|
require 'bosh/director/cloudcheck_helper'
|
90
93
|
require 'bosh/director/problem_handlers/base'
|
91
94
|
require 'bosh/director/problem_handlers/invalid_problem'
|
data/lib/bosh/director/errand.rb
CHANGED
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'membrane'
|
2
|
+
|
3
|
+
module Bosh::Director
|
4
|
+
class Errand::Result
|
5
|
+
attr_reader :exit_code
|
6
|
+
|
7
|
+
AGENT_RUN_ERRAND_RESULT_SCHEMA = ::Membrane::SchemaParser.parse do
|
8
|
+
{
|
9
|
+
'exit_code' => Integer,
|
10
|
+
'stdout' => String,
|
11
|
+
'stderr' => String,
|
12
|
+
}
|
13
|
+
end
|
14
|
+
|
15
|
+
# Explicitly write out schema of the director task result
|
16
|
+
# to avoid accidently leaking agent task result extra fields.
|
17
|
+
def self.from_agent_task_results(agent_task_result, logs_blobstore_id)
|
18
|
+
AGENT_RUN_ERRAND_RESULT_SCHEMA.validate(agent_task_result)
|
19
|
+
new(*agent_task_result.values_at('exit_code', 'stdout', 'stderr'), logs_blobstore_id)
|
20
|
+
rescue Membrane::SchemaValidationError => e
|
21
|
+
raise AgentInvalidTaskResult, e.message
|
22
|
+
end
|
23
|
+
|
24
|
+
def initialize(exit_code, stdout, stderr, logs_blobstore_id)
|
25
|
+
@exit_code = exit_code
|
26
|
+
@stdout = stdout
|
27
|
+
@stderr = stderr
|
28
|
+
@logs_blobstore_id = logs_blobstore_id
|
29
|
+
end
|
30
|
+
|
31
|
+
def short_description(job_name)
|
32
|
+
title_prefix = "Errand `#{job_name}'"
|
33
|
+
exit_code_suffix = "(exit code #{@exit_code})"
|
34
|
+
|
35
|
+
if @exit_code == 0
|
36
|
+
"#{title_prefix} completed successfully #{exit_code_suffix}"
|
37
|
+
elsif @exit_code > 128
|
38
|
+
"#{title_prefix} was canceled #{exit_code_suffix}"
|
39
|
+
else
|
40
|
+
"#{title_prefix} completed with error #{exit_code_suffix}"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_hash
|
45
|
+
{
|
46
|
+
'exit_code' => @exit_code,
|
47
|
+
'stdout' => @stdout,
|
48
|
+
'stderr' => @stderr,
|
49
|
+
'logs' => {
|
50
|
+
'blobstore_id' => @logs_blobstore_id,
|
51
|
+
},
|
52
|
+
}
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -1,51 +1,17 @@
|
|
1
|
-
require 'membrane'
|
2
|
-
|
3
1
|
module Bosh::Director
|
4
2
|
class Errand::Runner
|
5
|
-
class ErrandResult
|
6
|
-
attr_reader :exit_code
|
7
|
-
|
8
|
-
AGENT_TASK_RESULT_SCHEMA = ::Membrane::SchemaParser.parse do
|
9
|
-
{
|
10
|
-
'exit_code' => Integer,
|
11
|
-
'stdout' => String,
|
12
|
-
'stderr' => String,
|
13
|
-
}
|
14
|
-
end
|
15
|
-
|
16
|
-
# Explicitly write out schema of the director task result
|
17
|
-
# to avoid accidently leaking agent task result extra fields.
|
18
|
-
def self.from_agent_task_result(agent_task_result)
|
19
|
-
AGENT_TASK_RESULT_SCHEMA.validate(agent_task_result)
|
20
|
-
new(*agent_task_result.values_at('exit_code', 'stdout', 'stderr'))
|
21
|
-
rescue Membrane::SchemaValidationError => e
|
22
|
-
raise AgentInvalidTaskResult, e.message
|
23
|
-
end
|
24
|
-
|
25
|
-
def initialize(exit_code, stdout, stderr)
|
26
|
-
@exit_code = exit_code
|
27
|
-
@stdout = stdout
|
28
|
-
@stderr = stderr
|
29
|
-
end
|
30
|
-
|
31
|
-
def to_hash
|
32
|
-
{
|
33
|
-
'exit_code' => @exit_code,
|
34
|
-
'stdout' => @stdout,
|
35
|
-
'stderr' => @stderr,
|
36
|
-
}
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
3
|
# @param [Bosh::Director::DeploymentPlan::Job] job
|
41
4
|
# @param [Bosh::Director::TaskResultFile] result_file
|
5
|
+
# @param [Bosh::Director::Api::InstanceManager] instance_manager
|
42
6
|
# @param [Bosh::Director::EventLog::Log] event_log
|
43
|
-
|
7
|
+
# @param [Bosh::Director::LogsFetcher] logs_fetcher
|
8
|
+
def initialize(job, result_file, instance_manager, event_log, logs_fetcher)
|
44
9
|
@job = job
|
45
10
|
@result_file = result_file
|
46
11
|
@instance_manager = instance_manager
|
47
12
|
@event_log = event_log
|
48
13
|
@agent_task_id = nil
|
14
|
+
@logs_fetcher = logs_fetcher
|
49
15
|
end
|
50
16
|
|
51
17
|
# Runs errand on job instances
|
@@ -56,8 +22,8 @@ module Bosh::Director
|
|
56
22
|
end
|
57
23
|
|
58
24
|
agent_task_result = nil
|
59
|
-
|
60
25
|
event_log_stage = @event_log.begin_stage('Running errand', 1)
|
26
|
+
|
61
27
|
begin
|
62
28
|
event_log_stage.advance_and_track("#{@job.name}/#{instance.index}") do
|
63
29
|
start_errand_result = agent.start_errand
|
@@ -65,25 +31,30 @@ module Bosh::Director
|
|
65
31
|
agent_task_result = agent.wait_for_task(agent_task_id, &blk)
|
66
32
|
end
|
67
33
|
rescue TaskCancelled => e
|
34
|
+
# Existing run_errand long running task will return a result
|
35
|
+
# after agent cancels the task
|
68
36
|
agent_task_result = agent.wait_for_task(agent_task_id)
|
69
|
-
|
70
|
-
ensure
|
71
|
-
if agent_task_result
|
72
|
-
errand_result = ErrandResult.from_agent_task_result(agent_task_result)
|
73
|
-
@result_file.write(JSON.dump(errand_result.to_hash) + "\n")
|
74
|
-
end
|
37
|
+
@cancel_error = e
|
75
38
|
end
|
76
39
|
|
77
|
-
|
78
|
-
|
40
|
+
begin
|
41
|
+
logs_blobstore_id = @logs_fetcher.fetch(instance.model, 'job', nil)
|
42
|
+
rescue DirectorError => e
|
43
|
+
@fetch_logs_error = e
|
44
|
+
end
|
79
45
|
|
80
|
-
if
|
81
|
-
|
82
|
-
|
83
|
-
"#{title_prefix} was canceled #{exit_code_suffix}"
|
84
|
-
else
|
85
|
-
"#{title_prefix} completed with error #{exit_code_suffix}"
|
46
|
+
if agent_task_result
|
47
|
+
errand_result = Errand::Result.from_agent_task_results(agent_task_result, logs_blobstore_id)
|
48
|
+
@result_file.write(JSON.dump(errand_result.to_hash) + "\n")
|
86
49
|
end
|
50
|
+
|
51
|
+
# Prefer to raise cancel error because
|
52
|
+
# it was triggered before trying to fetch logs
|
53
|
+
raise @cancel_error if @cancel_error
|
54
|
+
|
55
|
+
raise @fetch_logs_error if @fetch_logs_error
|
56
|
+
|
57
|
+
errand_result.short_description(@job.name)
|
87
58
|
end
|
88
59
|
|
89
60
|
def cancel
|
@@ -11,25 +11,27 @@ module Bosh::Director
|
|
11
11
|
raise DirectorError, "Invalid director job class `#{job_class}'"
|
12
12
|
end
|
13
13
|
|
14
|
+
@task_id = task_id
|
15
|
+
setup_task_logging
|
16
|
+
|
14
17
|
task_manager = Bosh::Director::Api::TaskManager.new
|
15
18
|
|
16
19
|
@job_class = job_class
|
17
|
-
@task
|
18
|
-
|
19
|
-
setup_logging
|
20
|
+
@task_logger.info("Looking for task with task id #{@task_id}")
|
21
|
+
@task = task_manager.find_task(@task_id)
|
20
22
|
end
|
21
23
|
|
22
24
|
# Runs director job
|
23
25
|
def run(*args)
|
24
26
|
Config.current_job = nil
|
25
27
|
|
26
|
-
@
|
28
|
+
@task_logger.info("Starting task: #{@task_id}")
|
27
29
|
started_at = Time.now
|
28
30
|
|
29
|
-
with_thread_name("task:#{@
|
31
|
+
with_thread_name("task:#{@task_id}") { perform_job(*args) }
|
30
32
|
|
31
33
|
duration = Duration.duration(Time.now - started_at)
|
32
|
-
@
|
34
|
+
@task_logger.info("Task took #{duration} to process.")
|
33
35
|
end
|
34
36
|
|
35
37
|
# Task checkpoint: updates timestamp so running task isn't marked as
|
@@ -43,34 +45,31 @@ module Bosh::Director
|
|
43
45
|
|
44
46
|
# Sets up job logging.
|
45
47
|
# @return [void]
|
46
|
-
def
|
47
|
-
|
48
|
-
|
49
|
-
raise DirectorError,
|
50
|
-
"Task directory `#{@task.output}' is missing"
|
51
|
-
end
|
48
|
+
def setup_task_logging
|
49
|
+
log_dir = File.join(Config.base_dir, 'tasks', @task_id.to_s)
|
50
|
+
FileUtils.mkdir_p(log_dir)
|
52
51
|
|
53
|
-
debug_log = File.join(
|
54
|
-
event_log = File.join(
|
55
|
-
result_log = File.join(
|
52
|
+
debug_log = File.join(log_dir, 'debug')
|
53
|
+
event_log = File.join(log_dir, 'event')
|
54
|
+
result_log = File.join(log_dir, 'result')
|
56
55
|
|
57
|
-
@
|
58
|
-
@
|
59
|
-
@
|
56
|
+
@task_logger = Logger.new(debug_log)
|
57
|
+
@task_logger.level = Config.logger.level
|
58
|
+
@task_logger.formatter = ThreadFormatter.new
|
60
59
|
|
61
60
|
Config.event_log = EventLog::Log.new(event_log)
|
62
61
|
Config.result = TaskResultFile.new(result_log)
|
63
|
-
Config.logger = @
|
62
|
+
Config.logger = @task_logger
|
64
63
|
|
65
|
-
Config.db.logger = @
|
64
|
+
Config.db.logger = @task_logger
|
66
65
|
|
67
66
|
if Config.dns_enabled?
|
68
|
-
Config.dns_db.logger = @
|
67
|
+
Config.dns_db.logger = @task_logger
|
69
68
|
end
|
70
69
|
|
71
|
-
cpi_log = File.join(
|
72
|
-
Config.cloud_options[
|
73
|
-
Config.cloud_options[
|
70
|
+
cpi_log = File.join(log_dir, 'cpi')
|
71
|
+
Config.cloud_options['properties'] ||= {}
|
72
|
+
Config.cloud_options['properties']['cpi_log'] = cpi_log
|
74
73
|
end
|
75
74
|
|
76
75
|
# Instantiates and performs director job.
|
@@ -78,17 +77,17 @@ module Bosh::Director
|
|
78
77
|
# instantiate the new job object.
|
79
78
|
# @return [void]
|
80
79
|
def perform_job(*args)
|
81
|
-
@
|
80
|
+
@task_logger.info('Creating job')
|
82
81
|
|
83
82
|
job = @job_class.new(*args)
|
84
83
|
Config.current_job = job
|
85
84
|
|
86
|
-
job.task_id = @
|
85
|
+
job.task_id = @task_id
|
87
86
|
job.task_checkpoint # cancelled in the queue?
|
88
87
|
|
89
88
|
run_checkpointing
|
90
89
|
|
91
|
-
@
|
90
|
+
@task_logger.info("Performing task: #{@task.id}")
|
92
91
|
|
93
92
|
@task.state = :processing
|
94
93
|
@task.timestamp = Time.now
|
@@ -97,16 +96,16 @@ module Bosh::Director
|
|
97
96
|
|
98
97
|
result = job.perform
|
99
98
|
|
100
|
-
@
|
99
|
+
@task_logger.info('Done')
|
101
100
|
finish_task(:done, result)
|
102
101
|
|
103
102
|
rescue Bosh::Director::TaskCancelled => e
|
104
103
|
log_exception(e)
|
105
|
-
@
|
106
|
-
finish_task(:cancelled,
|
104
|
+
@task_logger.info("Task #{@task.id} cancelled")
|
105
|
+
finish_task(:cancelled, 'task cancelled')
|
107
106
|
rescue Exception => e
|
108
107
|
log_exception(e)
|
109
|
-
@
|
108
|
+
@task_logger.error("#{e}\n#{e.backtrace.join("\n")}")
|
110
109
|
finish_task(:error, e)
|
111
110
|
end
|
112
111
|
|
@@ -1,27 +1,23 @@
|
|
1
|
-
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
-
|
3
1
|
module Bosh::Director
|
4
2
|
module Jobs
|
5
3
|
class FetchLogs < BaseJob
|
6
4
|
include LockHelper
|
7
5
|
|
8
|
-
DEFAULT_BUNDLE_LIFETIME = 86400 * 10 # 10 days
|
9
|
-
|
10
6
|
@queue = :normal
|
11
7
|
|
12
|
-
attr_accessor :bundle_lifetime
|
13
|
-
|
14
8
|
def self.job_type
|
15
9
|
:fetch_logs
|
16
10
|
end
|
17
11
|
|
18
12
|
def initialize(instance_id, options = {})
|
19
|
-
@blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
|
20
13
|
@instance_id = instance_id
|
21
14
|
@log_type = options["type"] || "job"
|
22
15
|
@filters = options["filters"]
|
23
16
|
@instance_manager = Api::InstanceManager.new
|
24
|
-
|
17
|
+
|
18
|
+
blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
|
19
|
+
log_bundles_cleaner = LogBundlesCleaner.new(blobstore, 60 * 60 * 24 * 10, logger) # 10 days
|
20
|
+
@logs_fetcher = LogsFetcher.new(event_log, @instance_manager, log_bundles_cleaner, logger)
|
25
21
|
end
|
26
22
|
|
27
23
|
def perform
|
@@ -29,71 +25,15 @@ module Bosh::Director
|
|
29
25
|
desc = "#{instance.job}/#{instance.index}"
|
30
26
|
|
31
27
|
deployment = instance.deployment
|
32
|
-
|
33
28
|
if deployment.nil?
|
34
29
|
raise InstanceDeploymentMissing,
|
35
30
|
"`#{desc}' doesn't belong to any deployment"
|
36
31
|
end
|
37
32
|
|
38
33
|
with_deployment_lock(deployment) do
|
39
|
-
|
40
|
-
|
41
|
-
event_log.begin_stage("Fetching logs for #{desc}", 1)
|
42
|
-
|
43
|
-
agent = @instance_manager.agent_client_for(instance)
|
44
|
-
blobstore_id = nil
|
45
|
-
|
46
|
-
track_and_log("Finding and packing log files") do
|
47
|
-
logger.info("Fetching logs from agent: " +
|
48
|
-
"log_type='#{@log_type}', filters='#{@filters}'")
|
49
|
-
|
50
|
-
task = agent.fetch_logs(@log_type, @filters)
|
51
|
-
blobstore_id = task["blobstore_id"]
|
52
|
-
end
|
53
|
-
|
54
|
-
if blobstore_id.nil?
|
55
|
-
raise AgentTaskNoBlobstoreId,
|
56
|
-
"Agent didn't return a blobstore object id for packaged logs"
|
57
|
-
end
|
58
|
-
|
59
|
-
Models::LogBundle.create(:blobstore_id => blobstore_id,
|
60
|
-
:timestamp => Time.now)
|
61
|
-
|
62
|
-
# The returned value of this method is used as task result
|
63
|
-
# and gets extracted by CLI as a tarball blobstore id
|
64
|
-
blobstore_id
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
def cleanup_old_bundles
|
69
|
-
old_bundles = Models::LogBundle.filter("timestamp <= ?",
|
70
|
-
Time.now - bundle_lifetime)
|
71
|
-
count = old_bundles.count
|
72
|
-
|
73
|
-
if count == 0
|
74
|
-
logger.info("No old bundles to delete")
|
75
|
-
return
|
76
|
-
end
|
77
|
-
|
78
|
-
logger.info("Deleting #{count} old log bundle#{count > 1 ? "s" : ""}")
|
79
|
-
|
80
|
-
old_bundles.each do |bundle|
|
81
|
-
begin
|
82
|
-
logger.info("Deleting log bundle #{bundle.id}: " +
|
83
|
-
"#{bundle.blobstore_id}")
|
84
|
-
@blobstore.delete(bundle.blobstore_id)
|
85
|
-
bundle.delete
|
86
|
-
rescue Bosh::Blobstore::BlobstoreError => e
|
87
|
-
logger.warn("Could not delete #{bundle.blobstore_id}: #{e}")
|
88
|
-
# Assuming object has been deleted from blobstore by someone else,
|
89
|
-
# cleaning up DB record accordingly
|
90
|
-
if e.kind_of?(Bosh::Blobstore::NotFound)
|
91
|
-
bundle.delete
|
92
|
-
end
|
93
|
-
end
|
34
|
+
@logs_fetcher.fetch(instance, @log_type, @filter)
|
94
35
|
end
|
95
36
|
end
|
96
|
-
|
97
37
|
end
|
98
38
|
end
|
99
39
|
end
|
@@ -16,11 +16,13 @@ module Bosh::Director
|
|
16
16
|
@deployment_manager = Api::DeploymentManager.new
|
17
17
|
@instance_manager = Api::InstanceManager.new
|
18
18
|
@blobstore = App.instance.blobstores.blobstore
|
19
|
+
|
20
|
+
log_bundles_cleaner = LogBundlesCleaner.new(@blobstore, 60 * 60 * 24 * 10, logger) # 10 days
|
21
|
+
@logs_fetcher = LogsFetcher.new(event_log, @instance_manager, log_bundles_cleaner, logger)
|
19
22
|
end
|
20
23
|
|
21
24
|
def perform
|
22
25
|
deployment_model = @deployment_manager.find_by_name(@deployment_name)
|
23
|
-
|
24
26
|
manifest = Psych.load(deployment_model.manifest)
|
25
27
|
deployment = DeploymentPlan::Planner.parse(manifest, event_log, {})
|
26
28
|
|
@@ -39,18 +41,21 @@ module Bosh::Director
|
|
39
41
|
raise InstanceNotFound, "Instance `#{@deployment_name}/#{@errand_name}/0' doesn't exist"
|
40
42
|
end
|
41
43
|
|
42
|
-
runner = Errand::Runner.new(job, result_file, @instance_manager, event_log)
|
44
|
+
runner = Errand::Runner.new(job, result_file, @instance_manager, event_log, @logs_fetcher)
|
43
45
|
|
44
|
-
|
45
|
-
|
46
|
+
cancel_blk = lambda {
|
47
|
+
begin
|
48
|
+
task_checkpoint
|
49
|
+
rescue TaskCancelled => e
|
50
|
+
runner.cancel
|
51
|
+
raise e
|
52
|
+
end
|
53
|
+
}
|
46
54
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
runner.cancel
|
52
|
-
raise e
|
53
|
-
end
|
55
|
+
with_deployment_lock(deployment) do
|
56
|
+
with_updated_instances(deployment, job) do
|
57
|
+
logger.info('Starting to run errand')
|
58
|
+
runner.run(&cancel_blk)
|
54
59
|
end
|
55
60
|
end
|
56
61
|
end
|
@@ -62,51 +67,43 @@ module Bosh::Director
|
|
62
67
|
private
|
63
68
|
|
64
69
|
def with_updated_instances(deployment, job, &blk)
|
65
|
-
|
66
|
-
begin
|
67
|
-
logger.info('Starting to prepare for deployment')
|
68
|
-
prepare_deployment(deployment, job)
|
69
|
-
|
70
|
-
logger.info('Starting to update resource pool')
|
71
|
-
rp_manager(job).update
|
70
|
+
deployment_preparer = Errand::DeploymentPreparer.new(deployment, job, event_log, self)
|
72
71
|
|
73
|
-
|
74
|
-
|
72
|
+
rp_updaters = [ResourcePoolUpdater.new(job.resource_pool)]
|
73
|
+
rp_manager = DeploymentPlan::ResourcePools.new(event_log, rp_updaters)
|
75
74
|
|
76
|
-
|
77
|
-
ensure
|
78
|
-
ignore_cancellation do
|
79
|
-
logger.info('Starting to delete job instances')
|
80
|
-
job_manager(deployment, job).delete_instances
|
75
|
+
job_manager = Errand::JobManager.new(deployment, job, @blobstore, event_log)
|
81
76
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
77
|
+
begin
|
78
|
+
update_instances(deployment_preparer, rp_manager, job_manager)
|
79
|
+
blk.call
|
80
|
+
ensure
|
81
|
+
delete_instances(rp_manager, job_manager)
|
86
82
|
end
|
87
83
|
end
|
88
84
|
|
89
|
-
def
|
90
|
-
|
91
|
-
yield
|
92
|
-
@ignore_cancellation = false
|
93
|
-
end
|
94
|
-
|
95
|
-
def prepare_deployment(deployment, job)
|
96
|
-
deployment_preparer = Errand::DeploymentPreparer.new(
|
97
|
-
deployment, job, event_log, self)
|
98
|
-
|
85
|
+
def update_instances(deployment_preparer, rp_manager, job_manager)
|
86
|
+
logger.info('Starting to prepare for deployment')
|
99
87
|
deployment_preparer.prepare_deployment
|
100
88
|
deployment_preparer.prepare_job
|
101
|
-
end
|
102
89
|
|
103
|
-
|
104
|
-
|
105
|
-
|
90
|
+
logger.info('Starting to update resource pool')
|
91
|
+
rp_manager.update
|
92
|
+
|
93
|
+
logger.info('Starting to update job instances')
|
94
|
+
job_manager.update_instances
|
106
95
|
end
|
107
96
|
|
108
|
-
def
|
109
|
-
|
97
|
+
def delete_instances(rp_manager, job_manager)
|
98
|
+
@ignore_cancellation = true
|
99
|
+
|
100
|
+
logger.info('Starting to delete job instances')
|
101
|
+
job_manager.delete_instances
|
102
|
+
|
103
|
+
logger.info('Starting to refill resource pool')
|
104
|
+
rp_manager.refill
|
105
|
+
|
106
|
+
@ignore_cancellation = false
|
110
107
|
end
|
111
108
|
end
|
112
109
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Bosh::Director
|
2
|
+
class LogBundlesCleaner
|
3
|
+
def initialize(blobstore, log_bundle_ttl, logger)
|
4
|
+
@blobstore = blobstore
|
5
|
+
@bundle_lifetime = log_bundle_ttl
|
6
|
+
@logger = logger
|
7
|
+
end
|
8
|
+
|
9
|
+
def register_blobstore_id(blobstore_id)
|
10
|
+
@logger.info("Registering log bundle with blobstore id #{blobstore_id}")
|
11
|
+
Models::LogBundle.create(blobstore_id: blobstore_id, timestamp: Time.now)
|
12
|
+
end
|
13
|
+
|
14
|
+
def clean
|
15
|
+
cut_off_time = Time.now - @bundle_lifetime
|
16
|
+
old_bundles = Models::LogBundle.filter("timestamp <= ?", cut_off_time)
|
17
|
+
@logger.info("Deleting #{old_bundles.count} old log bundle(s) before #{cut_off_time}")
|
18
|
+
|
19
|
+
old_bundles.each do |bundle|
|
20
|
+
begin
|
21
|
+
@logger.info("Deleting log bundle #{bundle.id} with blobstore id #{bundle.blobstore_id}")
|
22
|
+
@blobstore.delete(bundle.blobstore_id)
|
23
|
+
bundle.delete
|
24
|
+
rescue Bosh::Blobstore::BlobstoreError => e
|
25
|
+
@logger.warn("Could not delete #{bundle.blobstore_id}: #{e.inspect}")
|
26
|
+
|
27
|
+
# Assuming object has been deleted from blobstore by someone else,
|
28
|
+
# cleaning up DB record accordingly
|
29
|
+
bundle.delete if e.kind_of?(Bosh::Blobstore::NotFound)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Bosh::Director
|
2
|
+
class LogsFetcher
|
3
|
+
# @param [Bosh::Director::EventLog::Log] event_log
|
4
|
+
# @param [Bosh::Director::Api::InstanceManager] instance_manager
|
5
|
+
# @param [Bosh::Director::LogBundlesCleaner] log_bundles_cleaner
|
6
|
+
def initialize(event_log, instance_manager, log_bundles_cleaner, logger)
|
7
|
+
@event_log = event_log
|
8
|
+
@instance_manager = instance_manager
|
9
|
+
@log_bundles_cleaner = log_bundles_cleaner
|
10
|
+
@logger = logger
|
11
|
+
end
|
12
|
+
|
13
|
+
# @param [Models::Instance] instance
|
14
|
+
# @param [String] log_type
|
15
|
+
# @param [Array] filters
|
16
|
+
def fetch(instance, log_type, filters)
|
17
|
+
@logger.info("Fetching logs from agent with log_type=#{log_type} filters=#{filters}")
|
18
|
+
|
19
|
+
@log_bundles_cleaner.clean
|
20
|
+
|
21
|
+
agent = @instance_manager.agent_client_for(instance)
|
22
|
+
blobstore_id = nil
|
23
|
+
|
24
|
+
stage = @event_log.begin_stage("Fetching logs for #{instance.job}/#{instance.index}", 1)
|
25
|
+
stage.advance_and_track('Finding and packing log files') do
|
26
|
+
fetch_logs_result = agent.fetch_logs(log_type, filters)
|
27
|
+
blobstore_id = fetch_logs_result['blobstore_id']
|
28
|
+
end
|
29
|
+
|
30
|
+
if blobstore_id.nil?
|
31
|
+
raise AgentTaskNoBlobstoreId,
|
32
|
+
"Agent didn't return a blobstore object id for packaged logs"
|
33
|
+
end
|
34
|
+
|
35
|
+
@log_bundles_cleaner.register_blobstore_id(blobstore_id)
|
36
|
+
|
37
|
+
blobstore_id
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bosh-director
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2524.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-
|
12
|
+
date: 2014-05-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bcrypt-ruby
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - ~>
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.
|
37
|
+
version: 1.2524.0
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,7 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - ~>
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.
|
45
|
+
version: 1.2524.0
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: bosh-core
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,7 +50,7 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - ~>
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 1.
|
53
|
+
version: 1.2524.0
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -58,7 +58,7 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - ~>
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.
|
61
|
+
version: 1.2524.0
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: bosh-director-core
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,7 +66,7 @@ dependencies:
|
|
66
66
|
requirements:
|
67
67
|
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: 1.
|
69
|
+
version: 1.2524.0
|
70
70
|
type: :runtime
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -74,7 +74,7 @@ dependencies:
|
|
74
74
|
requirements:
|
75
75
|
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version: 1.
|
77
|
+
version: 1.2524.0
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
79
|
name: bosh_common
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,7 +82,7 @@ dependencies:
|
|
82
82
|
requirements:
|
83
83
|
- - ~>
|
84
84
|
- !ruby/object:Gem::Version
|
85
|
-
version: 1.
|
85
|
+
version: 1.2524.0
|
86
86
|
type: :runtime
|
87
87
|
prerelease: false
|
88
88
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -90,7 +90,7 @@ dependencies:
|
|
90
90
|
requirements:
|
91
91
|
- - ~>
|
92
92
|
- !ruby/object:Gem::Version
|
93
|
-
version: 1.
|
93
|
+
version: 1.2524.0
|
94
94
|
- !ruby/object:Gem::Dependency
|
95
95
|
name: bosh_cpi
|
96
96
|
requirement: !ruby/object:Gem::Requirement
|
@@ -98,7 +98,7 @@ dependencies:
|
|
98
98
|
requirements:
|
99
99
|
- - ~>
|
100
100
|
- !ruby/object:Gem::Version
|
101
|
-
version: 1.
|
101
|
+
version: 1.2524.0
|
102
102
|
type: :runtime
|
103
103
|
prerelease: false
|
104
104
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -106,7 +106,7 @@ dependencies:
|
|
106
106
|
requirements:
|
107
107
|
- - ~>
|
108
108
|
- !ruby/object:Gem::Version
|
109
|
-
version: 1.
|
109
|
+
version: 1.2524.0
|
110
110
|
- !ruby/object:Gem::Dependency
|
111
111
|
name: bosh_openstack_cpi
|
112
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -114,7 +114,7 @@ dependencies:
|
|
114
114
|
requirements:
|
115
115
|
- - ~>
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 1.
|
117
|
+
version: 1.2524.0
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -122,7 +122,7 @@ dependencies:
|
|
122
122
|
requirements:
|
123
123
|
- - ~>
|
124
124
|
- !ruby/object:Gem::Version
|
125
|
-
version: 1.
|
125
|
+
version: 1.2524.0
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: bosh_aws_cpi
|
128
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -130,7 +130,7 @@ dependencies:
|
|
130
130
|
requirements:
|
131
131
|
- - ~>
|
132
132
|
- !ruby/object:Gem::Version
|
133
|
-
version: 1.
|
133
|
+
version: 1.2524.0
|
134
134
|
type: :runtime
|
135
135
|
prerelease: false
|
136
136
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -138,7 +138,7 @@ dependencies:
|
|
138
138
|
requirements:
|
139
139
|
- - ~>
|
140
140
|
- !ruby/object:Gem::Version
|
141
|
-
version: 1.
|
141
|
+
version: 1.2524.0
|
142
142
|
- !ruby/object:Gem::Dependency
|
143
143
|
name: bosh_vsphere_cpi
|
144
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -146,7 +146,7 @@ dependencies:
|
|
146
146
|
requirements:
|
147
147
|
- - ~>
|
148
148
|
- !ruby/object:Gem::Version
|
149
|
-
version: 1.
|
149
|
+
version: 1.2524.0
|
150
150
|
type: :runtime
|
151
151
|
prerelease: false
|
152
152
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -154,7 +154,7 @@ dependencies:
|
|
154
154
|
requirements:
|
155
155
|
- - ~>
|
156
156
|
- !ruby/object:Gem::Version
|
157
|
-
version: 1.
|
157
|
+
version: 1.2524.0
|
158
158
|
- !ruby/object:Gem::Dependency
|
159
159
|
name: bosh_warden_cpi
|
160
160
|
requirement: !ruby/object:Gem::Requirement
|
@@ -162,7 +162,7 @@ dependencies:
|
|
162
162
|
requirements:
|
163
163
|
- - ~>
|
164
164
|
- !ruby/object:Gem::Version
|
165
|
-
version: 1.
|
165
|
+
version: 1.2524.0
|
166
166
|
type: :runtime
|
167
167
|
prerelease: false
|
168
168
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -170,7 +170,7 @@ dependencies:
|
|
170
170
|
requirements:
|
171
171
|
- - ~>
|
172
172
|
- !ruby/object:Gem::Version
|
173
|
-
version: 1.
|
173
|
+
version: 1.2524.0
|
174
174
|
- !ruby/object:Gem::Dependency
|
175
175
|
name: bosh_vcloud_cpi
|
176
176
|
requirement: !ruby/object:Gem::Requirement
|
@@ -331,6 +331,22 @@ dependencies:
|
|
331
331
|
- - ~>
|
332
332
|
- !ruby/object:Gem::Version
|
333
333
|
version: 1.23.0
|
334
|
+
- !ruby/object:Gem::Dependency
|
335
|
+
name: resque-backtrace
|
336
|
+
requirement: !ruby/object:Gem::Requirement
|
337
|
+
none: false
|
338
|
+
requirements:
|
339
|
+
- - ~>
|
340
|
+
- !ruby/object:Gem::Version
|
341
|
+
version: 0.0.1
|
342
|
+
type: :runtime
|
343
|
+
prerelease: false
|
344
|
+
version_requirements: !ruby/object:Gem::Requirement
|
345
|
+
none: false
|
346
|
+
requirements:
|
347
|
+
- - ~>
|
348
|
+
- !ruby/object:Gem::Version
|
349
|
+
version: 0.0.1
|
334
350
|
- !ruby/object:Gem::Dependency
|
335
351
|
name: rufus-scheduler
|
336
352
|
requirement: !ruby/object:Gem::Requirement
|
@@ -509,7 +525,7 @@ dependencies:
|
|
509
525
|
version: '1.0'
|
510
526
|
description: ! 'BOSH Director
|
511
527
|
|
512
|
-
|
528
|
+
0501e4'
|
513
529
|
email: support@cloudfoundry.com
|
514
530
|
executables:
|
515
531
|
- bosh-director
|
@@ -647,6 +663,7 @@ files:
|
|
647
663
|
- lib/bosh/director/errand.rb
|
648
664
|
- lib/bosh/director/errand/deployment_preparer.rb
|
649
665
|
- lib/bosh/director/errand/job_manager.rb
|
666
|
+
- lib/bosh/director/errand/result.rb
|
650
667
|
- lib/bosh/director/errand/runner.rb
|
651
668
|
- lib/bosh/director/errors.rb
|
652
669
|
- lib/bosh/director/event_log.rb
|
@@ -686,6 +703,8 @@ files:
|
|
686
703
|
- lib/bosh/director/jobs/vm_state.rb
|
687
704
|
- lib/bosh/director/lock.rb
|
688
705
|
- lib/bosh/director/lock_helper.rb
|
706
|
+
- lib/bosh/director/log_bundles_cleaner.rb
|
707
|
+
- lib/bosh/director/logs_fetcher.rb
|
689
708
|
- lib/bosh/director/models.rb
|
690
709
|
- lib/bosh/director/models/compiled_package.rb
|
691
710
|
- lib/bosh/director/models/deployment.rb
|
@@ -765,7 +784,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
765
784
|
version: '0'
|
766
785
|
segments:
|
767
786
|
- 0
|
768
|
-
hash:
|
787
|
+
hash: 451459249491372801
|
769
788
|
requirements: []
|
770
789
|
rubyforge_project:
|
771
790
|
rubygems_version: 1.8.23
|