rest-ftp-daemon 0.250.5 → 0.300.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +19 -14
- data/README.md +12 -3
- data/bin/rest-ftp-daemon +102 -96
- data/config.ru +5 -5
- data/defaults.yml +61 -0
- data/lib/rest-ftp-daemon.rb +10 -4
- data/lib/rest-ftp-daemon/api/config.rb +3 -2
- data/lib/rest-ftp-daemon/api/dashboard.rb +1 -4
- data/lib/rest-ftp-daemon/api/debug.rb +30 -17
- data/lib/rest-ftp-daemon/api/job_presenter.rb +0 -2
- data/lib/rest-ftp-daemon/api/jobs.rb +4 -3
- data/lib/rest-ftp-daemon/api/root.rb +7 -10
- data/lib/rest-ftp-daemon/api/status.rb +7 -13
- data/lib/rest-ftp-daemon/constants.rb +27 -45
- data/lib/rest-ftp-daemon/counters.rb +0 -4
- data/lib/rest-ftp-daemon/helpers.rb +3 -18
- data/lib/rest-ftp-daemon/job.rb +16 -21
- data/lib/rest-ftp-daemon/job_queue.rb +21 -14
- data/lib/rest-ftp-daemon/launcher.rb +26 -0
- data/lib/rest-ftp-daemon/logger_pool.rb +9 -19
- data/lib/rest-ftp-daemon/metrics.rb +41 -0
- data/lib/rest-ftp-daemon/notification.rb +7 -10
- data/lib/rest-ftp-daemon/remote.rb +4 -4
- data/lib/rest-ftp-daemon/remote_ftp.rb +10 -10
- data/lib/rest-ftp-daemon/remote_sftp.rb +13 -24
- data/lib/rest-ftp-daemon/views/dashboard.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_footer.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_header.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +2 -2
- data/lib/rest-ftp-daemon/worker.rb +43 -12
- data/lib/rest-ftp-daemon/worker_conchita.rb +15 -28
- data/lib/rest-ftp-daemon/worker_job.rb +30 -21
- data/lib/rest-ftp-daemon/worker_pool.rb +59 -50
- data/lib/rest-ftp-daemon/worker_reporter.rb +70 -0
- data/lib/shared/conf.rb +195 -0
- data/lib/shared/logger_formatter.rb +31 -0
- data/lib/shared/logger_helper.rb +78 -0
- data/rest-ftp-daemon.gemspec +23 -22
- data/{rest-ftp-daemon.yml.sample → rest-ftp-daemon.sample.yml} +10 -7
- data/spec/spec_helper.rb +1 -1
- metadata +30 -12
- data/lib/rest-ftp-daemon/logger.rb +0 -57
- data/lib/rest-ftp-daemon/logger_helper.rb +0 -36
- data/lib/rest-ftp-daemon/settings.rb +0 -57
@@ -6,7 +6,7 @@
|
|
6
6
|
%link{ href:"/css/bootstrap.css" , rel: "stylesheet"}
|
7
7
|
%link{ href:"/css/main.css" , rel: "stylesheet"}
|
8
8
|
-# %link{ href:"http://fonts.googleapis.com/css?family=Inconsolata:400,700" , rel: "stylesheet"}
|
9
|
-
%title="#{
|
9
|
+
%title="#{Conf.host} [#{Conf.app_env}] #{Conf.app_name}"
|
10
10
|
|
11
11
|
%body
|
12
12
|
|
@@ -21,7 +21,7 @@
|
|
21
21
|
|
22
22
|
.row
|
23
23
|
#box-tokens.col-md-6
|
24
|
-
= render :dashboard_tokens, {tokens:
|
24
|
+
= render :dashboard_tokens, {tokens: Conf[:endpoints] || {}}
|
25
25
|
|
26
26
|
#box-workers.col-md-3
|
27
27
|
= render :dashboard_workers
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
.btn-group.btn-group-sm
|
19
19
|
.btn.btn-default.btn-info Host
|
20
|
-
.btn.btn-default=
|
20
|
+
.btn.btn-default= Conf.host
|
21
21
|
|
22
22
|
.btn-group.btn-group-sm
|
23
23
|
.btn.btn-default.btn-info IP
|
@@ -29,4 +29,4 @@
|
|
29
29
|
|
30
30
|
.btn-group.btn-group-sm
|
31
31
|
.btn.btn-default.btn-info Started
|
32
|
-
.btn.btn-default= Helpers.datetime_short(
|
32
|
+
.btn.btn-default= Helpers.datetime_short(Conf.app_started)
|
@@ -1,12 +1,8 @@
|
|
1
1
|
module RestFtpDaemon
|
2
2
|
class Worker
|
3
|
-
include LoggerHelper
|
3
|
+
include Shared::LoggerHelper
|
4
4
|
attr_reader :logger
|
5
5
|
|
6
|
-
if Settings.newrelic_enabled?
|
7
|
-
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
8
|
-
end
|
9
|
-
|
10
6
|
def initialize wid, pool = nil
|
11
7
|
# Logger
|
12
8
|
@logger = RestFtpDaemon::LoggerPool.instance.get :workers
|
@@ -20,15 +16,28 @@ module RestFtpDaemon
|
|
20
16
|
Thread.current.thread_variable_set :wid, wid
|
21
17
|
Thread.current.thread_variable_set :started_at, Time.now
|
22
18
|
worker_status WORKER_STATUS_STARTING
|
19
|
+
|
20
|
+
# Load corker conf
|
21
|
+
load_config wid
|
23
22
|
end
|
24
23
|
|
25
24
|
protected
|
26
25
|
|
27
|
-
def
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
26
|
+
def wait_according_to_config
|
27
|
+
# Sleep for a few seconds
|
28
|
+
worker_status WORKER_STATUS_WAITING
|
29
|
+
sleep @config[:timer] if @config.is_a? Hash
|
30
|
+
end
|
31
|
+
|
32
|
+
def log_prefix
|
33
|
+
[
|
34
|
+
Thread.current.thread_variable_get(:wid),
|
35
|
+
Thread.current.thread_variable_get(:jid),
|
36
|
+
nil
|
37
|
+
]
|
38
|
+
end
|
39
|
+
|
40
|
+
def work
|
32
41
|
end
|
33
42
|
|
34
43
|
def start
|
@@ -52,9 +61,9 @@ module RestFtpDaemon
|
|
52
61
|
|
53
62
|
# Log this status change
|
54
63
|
if job.is_a?(Job)
|
55
|
-
log_info "#{status}
|
64
|
+
log_info "status [#{status}] on job[#{job.id}] status[#{job.status}] error[#{job.error}]"
|
56
65
|
else
|
57
|
-
log_info "#{status}"
|
66
|
+
log_info "status [#{status}]"
|
58
67
|
end
|
59
68
|
end
|
60
69
|
|
@@ -63,5 +72,27 @@ module RestFtpDaemon
|
|
63
72
|
Thread.current.thread_variable_set :updated_at, Time.now
|
64
73
|
end
|
65
74
|
|
75
|
+
private
|
76
|
+
|
77
|
+
def load_config wid
|
78
|
+
# My debug
|
79
|
+
@debug = (Conf.at :debug, wid) == true
|
80
|
+
@log_worker_status_changes = @debug
|
81
|
+
|
82
|
+
# My configuration
|
83
|
+
@config = Conf[wid]
|
84
|
+
if !@config.is_a? Hash
|
85
|
+
return log_info "#{self.class.name}: missing #{wid}/* configuration"
|
86
|
+
elsif @config[:timer].nil?
|
87
|
+
return log_info "#{self.class.name}: missing #{wid}/timer value"
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
# NewRelic instrumentation
|
92
|
+
if Conf.newrelic_enabled?
|
93
|
+
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
94
|
+
add_transaction_tracer :work, category: :task
|
95
|
+
end
|
96
|
+
|
66
97
|
end
|
67
98
|
end
|
@@ -3,29 +3,25 @@ module RestFtpDaemon
|
|
3
3
|
# Worker used to clean up the queue deleting expired jobs
|
4
4
|
class ConchitaWorker < Worker
|
5
5
|
|
6
|
-
def initialize wid =
|
7
|
-
#
|
6
|
+
def initialize wid, pool = nil
|
7
|
+
# Call dady and load my conf
|
8
8
|
super
|
9
9
|
|
10
|
-
# Use debug ?
|
11
|
-
@debug = (Settings.at :debug, :conchita) == true
|
12
|
-
@log_worker_status_changes = @debug
|
13
|
-
|
14
|
-
# Conchita configuration
|
15
|
-
@conchita = Settings.conchita
|
16
|
-
if !@conchita.is_a? Hash
|
17
|
-
return log_info "ConchitaWorker: missing conchita.* configuration"
|
18
|
-
elsif @conchita[:timer].nil?
|
19
|
-
return log_info "ConchitaWorker: missing conchita.timer value"
|
20
|
-
end
|
21
|
-
|
22
10
|
# Start main loop
|
23
|
-
log_info "
|
11
|
+
log_info "#{self.class.name} starting", @config
|
24
12
|
start
|
25
13
|
end
|
26
14
|
|
27
15
|
protected
|
28
16
|
|
17
|
+
# def log_prefix
|
18
|
+
# [
|
19
|
+
# Thread.current.thread_variable_get(:wid),
|
20
|
+
# nil,
|
21
|
+
# nil
|
22
|
+
# ]
|
23
|
+
# end
|
24
|
+
|
29
25
|
def work
|
30
26
|
# Announce we are working
|
31
27
|
worker_status WORKER_STATUS_CLEANING
|
@@ -36,26 +32,17 @@ module RestFtpDaemon
|
|
36
32
|
$queue.expire JOB_STATUS_QUEUED, maxage(JOB_STATUS_QUEUED), @debug
|
37
33
|
|
38
34
|
# Force garbage collector
|
39
|
-
GC.start if @
|
35
|
+
GC.start if @config["garbage_collector"]
|
40
36
|
|
41
37
|
rescue StandardError => e
|
42
|
-
log_error "
|
38
|
+
log_error "EXCEPTION: #{e.inspect}"
|
43
39
|
sleep 1
|
44
40
|
else
|
45
|
-
|
46
|
-
worker_status WORKER_STATUS_WAITING
|
47
|
-
|
48
|
-
# Sleep for a few seconds
|
49
|
-
sleep @conchita[:timer]
|
41
|
+
wait_according_to_config
|
50
42
|
end
|
51
43
|
|
52
44
|
def maxage status
|
53
|
-
@
|
54
|
-
end
|
55
|
-
|
56
|
-
|
57
|
-
if Settings.newrelic_enabled?
|
58
|
-
add_transaction_tracer :work, category: :task
|
45
|
+
@config["clean_#{status}"] || 0
|
59
46
|
end
|
60
47
|
|
61
48
|
end
|
@@ -2,22 +2,40 @@ module RestFtpDaemon
|
|
2
2
|
|
3
3
|
# Worker used to process Jobs
|
4
4
|
class JobWorker < Worker
|
5
|
-
#attr_reader :pool
|
6
5
|
|
7
6
|
def initialize wid, pool
|
8
|
-
#
|
7
|
+
# Call dady and load my conf
|
9
8
|
super
|
10
9
|
|
11
|
-
# Timeout config
|
12
|
-
@timeout
|
10
|
+
# Timeout and retry config
|
11
|
+
@timeout = (Conf.at(:transfer, :timeout) rescue nil)
|
12
|
+
@retry = (Conf.at(:retry) rescue {})
|
13
|
+
|
14
|
+
# Retry config
|
15
|
+
@retry_on_errors = Conf.at(:retry, :on_errors)
|
16
|
+
@retry_max_age = Conf.at(:retry, :max_age)
|
17
|
+
@retry_max_runs = Conf.at(:retry, :max_runs)
|
18
|
+
@retry_delay = Conf.at(:retry, :delay)
|
19
|
+
|
13
20
|
|
14
21
|
# Start main loop
|
15
|
-
log_info "JobWorker initializing",
|
22
|
+
log_info "JobWorker initializing", {
|
23
|
+
wid: wid,
|
24
|
+
pool: pool,
|
25
|
+
timeout: @timeout
|
26
|
+
}
|
16
27
|
start
|
17
28
|
end
|
18
29
|
|
19
30
|
protected
|
20
31
|
|
32
|
+
|
33
|
+
def load_config wid
|
34
|
+
# Do nothing
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
21
39
|
def work
|
22
40
|
# Wait for a job to be available in the queue
|
23
41
|
worker_status WORKER_STATUS_WAITING
|
@@ -31,26 +49,21 @@ module RestFtpDaemon
|
|
31
49
|
#sleep 1
|
32
50
|
|
33
51
|
# If job status requires a retry, just restack it
|
34
|
-
on_errors = Settings.at(:retry, :on_errors)
|
35
|
-
max_age = Settings.at(:retry, :max_age)
|
36
|
-
max_runs = Settings.at(:retry, :max_runs)
|
37
|
-
delay = Settings.at(:retry, :delay)
|
38
|
-
|
39
52
|
if !job.error
|
40
53
|
#log_info "job succeeded"
|
41
54
|
|
42
|
-
elsif !(
|
55
|
+
elsif !(@retry_on_errors.is_a?(Enumerable) && @retry_on_errors.include?(job.error))
|
43
56
|
log_error "not retrying: error not eligible"
|
44
57
|
|
45
|
-
elsif
|
46
|
-
log_error "not retrying: max_age reached (#{
|
58
|
+
elsif @retry_max_age && (job.age >= @retry_max_age)
|
59
|
+
log_error "not retrying: max_age reached (#{@retry_max_age} s)"
|
47
60
|
|
48
|
-
elsif
|
49
|
-
log_error "not retrying: max_runs reached (#{
|
61
|
+
elsif @retry_max_runs && (job.runs >= @retry_max_runs)
|
62
|
+
log_error "not retrying: max_runs reached (#{@retry_max_runs} tries)"
|
50
63
|
|
51
64
|
else
|
52
65
|
# Delay cannot be negative, and will be 1s minimum
|
53
|
-
retry_after = [
|
66
|
+
retry_after = [@retry_delay || DEFAULT_RETRY_DELAY, 1].max
|
54
67
|
log_info "retrying job: waiting for #{retry_after} seconds"
|
55
68
|
|
56
69
|
# Wait !
|
@@ -71,8 +84,8 @@ module RestFtpDaemon
|
|
71
84
|
|
72
85
|
def work_on_job job
|
73
86
|
# Prepare job and worker for processing
|
74
|
-
worker_status WORKER_STATUS_RUNNING, job
|
75
87
|
worker_jid job.id
|
88
|
+
worker_status WORKER_STATUS_RUNNING, job
|
76
89
|
job.wid = Thread.current.thread_variable_get :wid
|
77
90
|
|
78
91
|
# Processs this job protected by a timeout
|
@@ -101,9 +114,5 @@ module RestFtpDaemon
|
|
101
114
|
job.oops_after_crash ex unless job.nil?
|
102
115
|
end
|
103
116
|
|
104
|
-
if Settings.newrelic_enabled?
|
105
|
-
add_transaction_tracer :work, category: :task
|
106
|
-
end
|
107
|
-
|
108
117
|
end
|
109
118
|
end
|
@@ -2,14 +2,10 @@ module RestFtpDaemon
|
|
2
2
|
|
3
3
|
# Handles a pool of Worker objects
|
4
4
|
class WorkerPool
|
5
|
-
include LoggerHelper
|
5
|
+
include Shared::LoggerHelper
|
6
6
|
attr_reader :logger
|
7
7
|
attr_reader :wid
|
8
8
|
|
9
|
-
if Settings.newrelic_enabled?
|
10
|
-
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
11
|
-
end
|
12
|
-
|
13
9
|
def initialize
|
14
10
|
# Logger
|
15
11
|
@logger = RestFtpDaemon::LoggerPool.instance.get :workers
|
@@ -17,93 +13,106 @@ module RestFtpDaemon
|
|
17
13
|
# Prepare status hash and vars
|
18
14
|
@statuses = {}
|
19
15
|
@workers = {}
|
20
|
-
@conchita = nil
|
21
16
|
@mutex = Mutex.new
|
22
17
|
|
23
18
|
# Identifiers generator
|
24
|
-
@
|
19
|
+
@last_worker_id = 0
|
25
20
|
|
26
21
|
# Create worker threads
|
27
22
|
create_threads
|
28
23
|
end
|
29
24
|
|
30
25
|
def worker_variables
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
vars[var] = worker.thread_variable_get var
|
35
|
-
end
|
36
|
-
vars
|
26
|
+
vars = {}
|
27
|
+
@workers.collect do |wid, worker|
|
28
|
+
vars[wid] = thread_variables worker
|
37
29
|
end
|
30
|
+
vars
|
38
31
|
end
|
39
32
|
|
40
33
|
def worker_alive? wid
|
41
34
|
@workers[wid] && @workers[wid].alive?
|
42
35
|
end
|
43
36
|
|
37
|
+
protected
|
38
|
+
|
39
|
+
def log_prefix
|
40
|
+
[nil, nil, nil]
|
41
|
+
end
|
42
|
+
|
44
43
|
private
|
45
44
|
|
46
|
-
def
|
45
|
+
def thread_variables thread
|
46
|
+
vars = {}
|
47
|
+
thread.thread_variables.each do |var|
|
48
|
+
vars[var] = thread.thread_variable_get var
|
49
|
+
end
|
50
|
+
vars
|
51
|
+
end
|
52
|
+
|
53
|
+
def next_worker_id
|
47
54
|
@mutex.synchronize do
|
48
|
-
@
|
55
|
+
@last_worker_id += 1
|
49
56
|
end
|
50
|
-
"w#{@
|
57
|
+
"w#{@last_worker_id}"
|
51
58
|
end
|
52
59
|
|
53
60
|
def create_threads
|
54
|
-
# Read configuration
|
55
|
-
pools =
|
61
|
+
# Read configuration or initialize with empty hash
|
62
|
+
pools = Conf[:pools]
|
63
|
+
pools = {} unless pools.is_a? Hash
|
56
64
|
|
57
65
|
# Minimum one worker on DEFAULT_POOL
|
58
66
|
if !(pools.is_a? Hash)
|
59
|
-
log_error "create_threads: one
|
67
|
+
log_error "create_threads: one JobWorker is the minimum (#{pools.inspect}"
|
60
68
|
end
|
61
|
-
|
69
|
+
log_info "WorkerPool creating workers - JobWorker #{pools.to_hash.inspect}"
|
62
70
|
|
63
|
-
#
|
64
|
-
|
71
|
+
# Start ConchitaWorker and ReporterWorker
|
72
|
+
create_thread :conchita, ConchitaWorker
|
73
|
+
create_thread :reporter, ReporterWorker
|
74
|
+
|
75
|
+
# Start JobWorkers threads, ensure we have at least one worker in default pool
|
76
|
+
pools[DEFAULT_POOL] ||= 1
|
65
77
|
pools.each do |pool, count|
|
66
|
-
# Start worker threads for each pool
|
67
|
-
log_info "WorkerPool creating JobWorker [#{pool}] x#{count}"
|
68
78
|
count.times do
|
69
|
-
wid =
|
70
|
-
@workers[wid] =
|
79
|
+
wid = next_worker_id
|
80
|
+
@workers[wid] = create_thread wid, JobWorker, pool
|
71
81
|
end
|
72
82
|
end
|
73
83
|
|
74
|
-
# Start conchita thread
|
75
|
-
log_info "WorkerPool creating ConchitaWorker"
|
76
|
-
@conchita = create_conchita_thread
|
77
|
-
|
78
84
|
rescue StandardError => ex
|
79
|
-
log_error "
|
80
|
-
end
|
81
|
-
|
82
|
-
def create_worker_thread wid, pool
|
83
|
-
Thread.new wid do
|
84
|
-
begin
|
85
|
-
worker = JobWorker.new wid, pool
|
86
|
-
log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
|
87
|
-
rescue StandardError => ex
|
88
|
-
log_error "JobWorker EXCEPTION: #{ex.message}"
|
89
|
-
end
|
90
|
-
end
|
85
|
+
log_error "EXCEPTION: #{ex.message}", ex.backtrace
|
91
86
|
end
|
92
87
|
|
93
|
-
def
|
94
|
-
|
88
|
+
# def create_worker_thread wid, pool
|
89
|
+
# Thread.new wid do
|
90
|
+
# begin
|
91
|
+
# worker = JobWorker.new wid, pool
|
92
|
+
# #log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
|
93
|
+
# rescue StandardError => ex
|
94
|
+
# log_error "JobWorker EXCEPTION: #{ex.message} #{e.backtrace}"
|
95
|
+
# end
|
96
|
+
# end
|
97
|
+
# end
|
98
|
+
|
99
|
+
def create_thread wid, klass, pool = nil
|
100
|
+
# Spawn thread and add it to my index
|
101
|
+
log_info "spawning #{klass.name} [#{wid}] [#{pool}]"
|
102
|
+
@workers[wid] = Thread.new do
|
95
103
|
begin
|
96
|
-
worker =
|
97
|
-
log_info "ConchitaWorker: #{worker}"
|
104
|
+
worker = klass.new wid, pool
|
98
105
|
rescue StandardError => ex
|
99
|
-
log_error "
|
106
|
+
log_error "#{klass.name} EXCEPTION: #{ex.message}"
|
100
107
|
end
|
101
108
|
end
|
102
109
|
end
|
103
110
|
|
104
|
-
|
105
|
-
|
106
|
-
|
111
|
+
# NewRelic instrumentation
|
112
|
+
if Conf.newrelic_enabled?
|
113
|
+
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
114
|
+
# add_transaction_tracer :create_conchita_thread, category: :task
|
115
|
+
add_transaction_tracer :create_thread, category: :task
|
107
116
|
end
|
108
117
|
|
109
118
|
end
|