rest-ftp-daemon 0.434.0 → 0.435.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile.lock +10 -10
- data/lib/rest-ftp-daemon/api/config.rb +8 -0
- data/lib/rest-ftp-daemon/api/dashboard.rb +5 -1
- data/lib/rest-ftp-daemon/api/debug.rb +3 -0
- data/lib/rest-ftp-daemon/api/jobs.rb +7 -0
- data/lib/rest-ftp-daemon/api/root.rb +18 -17
- data/lib/rest-ftp-daemon/api/status.rb +7 -0
- data/lib/rest-ftp-daemon/constants.rb +10 -17
- data/lib/rest-ftp-daemon/job.rb +18 -12
- data/lib/rest-ftp-daemon/job_queue.rb +2 -8
- data/lib/rest-ftp-daemon/jobs/transfer.rb +4 -4
- data/lib/rest-ftp-daemon/location.rb +0 -4
- data/lib/rest-ftp-daemon/notification.rb +6 -4
- data/lib/rest-ftp-daemon/remote/base.rb +9 -4
- data/lib/rest-ftp-daemon/views/dashboard_footer.haml +1 -1
- data/lib/rest-ftp-daemon/worker_pool.rb +1 -9
- data/lib/rest-ftp-daemon/workers/conchita.rb +1 -0
- data/lib/rest-ftp-daemon/workers/reporter.rb +1 -0
- data/lib/rest-ftp-daemon/workers/transfer.rb +21 -14
- data/lib/rest-ftp-daemon/workers/worker.rb +14 -0
- data/rest-ftp-daemon.gemspec +2 -2
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7c055c5d41a1ed721a1a3ff65bb7b27b9c3aee74
|
4
|
+
data.tar.gz: 84ef017d71bbf52321f3956a0303a5ccee879a80
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ee3b4ab65e21ca786a9d41766c71a783216765ad8651d579e6c1c46763432977a349516df998d5e448358737401f481f13fdfe41c31f54a62bbbedcde1a38acc
|
7
|
+
data.tar.gz: cd89a20b329a1172279ef5873489dfdf86dd23f4b25a39b4840ea4b52a5b08bc2a016aaced357eb2281e8e71ef7f1fed5bacb2fbbc1055b1ae07528f3784b0bf
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rest-ftp-daemon (0.
|
4
|
+
rest-ftp-daemon (0.435.0)
|
5
5
|
activesupport (~> 4.2)
|
6
6
|
api-auth
|
7
7
|
aws-sdk-resources (~> 2)
|
8
|
-
bmc-daemon-lib (~> 0.
|
8
|
+
bmc-daemon-lib (~> 0.4.1)
|
9
9
|
double-bag-ftps
|
10
10
|
facter
|
11
11
|
get_process_mem
|
@@ -37,16 +37,16 @@ GEM
|
|
37
37
|
addressable (2.4.0)
|
38
38
|
api-auth (2.0.1)
|
39
39
|
ast (2.3.0)
|
40
|
-
aws-sdk-core (2.6.
|
40
|
+
aws-sdk-core (2.6.2)
|
41
41
|
jmespath (~> 1.0)
|
42
|
-
aws-sdk-resources (2.6.
|
43
|
-
aws-sdk-core (= 2.6.
|
42
|
+
aws-sdk-resources (2.6.2)
|
43
|
+
aws-sdk-core (= 2.6.2)
|
44
44
|
axiom-types (0.1.1)
|
45
45
|
descendants_tracker (~> 0.0.4)
|
46
46
|
ice_nine (~> 0.11.0)
|
47
47
|
thread_safe (~> 0.3, >= 0.3.1)
|
48
|
-
bmc-daemon-lib (0.
|
49
|
-
chamber (~> 2.9)
|
48
|
+
bmc-daemon-lib (0.4.1)
|
49
|
+
chamber (~> 2.9.1)
|
50
50
|
builder (3.2.2)
|
51
51
|
chamber (2.9.1)
|
52
52
|
hashie (~> 3.3)
|
@@ -119,7 +119,7 @@ GEM
|
|
119
119
|
net-ssh (3.2.0)
|
120
120
|
netrc (0.11.0)
|
121
121
|
newrelic_rpm (3.16.2.321)
|
122
|
-
parser (2.3.1.
|
122
|
+
parser (2.3.1.4)
|
123
123
|
ast (~> 2.2)
|
124
124
|
powerpack (0.1.1)
|
125
125
|
pry (0.10.4)
|
@@ -130,7 +130,7 @@ GEM
|
|
130
130
|
rack-accept (0.4.5)
|
131
131
|
rack (>= 0.4)
|
132
132
|
rainbow (2.1.0)
|
133
|
-
rake (11.
|
133
|
+
rake (11.3.0)
|
134
134
|
representable (3.0.0)
|
135
135
|
declarative (~> 0.0.5)
|
136
136
|
uber (~> 0.0.15)
|
@@ -138,7 +138,7 @@ GEM
|
|
138
138
|
http-cookie (>= 1.0.2, < 2.0)
|
139
139
|
mime-types (>= 1.16, < 3.0)
|
140
140
|
netrc (~> 0.7)
|
141
|
-
rollbar (2.13.
|
141
|
+
rollbar (2.13.1)
|
142
142
|
multi_json
|
143
143
|
rspec (3.5.0)
|
144
144
|
rspec-core (~> 3.5.0)
|
@@ -6,11 +6,15 @@ require "grape"
|
|
6
6
|
|
7
7
|
module RestFtpDaemon
|
8
8
|
module API
|
9
|
-
class
|
9
|
+
class Dashboard < Grape::API
|
10
10
|
include BmcDaemonLib
|
11
11
|
|
12
12
|
### HELPERS
|
13
13
|
helpers do
|
14
|
+
def log_context
|
15
|
+
{caller: "API::Dashboard"}
|
16
|
+
end
|
17
|
+
|
14
18
|
def render name, values={}
|
15
19
|
# Prepare template engine
|
16
20
|
template = File.read("#{Conf.app_libs}/views/#{name}.haml")
|
@@ -5,6 +5,13 @@ module RestFtpDaemon
|
|
5
5
|
class Jobs < Grape::API
|
6
6
|
include BmcDaemonLib
|
7
7
|
|
8
|
+
### HELPERS
|
9
|
+
helpers do
|
10
|
+
def log_context
|
11
|
+
{caller: "API::Jobs"}
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
8
15
|
### EXCEPTIONS HANDLERS
|
9
16
|
rescue_from RestFtpDaemon::JobNotFound do |exception|
|
10
17
|
exception_error :api_job_not_found, 404, exception
|
@@ -7,16 +7,22 @@ module RestFtpDaemon
|
|
7
7
|
module API
|
8
8
|
class Root < Grape::API
|
9
9
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
10
|
-
include BmcDaemonLib
|
10
|
+
#include BmcDaemonLib
|
11
|
+
include BmcDaemonLib::LoggerHelper
|
12
|
+
|
11
13
|
|
12
14
|
### LOGGING & HELPERS
|
15
|
+
logger BmcDaemonLib::LoggerPool.instance.get :api
|
13
16
|
helpers RestFtpDaemon::CommonHelpers
|
14
17
|
helpers RestFtpDaemon::ApiHelpers
|
15
18
|
helpers BmcDaemonLib::LoggerHelper
|
16
19
|
|
17
20
|
helpers do
|
18
|
-
|
19
|
-
|
21
|
+
|
22
|
+
include BmcDaemonLib::LoggerHelper
|
23
|
+
|
24
|
+
def log_context
|
25
|
+
{caller: "API::Root"}
|
20
26
|
end
|
21
27
|
|
22
28
|
def logger
|
@@ -28,7 +34,7 @@ module RestFtpDaemon
|
|
28
34
|
lines = exception.message.lines.collect(&:strip).reject(&:empty?)
|
29
35
|
|
30
36
|
# Log error to file
|
31
|
-
log_error "#{
|
37
|
+
log_error "[#{error}] [#{http_code}] #{lines.shift} ", lines
|
32
38
|
|
33
39
|
# Return error
|
34
40
|
error!({
|
@@ -48,14 +54,14 @@ module RestFtpDaemon
|
|
48
54
|
|
49
55
|
## EXCEPTION HANDLERS
|
50
56
|
rescue_from :all do |exception|
|
51
|
-
Rollbar.error exception, "api
|
57
|
+
Rollbar.error exception, "api: #{exception.class.name}: #{exception.message}"
|
58
|
+
# Rollbar.error exception, "api [#{error}]: #{exception.class.name}: #{exception.message}"
|
52
59
|
#error!({error: :internal_server_error, message: exception.message}, 500)
|
53
|
-
exception_error :
|
60
|
+
exception_error :api_error, 500, exception
|
54
61
|
end
|
55
62
|
|
56
63
|
|
57
64
|
### CLASS CONFIG
|
58
|
-
logger BmcDaemonLib::LoggerPool.instance.get :api
|
59
65
|
do_not_route_head!
|
60
66
|
do_not_route_options!
|
61
67
|
# version 'v1'
|
@@ -66,20 +72,15 @@ module RestFtpDaemon
|
|
66
72
|
default_format :json
|
67
73
|
|
68
74
|
# Pretty JSON
|
69
|
-
formatter :json_tmp, ->(object, env) do
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
#if object.respond_to? to_hash
|
74
|
-
JSON.pretty_generate(object)
|
75
|
-
#end
|
76
|
-
put "-----"
|
77
|
-
end
|
75
|
+
# formatter :json_tmp, ->(object, env) do
|
76
|
+
# puts object.inspect
|
77
|
+
# JSON.pretty_generate(object)
|
78
|
+
# end
|
78
79
|
|
79
80
|
### MOUNTPOINTS
|
80
81
|
mount RestFtpDaemon::API::Status => MOUNT_STATUS
|
81
82
|
mount RestFtpDaemon::API::Jobs => MOUNT_JOBS
|
82
|
-
mount RestFtpDaemon::API::
|
83
|
+
mount RestFtpDaemon::API::Dashboard => MOUNT_BOARD
|
83
84
|
mount RestFtpDaemon::API::Config => MOUNT_CONFIG
|
84
85
|
mount RestFtpDaemon::API::Debug => MOUNT_DEBUG
|
85
86
|
|
@@ -16,8 +16,6 @@ JOB_FFMPEG_ATTRIBUTES = [:video_codec, :video_bitrate, :video_bitrate_toleranc
|
|
16
16
|
|
17
17
|
# Internal job infos
|
18
18
|
INFO_PARAMS = :params
|
19
|
-
|
20
|
-
|
21
19
|
INFO_ERROR_MESSAGE = :error_message
|
22
20
|
INFO_ERROR_EXCEPTION = :error_exception
|
23
21
|
INFO_ERROR_BACKTRACE = :error_backtrace
|
@@ -31,22 +29,17 @@ INFO_TRANFER_PROGRESS = :progress
|
|
31
29
|
INFO_TRANFER_BITRATE = :bitrate
|
32
30
|
INFO_TARGET_FILES = :target_files
|
33
31
|
|
34
|
-
|
35
32
|
# Constants: logger
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
LOG_PREFIX_WID = 8
|
47
|
-
LOG_PREFIX_JID = JOB_IDENT_LEN + 4
|
48
|
-
LOG_PREFIX_ID = 5
|
49
|
-
LOG_PREFIX_FORMAT = "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s "
|
33
|
+
LOGGER_FORMAT = {
|
34
|
+
# context: "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s ",
|
35
|
+
# context: "wid:%-8{wid} jid:%-12{jid} id:%-5{id}",
|
36
|
+
context: {
|
37
|
+
caller: "%-17s",
|
38
|
+
wid: "%-10s",
|
39
|
+
jid: "%-10s",
|
40
|
+
id: "%-8s",
|
41
|
+
}
|
42
|
+
}
|
50
43
|
|
51
44
|
|
52
45
|
# Constants: logger to be cleaned up
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -7,12 +7,11 @@ require "securerandom"
|
|
7
7
|
|
8
8
|
module RestFtpDaemon
|
9
9
|
class Job
|
10
|
+
include BmcDaemonLib::LoggerHelper
|
10
11
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
11
12
|
include CommonHelpers
|
12
13
|
|
13
14
|
# Logging
|
14
|
-
attr_reader :logger
|
15
|
-
include BmcDaemonLib::LoggerHelper
|
16
15
|
|
17
16
|
# Fields to be imported from params
|
18
17
|
IMPORTED = %w(type priority pool label priority source target overwrite notify mkdir tempfile video_options video_custom)
|
@@ -59,18 +58,16 @@ module RestFtpDaemon
|
|
59
58
|
@status = nil
|
60
59
|
@tentatives = 0
|
61
60
|
@wid = nil
|
62
|
-
|
63
|
-
# Update timestamps
|
64
61
|
@created_at = Time.now
|
65
62
|
|
63
|
+
# Logger # FIXME: should be :jobs
|
64
|
+
log_pipe :transfer
|
65
|
+
|
66
66
|
# Prepare configuration
|
67
67
|
@config = Conf[:transfer] || {}
|
68
68
|
@endpoints = Conf[:endpoints] || {}
|
69
69
|
@pools = Conf[:pools] || {}
|
70
70
|
|
71
|
-
# Logger
|
72
|
-
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
73
|
-
|
74
71
|
# Import query params
|
75
72
|
set_info INFO_PARAMS, params
|
76
73
|
IMPORTED.each do |field|
|
@@ -83,11 +80,16 @@ module RestFtpDaemon
|
|
83
80
|
# Prepare sources/target
|
84
81
|
raise RestFtpDaemon::JobAttributeMissing, "source" unless params[:source]
|
85
82
|
@source_loc = Location.new(params[:source])
|
86
|
-
log_info "Job.initialize source #{@source_loc.uri}"
|
87
83
|
|
88
84
|
raise RestFtpDaemon::JobAttributeMissing, "target" unless params[:target]
|
89
85
|
@target_loc = Location.new(params[:target])
|
90
|
-
|
86
|
+
|
87
|
+
# We're done!
|
88
|
+
log_info "Job.initialized", {
|
89
|
+
source: @source_loc.uri,
|
90
|
+
target: @target_loc.uri,
|
91
|
+
pool: @pool,
|
92
|
+
}
|
91
93
|
end
|
92
94
|
|
93
95
|
def reset
|
@@ -220,8 +222,12 @@ module RestFtpDaemon
|
|
220
222
|
|
221
223
|
private
|
222
224
|
|
223
|
-
def
|
224
|
-
|
225
|
+
def log_context
|
226
|
+
{
|
227
|
+
wid: @wid,
|
228
|
+
jid: @id,
|
229
|
+
#id: @id,
|
230
|
+
}
|
225
231
|
end
|
226
232
|
|
227
233
|
def touch_job
|
@@ -309,7 +315,7 @@ module RestFtpDaemon
|
|
309
315
|
end
|
310
316
|
|
311
317
|
# Log to Rollbar
|
312
|
-
Rollbar.warning
|
318
|
+
Rollbar.warning exception, "oops [#{error}]: #{exception.class.name}: #{exception.message}"
|
313
319
|
|
314
320
|
# Close ftp connexion if open
|
315
321
|
@remote.close unless @remote.nil? || !@remote.connected?
|
@@ -4,13 +4,11 @@ require 'singleton'
|
|
4
4
|
module RestFtpDaemon
|
5
5
|
class JobQueue
|
6
6
|
include Singleton
|
7
|
-
|
8
7
|
include BmcDaemonLib::LoggerHelper
|
9
8
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
10
9
|
include CommonHelpers
|
11
10
|
|
12
11
|
# Class options
|
13
|
-
attr_reader :logger
|
14
12
|
attr_reader :jobs
|
15
13
|
|
16
14
|
def initialize
|
@@ -28,10 +26,10 @@ module RestFtpDaemon
|
|
28
26
|
@mutex = Mutex.new
|
29
27
|
|
30
28
|
# Logger
|
31
|
-
|
29
|
+
log_pipe :queue
|
32
30
|
|
33
31
|
# Identifiers generator
|
34
|
-
@prefix = identifier
|
32
|
+
@prefix = identifier(JOB_IDENT_LEN)
|
35
33
|
log_info "JobQueue initialized (prefix: #{@prefix})"
|
36
34
|
end
|
37
35
|
|
@@ -259,10 +257,6 @@ module RestFtpDaemon
|
|
259
257
|
|
260
258
|
protected
|
261
259
|
|
262
|
-
def log_prefix
|
263
|
-
[nil, nil, nil]
|
264
|
-
end
|
265
|
-
|
266
260
|
def prefixed_id id
|
267
261
|
"#{@prefix}.#{id}"
|
268
262
|
end
|
@@ -22,16 +22,16 @@ module RestFtpDaemon
|
|
22
22
|
case target_uri
|
23
23
|
when URI::FTP
|
24
24
|
log_info "JobTransfer.do_before target_method FTP"
|
25
|
-
@remote = Remote::RemoteFTP.new @target_loc,
|
25
|
+
@remote = Remote::RemoteFTP.new @target_loc, log_context, @config[:debug_ftp]
|
26
26
|
when URI::FTPES, URI::FTPS
|
27
27
|
log_info "JobTransfer.do_before target_method FTPES/FTPS"
|
28
|
-
@remote = Remote::RemoteFTP.new @target_loc,
|
28
|
+
@remote = Remote::RemoteFTP.new @target_loc, log_context, @config[:debug_ftps], :ftpes
|
29
29
|
when URI::SFTP
|
30
30
|
log_info "JobTransfer.do_before target_method SFTP"
|
31
|
-
@remote = Remote::RemoteSFTP.new @target_loc,
|
31
|
+
@remote = Remote::RemoteSFTP.new @target_loc, log_context, @config[:debug_sftp]
|
32
32
|
when URI::S3
|
33
33
|
log_info "JobTransfer.do_before target_method S3"
|
34
|
-
@remote = Remote::RemoteS3.new @target_loc,
|
34
|
+
@remote = Remote::RemoteS3.new @target_loc, log_context, @config[:debug_s3]
|
35
35
|
else
|
36
36
|
message = "unknown scheme [#{@target_loc.scheme}] [#{target_uri.class.name}]"
|
37
37
|
log_info "JobTransfer.do_before #{message}"
|
@@ -8,7 +8,6 @@ module RestFtpDaemon
|
|
8
8
|
include CommonHelpers
|
9
9
|
|
10
10
|
# Class options
|
11
|
-
attr_reader :logger
|
12
11
|
attr_accessor :job_id
|
13
12
|
attr_accessor :signal
|
14
13
|
attr_accessor :error
|
@@ -27,7 +26,7 @@ module RestFtpDaemon
|
|
27
26
|
@jid = nil
|
28
27
|
|
29
28
|
# Logger
|
30
|
-
|
29
|
+
log_pipe :notify
|
31
30
|
|
32
31
|
# Handle the notification
|
33
32
|
log_info "initialized [#{@url}]"
|
@@ -104,8 +103,11 @@ module RestFtpDaemon
|
|
104
103
|
log_error "UNHANDLED EXCEPTION: #{e.message}", e.backtrace
|
105
104
|
end
|
106
105
|
|
107
|
-
def
|
108
|
-
|
106
|
+
def log_context
|
107
|
+
{
|
108
|
+
jid: @jid,
|
109
|
+
id: @id,
|
110
|
+
}
|
109
111
|
end
|
110
112
|
|
111
113
|
end
|
@@ -5,14 +5,13 @@ module RestFtpDaemon
|
|
5
5
|
include BmcDaemonLib::LoggerHelper
|
6
6
|
|
7
7
|
# Class options
|
8
|
-
attr_reader :logger
|
9
8
|
attr_reader :log_prefix
|
10
9
|
attr_accessor :job
|
11
10
|
|
12
11
|
# Delegate set_info info to Job
|
13
12
|
delegate :set_info, to: :job
|
14
13
|
|
15
|
-
def initialize target,
|
14
|
+
def initialize target, context, debug = false, ftpes = false
|
16
15
|
# Init
|
17
16
|
@target = target
|
18
17
|
@ftpes = ftpes
|
@@ -22,8 +21,8 @@ module RestFtpDaemon
|
|
22
21
|
@job = Job.new(nil, {})
|
23
22
|
|
24
23
|
# Logger
|
25
|
-
@
|
26
|
-
|
24
|
+
@context = context || {}
|
25
|
+
log_pipe :transfer
|
27
26
|
|
28
27
|
# Annnounce object
|
29
28
|
log_info "RemoteBase.initialize debug[#{debug}] target[#{target.path}] "
|
@@ -61,6 +60,12 @@ module RestFtpDaemon
|
|
61
60
|
puts "-------------------- SESSION CLOSING --------------------------"
|
62
61
|
end
|
63
62
|
|
63
|
+
protected
|
64
|
+
|
65
|
+
def log_context
|
66
|
+
@context
|
67
|
+
end
|
68
|
+
|
64
69
|
private
|
65
70
|
|
66
71
|
def split_path path
|
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
©
|
7
7
|
= "2014-#{Time.now.year}"
|
8
|
-
%a{href: "http://bmconseil.com/?ref=rftpd-
|
8
|
+
%a{href: "http://bmconseil.com/?ref=rftpd-dashboard", target: TARGET_BLANK} Bruno Medici Consultant
|
9
9
|
|
10
10
|
·
|
11
11
|
%a{href: "http://github.com/bmedici/rest-ftp-daemon/", target: TARGET_BLANK} GitHub
|
@@ -4,17 +4,15 @@ require 'singleton'
|
|
4
4
|
module RestFtpDaemon
|
5
5
|
class WorkerPool
|
6
6
|
include Singleton
|
7
|
-
|
8
7
|
include BmcDaemonLib::LoggerHelper
|
9
8
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
10
9
|
|
11
10
|
# Class options
|
12
|
-
attr_reader :logger
|
13
11
|
attr_reader :wid
|
14
12
|
|
15
13
|
def initialize
|
16
14
|
# Logger
|
17
|
-
|
15
|
+
log_pipe :workers
|
18
16
|
|
19
17
|
# Prepare status hash and vars
|
20
18
|
@statuses = {}
|
@@ -65,12 +63,6 @@ module RestFtpDaemon
|
|
65
63
|
@workers[wid] && @workers[wid].alive?
|
66
64
|
end
|
67
65
|
|
68
|
-
protected
|
69
|
-
|
70
|
-
def log_prefix
|
71
|
-
[nil, nil, nil]
|
72
|
-
end
|
73
|
-
|
74
66
|
private
|
75
67
|
|
76
68
|
def thread_variables thread
|
@@ -11,6 +11,7 @@ module RestFtpDaemon
|
|
11
11
|
@endpoints = Conf[:endpoints]
|
12
12
|
|
13
13
|
# Timeout and retry config
|
14
|
+
return "timeout disabled" if disabled?(@config[:timeout])
|
14
15
|
return "invalid timeout" unless @config[:timeout].to_i > 0
|
15
16
|
|
16
17
|
# Log that
|
@@ -24,7 +25,7 @@ module RestFtpDaemon
|
|
24
25
|
|
25
26
|
def worker_after
|
26
27
|
# Clean worker status
|
27
|
-
|
28
|
+
worker_set_jid nil
|
28
29
|
end
|
29
30
|
|
30
31
|
private
|
@@ -35,23 +36,24 @@ module RestFtpDaemon
|
|
35
36
|
job = RestFtpDaemon::JobQueue.instance.pop @pool
|
36
37
|
|
37
38
|
# Work on this job
|
38
|
-
|
39
|
+
job_process job
|
39
40
|
|
40
41
|
# Clean job status
|
41
42
|
job.wid = nil
|
42
43
|
#sleep 1
|
43
44
|
|
44
45
|
# Handle the retry if needed
|
45
|
-
|
46
|
+
job_result job
|
46
47
|
end
|
47
48
|
|
48
|
-
def
|
49
|
+
def job_process job
|
49
50
|
# Prepare job and worker for processing
|
50
|
-
|
51
|
+
worker_set_jid job.id
|
51
52
|
worker_status WORKER_STATUS_RUNNING, job
|
52
53
|
job.wid = Thread.current.thread_variable_get :wid
|
53
54
|
|
54
55
|
# Processs this job protected by a timeout
|
56
|
+
log_info "job_process: start working"
|
55
57
|
Timeout.timeout(@config[:timeout], RestFtpDaemon::JobTimeout) do
|
56
58
|
job.start
|
57
59
|
end
|
@@ -60,45 +62,45 @@ module RestFtpDaemon
|
|
60
62
|
RestFtpDaemon::Counters.instance.increment :jobs, :processed
|
61
63
|
|
62
64
|
rescue RestFtpDaemon::JobTimeout => ex
|
63
|
-
log_error "
|
65
|
+
log_error "job_process: TIMEOUT: started_at[#{job.started_at}] started_since[#{job.started_since}] #{ex.message}", ex.backtrace
|
64
66
|
worker_status WORKER_STATUS_TIMEOUT, job
|
65
67
|
|
66
68
|
# Inform the job
|
67
69
|
job.oops_end(:timeout, ex) unless job.nil?
|
68
70
|
|
69
71
|
rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::JobAttributeMissing, StandardError => ex
|
70
|
-
log_error "
|
72
|
+
log_error "job_process: CRASHED: ex[#{ex.class}] #{ex.message}", ex.backtrace
|
71
73
|
worker_status WORKER_STATUS_CRASHED
|
72
74
|
|
73
75
|
# Inform the job
|
74
76
|
job.oops_end(:crashed, ex) unless job.nil?
|
75
77
|
end
|
76
78
|
|
77
|
-
def
|
79
|
+
def job_result job
|
78
80
|
# If job status requires a retry, just restack it
|
79
81
|
if !job.error
|
80
82
|
# Processing successful
|
81
|
-
|
83
|
+
log_info "job_result: finished with success"
|
82
84
|
worker_status WORKER_STATUS_FINISHED, job
|
83
85
|
|
84
86
|
elsif error_not_eligible(job)
|
85
|
-
log_error "not retrying [#{job.error}] retry_on not eligible"
|
87
|
+
log_error "job_result: not retrying [#{job.error}] retry_on not eligible"
|
86
88
|
|
87
89
|
elsif error_reached_for(job)
|
88
|
-
log_error "not retrying [#{job.error}] retry_for reached [#{@config[:retry_for]}s]"
|
90
|
+
log_error "job_result: not retrying [#{job.error}] retry_for reached [#{@config[:retry_for]}s]"
|
89
91
|
|
90
92
|
elsif error_reached_max(job)
|
91
|
-
log_error "not retrying [#{job.error}] retry_max reached #{tentatives(job)}"
|
93
|
+
log_error "job_result: not retrying [#{job.error}] retry_max reached #{tentatives(job)}"
|
92
94
|
|
93
95
|
else
|
94
96
|
# Delay cannot be negative, and will be 1s minimum
|
95
97
|
retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
|
96
|
-
log_info "
|
98
|
+
log_info "job_result: retrying job [#{job.id}] in [#{retry_after}s] - tried #{tentatives(job)}"
|
97
99
|
|
98
100
|
# Wait !
|
99
101
|
worker_status WORKER_STATUS_RETRYING, job
|
100
102
|
sleep retry_after
|
101
|
-
log_debug "job [#{job.id}] requeued after [#{retry_after}s] delay"
|
103
|
+
log_debug "job_result: job [#{job.id}] requeued after [#{retry_after}s] delay"
|
102
104
|
|
103
105
|
# Now, requeue this job
|
104
106
|
RestFtpDaemon::JobQueue.instance.requeue job
|
@@ -129,6 +131,11 @@ module RestFtpDaemon
|
|
129
131
|
return job.tentatives >= @config[:retry_max]
|
130
132
|
end
|
131
133
|
|
134
|
+
def worker_set_jid jid
|
135
|
+
Thread.current.thread_variable_set :jid, jid
|
136
|
+
Thread.current.thread_variable_set :updated_at, Time.now
|
137
|
+
end
|
138
|
+
|
132
139
|
def tentatives job
|
133
140
|
"[#{job.tentatives}/#{@config[:retry_max]}]"
|
134
141
|
end
|
@@ -8,5 +8,19 @@ module RestFtpDaemon
|
|
8
8
|
add_transaction_tracer :worker_init, category: :task
|
9
9
|
add_transaction_tracer :worker_after, category: :task
|
10
10
|
add_transaction_tracer :worker_process, category: :task
|
11
|
+
|
12
|
+
protected
|
13
|
+
|
14
|
+
def log_context
|
15
|
+
{
|
16
|
+
wid: Thread.current.thread_variable_get(:wid),
|
17
|
+
jid: Thread.current.thread_variable_get(:jid),
|
18
|
+
}
|
19
|
+
end
|
20
|
+
|
21
|
+
def disabled? value
|
22
|
+
value.nil? || value === false || value == 0
|
23
|
+
end
|
24
|
+
|
11
25
|
end
|
12
26
|
end
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
|
4
4
|
# Project version
|
5
|
-
spec.version = "0.
|
5
|
+
spec.version = "0.435.0"
|
6
6
|
|
7
7
|
# Project description
|
8
8
|
spec.name = "rest-ftp-daemon"
|
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
|
|
32
32
|
spec.add_development_dependency "ruby-prof"
|
33
33
|
|
34
34
|
# Runtime dependencies
|
35
|
-
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.
|
35
|
+
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.4.1"
|
36
36
|
spec.add_runtime_dependency "json", "~> 1.8"
|
37
37
|
spec.add_runtime_dependency "thin", "~> 1.7"
|
38
38
|
spec.add_runtime_dependency "activesupport", "~> 4.2"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.435.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -114,14 +114,14 @@ dependencies:
|
|
114
114
|
requirements:
|
115
115
|
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.
|
117
|
+
version: 0.4.1
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.
|
124
|
+
version: 0.4.1
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: json
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|