rest-ftp-daemon 0.434.0 → 0.435.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile.lock +10 -10
- data/lib/rest-ftp-daemon/api/config.rb +8 -0
- data/lib/rest-ftp-daemon/api/dashboard.rb +5 -1
- data/lib/rest-ftp-daemon/api/debug.rb +3 -0
- data/lib/rest-ftp-daemon/api/jobs.rb +7 -0
- data/lib/rest-ftp-daemon/api/root.rb +18 -17
- data/lib/rest-ftp-daemon/api/status.rb +7 -0
- data/lib/rest-ftp-daemon/constants.rb +10 -17
- data/lib/rest-ftp-daemon/job.rb +18 -12
- data/lib/rest-ftp-daemon/job_queue.rb +2 -8
- data/lib/rest-ftp-daemon/jobs/transfer.rb +4 -4
- data/lib/rest-ftp-daemon/location.rb +0 -4
- data/lib/rest-ftp-daemon/notification.rb +6 -4
- data/lib/rest-ftp-daemon/remote/base.rb +9 -4
- data/lib/rest-ftp-daemon/views/dashboard_footer.haml +1 -1
- data/lib/rest-ftp-daemon/worker_pool.rb +1 -9
- data/lib/rest-ftp-daemon/workers/conchita.rb +1 -0
- data/lib/rest-ftp-daemon/workers/reporter.rb +1 -0
- data/lib/rest-ftp-daemon/workers/transfer.rb +21 -14
- data/lib/rest-ftp-daemon/workers/worker.rb +14 -0
- data/rest-ftp-daemon.gemspec +2 -2
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7c055c5d41a1ed721a1a3ff65bb7b27b9c3aee74
|
4
|
+
data.tar.gz: 84ef017d71bbf52321f3956a0303a5ccee879a80
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ee3b4ab65e21ca786a9d41766c71a783216765ad8651d579e6c1c46763432977a349516df998d5e448358737401f481f13fdfe41c31f54a62bbbedcde1a38acc
|
7
|
+
data.tar.gz: cd89a20b329a1172279ef5873489dfdf86dd23f4b25a39b4840ea4b52a5b08bc2a016aaced357eb2281e8e71ef7f1fed5bacb2fbbc1055b1ae07528f3784b0bf
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rest-ftp-daemon (0.
|
4
|
+
rest-ftp-daemon (0.435.0)
|
5
5
|
activesupport (~> 4.2)
|
6
6
|
api-auth
|
7
7
|
aws-sdk-resources (~> 2)
|
8
|
-
bmc-daemon-lib (~> 0.
|
8
|
+
bmc-daemon-lib (~> 0.4.1)
|
9
9
|
double-bag-ftps
|
10
10
|
facter
|
11
11
|
get_process_mem
|
@@ -37,16 +37,16 @@ GEM
|
|
37
37
|
addressable (2.4.0)
|
38
38
|
api-auth (2.0.1)
|
39
39
|
ast (2.3.0)
|
40
|
-
aws-sdk-core (2.6.
|
40
|
+
aws-sdk-core (2.6.2)
|
41
41
|
jmespath (~> 1.0)
|
42
|
-
aws-sdk-resources (2.6.
|
43
|
-
aws-sdk-core (= 2.6.
|
42
|
+
aws-sdk-resources (2.6.2)
|
43
|
+
aws-sdk-core (= 2.6.2)
|
44
44
|
axiom-types (0.1.1)
|
45
45
|
descendants_tracker (~> 0.0.4)
|
46
46
|
ice_nine (~> 0.11.0)
|
47
47
|
thread_safe (~> 0.3, >= 0.3.1)
|
48
|
-
bmc-daemon-lib (0.
|
49
|
-
chamber (~> 2.9)
|
48
|
+
bmc-daemon-lib (0.4.1)
|
49
|
+
chamber (~> 2.9.1)
|
50
50
|
builder (3.2.2)
|
51
51
|
chamber (2.9.1)
|
52
52
|
hashie (~> 3.3)
|
@@ -119,7 +119,7 @@ GEM
|
|
119
119
|
net-ssh (3.2.0)
|
120
120
|
netrc (0.11.0)
|
121
121
|
newrelic_rpm (3.16.2.321)
|
122
|
-
parser (2.3.1.
|
122
|
+
parser (2.3.1.4)
|
123
123
|
ast (~> 2.2)
|
124
124
|
powerpack (0.1.1)
|
125
125
|
pry (0.10.4)
|
@@ -130,7 +130,7 @@ GEM
|
|
130
130
|
rack-accept (0.4.5)
|
131
131
|
rack (>= 0.4)
|
132
132
|
rainbow (2.1.0)
|
133
|
-
rake (11.
|
133
|
+
rake (11.3.0)
|
134
134
|
representable (3.0.0)
|
135
135
|
declarative (~> 0.0.5)
|
136
136
|
uber (~> 0.0.15)
|
@@ -138,7 +138,7 @@ GEM
|
|
138
138
|
http-cookie (>= 1.0.2, < 2.0)
|
139
139
|
mime-types (>= 1.16, < 3.0)
|
140
140
|
netrc (~> 0.7)
|
141
|
-
rollbar (2.13.
|
141
|
+
rollbar (2.13.1)
|
142
142
|
multi_json
|
143
143
|
rspec (3.5.0)
|
144
144
|
rspec-core (~> 3.5.0)
|
@@ -6,11 +6,15 @@ require "grape"
|
|
6
6
|
|
7
7
|
module RestFtpDaemon
|
8
8
|
module API
|
9
|
-
class
|
9
|
+
class Dashboard < Grape::API
|
10
10
|
include BmcDaemonLib
|
11
11
|
|
12
12
|
### HELPERS
|
13
13
|
helpers do
|
14
|
+
def log_context
|
15
|
+
{caller: "API::Dashboard"}
|
16
|
+
end
|
17
|
+
|
14
18
|
def render name, values={}
|
15
19
|
# Prepare template engine
|
16
20
|
template = File.read("#{Conf.app_libs}/views/#{name}.haml")
|
@@ -5,6 +5,13 @@ module RestFtpDaemon
|
|
5
5
|
class Jobs < Grape::API
|
6
6
|
include BmcDaemonLib
|
7
7
|
|
8
|
+
### HELPERS
|
9
|
+
helpers do
|
10
|
+
def log_context
|
11
|
+
{caller: "API::Jobs"}
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
8
15
|
### EXCEPTIONS HANDLERS
|
9
16
|
rescue_from RestFtpDaemon::JobNotFound do |exception|
|
10
17
|
exception_error :api_job_not_found, 404, exception
|
@@ -7,16 +7,22 @@ module RestFtpDaemon
|
|
7
7
|
module API
|
8
8
|
class Root < Grape::API
|
9
9
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
10
|
-
include BmcDaemonLib
|
10
|
+
#include BmcDaemonLib
|
11
|
+
include BmcDaemonLib::LoggerHelper
|
12
|
+
|
11
13
|
|
12
14
|
### LOGGING & HELPERS
|
15
|
+
logger BmcDaemonLib::LoggerPool.instance.get :api
|
13
16
|
helpers RestFtpDaemon::CommonHelpers
|
14
17
|
helpers RestFtpDaemon::ApiHelpers
|
15
18
|
helpers BmcDaemonLib::LoggerHelper
|
16
19
|
|
17
20
|
helpers do
|
18
|
-
|
19
|
-
|
21
|
+
|
22
|
+
include BmcDaemonLib::LoggerHelper
|
23
|
+
|
24
|
+
def log_context
|
25
|
+
{caller: "API::Root"}
|
20
26
|
end
|
21
27
|
|
22
28
|
def logger
|
@@ -28,7 +34,7 @@ module RestFtpDaemon
|
|
28
34
|
lines = exception.message.lines.collect(&:strip).reject(&:empty?)
|
29
35
|
|
30
36
|
# Log error to file
|
31
|
-
log_error "#{
|
37
|
+
log_error "[#{error}] [#{http_code}] #{lines.shift} ", lines
|
32
38
|
|
33
39
|
# Return error
|
34
40
|
error!({
|
@@ -48,14 +54,14 @@ module RestFtpDaemon
|
|
48
54
|
|
49
55
|
## EXCEPTION HANDLERS
|
50
56
|
rescue_from :all do |exception|
|
51
|
-
Rollbar.error exception, "api
|
57
|
+
Rollbar.error exception, "api: #{exception.class.name}: #{exception.message}"
|
58
|
+
# Rollbar.error exception, "api [#{error}]: #{exception.class.name}: #{exception.message}"
|
52
59
|
#error!({error: :internal_server_error, message: exception.message}, 500)
|
53
|
-
exception_error :
|
60
|
+
exception_error :api_error, 500, exception
|
54
61
|
end
|
55
62
|
|
56
63
|
|
57
64
|
### CLASS CONFIG
|
58
|
-
logger BmcDaemonLib::LoggerPool.instance.get :api
|
59
65
|
do_not_route_head!
|
60
66
|
do_not_route_options!
|
61
67
|
# version 'v1'
|
@@ -66,20 +72,15 @@ module RestFtpDaemon
|
|
66
72
|
default_format :json
|
67
73
|
|
68
74
|
# Pretty JSON
|
69
|
-
formatter :json_tmp, ->(object, env) do
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
#if object.respond_to? to_hash
|
74
|
-
JSON.pretty_generate(object)
|
75
|
-
#end
|
76
|
-
put "-----"
|
77
|
-
end
|
75
|
+
# formatter :json_tmp, ->(object, env) do
|
76
|
+
# puts object.inspect
|
77
|
+
# JSON.pretty_generate(object)
|
78
|
+
# end
|
78
79
|
|
79
80
|
### MOUNTPOINTS
|
80
81
|
mount RestFtpDaemon::API::Status => MOUNT_STATUS
|
81
82
|
mount RestFtpDaemon::API::Jobs => MOUNT_JOBS
|
82
|
-
mount RestFtpDaemon::API::
|
83
|
+
mount RestFtpDaemon::API::Dashboard => MOUNT_BOARD
|
83
84
|
mount RestFtpDaemon::API::Config => MOUNT_CONFIG
|
84
85
|
mount RestFtpDaemon::API::Debug => MOUNT_DEBUG
|
85
86
|
|
@@ -16,8 +16,6 @@ JOB_FFMPEG_ATTRIBUTES = [:video_codec, :video_bitrate, :video_bitrate_toleranc
|
|
16
16
|
|
17
17
|
# Internal job infos
|
18
18
|
INFO_PARAMS = :params
|
19
|
-
|
20
|
-
|
21
19
|
INFO_ERROR_MESSAGE = :error_message
|
22
20
|
INFO_ERROR_EXCEPTION = :error_exception
|
23
21
|
INFO_ERROR_BACKTRACE = :error_backtrace
|
@@ -31,22 +29,17 @@ INFO_TRANFER_PROGRESS = :progress
|
|
31
29
|
INFO_TRANFER_BITRATE = :bitrate
|
32
30
|
INFO_TARGET_FILES = :target_files
|
33
31
|
|
34
|
-
|
35
32
|
# Constants: logger
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
LOG_PREFIX_WID = 8
|
47
|
-
LOG_PREFIX_JID = JOB_IDENT_LEN + 4
|
48
|
-
LOG_PREFIX_ID = 5
|
49
|
-
LOG_PREFIX_FORMAT = "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s "
|
33
|
+
LOGGER_FORMAT = {
|
34
|
+
# context: "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s ",
|
35
|
+
# context: "wid:%-8{wid} jid:%-12{jid} id:%-5{id}",
|
36
|
+
context: {
|
37
|
+
caller: "%-17s",
|
38
|
+
wid: "%-10s",
|
39
|
+
jid: "%-10s",
|
40
|
+
id: "%-8s",
|
41
|
+
}
|
42
|
+
}
|
50
43
|
|
51
44
|
|
52
45
|
# Constants: logger to be cleaned up
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -7,12 +7,11 @@ require "securerandom"
|
|
7
7
|
|
8
8
|
module RestFtpDaemon
|
9
9
|
class Job
|
10
|
+
include BmcDaemonLib::LoggerHelper
|
10
11
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
11
12
|
include CommonHelpers
|
12
13
|
|
13
14
|
# Logging
|
14
|
-
attr_reader :logger
|
15
|
-
include BmcDaemonLib::LoggerHelper
|
16
15
|
|
17
16
|
# Fields to be imported from params
|
18
17
|
IMPORTED = %w(type priority pool label priority source target overwrite notify mkdir tempfile video_options video_custom)
|
@@ -59,18 +58,16 @@ module RestFtpDaemon
|
|
59
58
|
@status = nil
|
60
59
|
@tentatives = 0
|
61
60
|
@wid = nil
|
62
|
-
|
63
|
-
# Update timestamps
|
64
61
|
@created_at = Time.now
|
65
62
|
|
63
|
+
# Logger # FIXME: should be :jobs
|
64
|
+
log_pipe :transfer
|
65
|
+
|
66
66
|
# Prepare configuration
|
67
67
|
@config = Conf[:transfer] || {}
|
68
68
|
@endpoints = Conf[:endpoints] || {}
|
69
69
|
@pools = Conf[:pools] || {}
|
70
70
|
|
71
|
-
# Logger
|
72
|
-
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
73
|
-
|
74
71
|
# Import query params
|
75
72
|
set_info INFO_PARAMS, params
|
76
73
|
IMPORTED.each do |field|
|
@@ -83,11 +80,16 @@ module RestFtpDaemon
|
|
83
80
|
# Prepare sources/target
|
84
81
|
raise RestFtpDaemon::JobAttributeMissing, "source" unless params[:source]
|
85
82
|
@source_loc = Location.new(params[:source])
|
86
|
-
log_info "Job.initialize source #{@source_loc.uri}"
|
87
83
|
|
88
84
|
raise RestFtpDaemon::JobAttributeMissing, "target" unless params[:target]
|
89
85
|
@target_loc = Location.new(params[:target])
|
90
|
-
|
86
|
+
|
87
|
+
# We're done!
|
88
|
+
log_info "Job.initialized", {
|
89
|
+
source: @source_loc.uri,
|
90
|
+
target: @target_loc.uri,
|
91
|
+
pool: @pool,
|
92
|
+
}
|
91
93
|
end
|
92
94
|
|
93
95
|
def reset
|
@@ -220,8 +222,12 @@ module RestFtpDaemon
|
|
220
222
|
|
221
223
|
private
|
222
224
|
|
223
|
-
def
|
224
|
-
|
225
|
+
def log_context
|
226
|
+
{
|
227
|
+
wid: @wid,
|
228
|
+
jid: @id,
|
229
|
+
#id: @id,
|
230
|
+
}
|
225
231
|
end
|
226
232
|
|
227
233
|
def touch_job
|
@@ -309,7 +315,7 @@ module RestFtpDaemon
|
|
309
315
|
end
|
310
316
|
|
311
317
|
# Log to Rollbar
|
312
|
-
Rollbar.warning
|
318
|
+
Rollbar.warning exception, "oops [#{error}]: #{exception.class.name}: #{exception.message}"
|
313
319
|
|
314
320
|
# Close ftp connexion if open
|
315
321
|
@remote.close unless @remote.nil? || !@remote.connected?
|
@@ -4,13 +4,11 @@ require 'singleton'
|
|
4
4
|
module RestFtpDaemon
|
5
5
|
class JobQueue
|
6
6
|
include Singleton
|
7
|
-
|
8
7
|
include BmcDaemonLib::LoggerHelper
|
9
8
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
10
9
|
include CommonHelpers
|
11
10
|
|
12
11
|
# Class options
|
13
|
-
attr_reader :logger
|
14
12
|
attr_reader :jobs
|
15
13
|
|
16
14
|
def initialize
|
@@ -28,10 +26,10 @@ module RestFtpDaemon
|
|
28
26
|
@mutex = Mutex.new
|
29
27
|
|
30
28
|
# Logger
|
31
|
-
|
29
|
+
log_pipe :queue
|
32
30
|
|
33
31
|
# Identifiers generator
|
34
|
-
@prefix = identifier
|
32
|
+
@prefix = identifier(JOB_IDENT_LEN)
|
35
33
|
log_info "JobQueue initialized (prefix: #{@prefix})"
|
36
34
|
end
|
37
35
|
|
@@ -259,10 +257,6 @@ module RestFtpDaemon
|
|
259
257
|
|
260
258
|
protected
|
261
259
|
|
262
|
-
def log_prefix
|
263
|
-
[nil, nil, nil]
|
264
|
-
end
|
265
|
-
|
266
260
|
def prefixed_id id
|
267
261
|
"#{@prefix}.#{id}"
|
268
262
|
end
|
@@ -22,16 +22,16 @@ module RestFtpDaemon
|
|
22
22
|
case target_uri
|
23
23
|
when URI::FTP
|
24
24
|
log_info "JobTransfer.do_before target_method FTP"
|
25
|
-
@remote = Remote::RemoteFTP.new @target_loc,
|
25
|
+
@remote = Remote::RemoteFTP.new @target_loc, log_context, @config[:debug_ftp]
|
26
26
|
when URI::FTPES, URI::FTPS
|
27
27
|
log_info "JobTransfer.do_before target_method FTPES/FTPS"
|
28
|
-
@remote = Remote::RemoteFTP.new @target_loc,
|
28
|
+
@remote = Remote::RemoteFTP.new @target_loc, log_context, @config[:debug_ftps], :ftpes
|
29
29
|
when URI::SFTP
|
30
30
|
log_info "JobTransfer.do_before target_method SFTP"
|
31
|
-
@remote = Remote::RemoteSFTP.new @target_loc,
|
31
|
+
@remote = Remote::RemoteSFTP.new @target_loc, log_context, @config[:debug_sftp]
|
32
32
|
when URI::S3
|
33
33
|
log_info "JobTransfer.do_before target_method S3"
|
34
|
-
@remote = Remote::RemoteS3.new @target_loc,
|
34
|
+
@remote = Remote::RemoteS3.new @target_loc, log_context, @config[:debug_s3]
|
35
35
|
else
|
36
36
|
message = "unknown scheme [#{@target_loc.scheme}] [#{target_uri.class.name}]"
|
37
37
|
log_info "JobTransfer.do_before #{message}"
|
@@ -8,7 +8,6 @@ module RestFtpDaemon
|
|
8
8
|
include CommonHelpers
|
9
9
|
|
10
10
|
# Class options
|
11
|
-
attr_reader :logger
|
12
11
|
attr_accessor :job_id
|
13
12
|
attr_accessor :signal
|
14
13
|
attr_accessor :error
|
@@ -27,7 +26,7 @@ module RestFtpDaemon
|
|
27
26
|
@jid = nil
|
28
27
|
|
29
28
|
# Logger
|
30
|
-
|
29
|
+
log_pipe :notify
|
31
30
|
|
32
31
|
# Handle the notification
|
33
32
|
log_info "initialized [#{@url}]"
|
@@ -104,8 +103,11 @@ module RestFtpDaemon
|
|
104
103
|
log_error "UNHANDLED EXCEPTION: #{e.message}", e.backtrace
|
105
104
|
end
|
106
105
|
|
107
|
-
def
|
108
|
-
|
106
|
+
def log_context
|
107
|
+
{
|
108
|
+
jid: @jid,
|
109
|
+
id: @id,
|
110
|
+
}
|
109
111
|
end
|
110
112
|
|
111
113
|
end
|
@@ -5,14 +5,13 @@ module RestFtpDaemon
|
|
5
5
|
include BmcDaemonLib::LoggerHelper
|
6
6
|
|
7
7
|
# Class options
|
8
|
-
attr_reader :logger
|
9
8
|
attr_reader :log_prefix
|
10
9
|
attr_accessor :job
|
11
10
|
|
12
11
|
# Delegate set_info info to Job
|
13
12
|
delegate :set_info, to: :job
|
14
13
|
|
15
|
-
def initialize target,
|
14
|
+
def initialize target, context, debug = false, ftpes = false
|
16
15
|
# Init
|
17
16
|
@target = target
|
18
17
|
@ftpes = ftpes
|
@@ -22,8 +21,8 @@ module RestFtpDaemon
|
|
22
21
|
@job = Job.new(nil, {})
|
23
22
|
|
24
23
|
# Logger
|
25
|
-
@
|
26
|
-
|
24
|
+
@context = context || {}
|
25
|
+
log_pipe :transfer
|
27
26
|
|
28
27
|
# Annnounce object
|
29
28
|
log_info "RemoteBase.initialize debug[#{debug}] target[#{target.path}] "
|
@@ -61,6 +60,12 @@ module RestFtpDaemon
|
|
61
60
|
puts "-------------------- SESSION CLOSING --------------------------"
|
62
61
|
end
|
63
62
|
|
63
|
+
protected
|
64
|
+
|
65
|
+
def log_context
|
66
|
+
@context
|
67
|
+
end
|
68
|
+
|
64
69
|
private
|
65
70
|
|
66
71
|
def split_path path
|
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
©
|
7
7
|
= "2014-#{Time.now.year}"
|
8
|
-
%a{href: "http://bmconseil.com/?ref=rftpd-
|
8
|
+
%a{href: "http://bmconseil.com/?ref=rftpd-dashboard", target: TARGET_BLANK} Bruno Medici Consultant
|
9
9
|
|
10
10
|
·
|
11
11
|
%a{href: "http://github.com/bmedici/rest-ftp-daemon/", target: TARGET_BLANK} GitHub
|
@@ -4,17 +4,15 @@ require 'singleton'
|
|
4
4
|
module RestFtpDaemon
|
5
5
|
class WorkerPool
|
6
6
|
include Singleton
|
7
|
-
|
8
7
|
include BmcDaemonLib::LoggerHelper
|
9
8
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
10
9
|
|
11
10
|
# Class options
|
12
|
-
attr_reader :logger
|
13
11
|
attr_reader :wid
|
14
12
|
|
15
13
|
def initialize
|
16
14
|
# Logger
|
17
|
-
|
15
|
+
log_pipe :workers
|
18
16
|
|
19
17
|
# Prepare status hash and vars
|
20
18
|
@statuses = {}
|
@@ -65,12 +63,6 @@ module RestFtpDaemon
|
|
65
63
|
@workers[wid] && @workers[wid].alive?
|
66
64
|
end
|
67
65
|
|
68
|
-
protected
|
69
|
-
|
70
|
-
def log_prefix
|
71
|
-
[nil, nil, nil]
|
72
|
-
end
|
73
|
-
|
74
66
|
private
|
75
67
|
|
76
68
|
def thread_variables thread
|
@@ -11,6 +11,7 @@ module RestFtpDaemon
|
|
11
11
|
@endpoints = Conf[:endpoints]
|
12
12
|
|
13
13
|
# Timeout and retry config
|
14
|
+
return "timeout disabled" if disabled?(@config[:timeout])
|
14
15
|
return "invalid timeout" unless @config[:timeout].to_i > 0
|
15
16
|
|
16
17
|
# Log that
|
@@ -24,7 +25,7 @@ module RestFtpDaemon
|
|
24
25
|
|
25
26
|
def worker_after
|
26
27
|
# Clean worker status
|
27
|
-
|
28
|
+
worker_set_jid nil
|
28
29
|
end
|
29
30
|
|
30
31
|
private
|
@@ -35,23 +36,24 @@ module RestFtpDaemon
|
|
35
36
|
job = RestFtpDaemon::JobQueue.instance.pop @pool
|
36
37
|
|
37
38
|
# Work on this job
|
38
|
-
|
39
|
+
job_process job
|
39
40
|
|
40
41
|
# Clean job status
|
41
42
|
job.wid = nil
|
42
43
|
#sleep 1
|
43
44
|
|
44
45
|
# Handle the retry if needed
|
45
|
-
|
46
|
+
job_result job
|
46
47
|
end
|
47
48
|
|
48
|
-
def
|
49
|
+
def job_process job
|
49
50
|
# Prepare job and worker for processing
|
50
|
-
|
51
|
+
worker_set_jid job.id
|
51
52
|
worker_status WORKER_STATUS_RUNNING, job
|
52
53
|
job.wid = Thread.current.thread_variable_get :wid
|
53
54
|
|
54
55
|
# Processs this job protected by a timeout
|
56
|
+
log_info "job_process: start working"
|
55
57
|
Timeout.timeout(@config[:timeout], RestFtpDaemon::JobTimeout) do
|
56
58
|
job.start
|
57
59
|
end
|
@@ -60,45 +62,45 @@ module RestFtpDaemon
|
|
60
62
|
RestFtpDaemon::Counters.instance.increment :jobs, :processed
|
61
63
|
|
62
64
|
rescue RestFtpDaemon::JobTimeout => ex
|
63
|
-
log_error "
|
65
|
+
log_error "job_process: TIMEOUT: started_at[#{job.started_at}] started_since[#{job.started_since}] #{ex.message}", ex.backtrace
|
64
66
|
worker_status WORKER_STATUS_TIMEOUT, job
|
65
67
|
|
66
68
|
# Inform the job
|
67
69
|
job.oops_end(:timeout, ex) unless job.nil?
|
68
70
|
|
69
71
|
rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::JobAttributeMissing, StandardError => ex
|
70
|
-
log_error "
|
72
|
+
log_error "job_process: CRASHED: ex[#{ex.class}] #{ex.message}", ex.backtrace
|
71
73
|
worker_status WORKER_STATUS_CRASHED
|
72
74
|
|
73
75
|
# Inform the job
|
74
76
|
job.oops_end(:crashed, ex) unless job.nil?
|
75
77
|
end
|
76
78
|
|
77
|
-
def
|
79
|
+
def job_result job
|
78
80
|
# If job status requires a retry, just restack it
|
79
81
|
if !job.error
|
80
82
|
# Processing successful
|
81
|
-
|
83
|
+
log_info "job_result: finished with success"
|
82
84
|
worker_status WORKER_STATUS_FINISHED, job
|
83
85
|
|
84
86
|
elsif error_not_eligible(job)
|
85
|
-
log_error "not retrying [#{job.error}] retry_on not eligible"
|
87
|
+
log_error "job_result: not retrying [#{job.error}] retry_on not eligible"
|
86
88
|
|
87
89
|
elsif error_reached_for(job)
|
88
|
-
log_error "not retrying [#{job.error}] retry_for reached [#{@config[:retry_for]}s]"
|
90
|
+
log_error "job_result: not retrying [#{job.error}] retry_for reached [#{@config[:retry_for]}s]"
|
89
91
|
|
90
92
|
elsif error_reached_max(job)
|
91
|
-
log_error "not retrying [#{job.error}] retry_max reached #{tentatives(job)}"
|
93
|
+
log_error "job_result: not retrying [#{job.error}] retry_max reached #{tentatives(job)}"
|
92
94
|
|
93
95
|
else
|
94
96
|
# Delay cannot be negative, and will be 1s minimum
|
95
97
|
retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
|
96
|
-
log_info "
|
98
|
+
log_info "job_result: retrying job [#{job.id}] in [#{retry_after}s] - tried #{tentatives(job)}"
|
97
99
|
|
98
100
|
# Wait !
|
99
101
|
worker_status WORKER_STATUS_RETRYING, job
|
100
102
|
sleep retry_after
|
101
|
-
log_debug "job [#{job.id}] requeued after [#{retry_after}s] delay"
|
103
|
+
log_debug "job_result: job [#{job.id}] requeued after [#{retry_after}s] delay"
|
102
104
|
|
103
105
|
# Now, requeue this job
|
104
106
|
RestFtpDaemon::JobQueue.instance.requeue job
|
@@ -129,6 +131,11 @@ module RestFtpDaemon
|
|
129
131
|
return job.tentatives >= @config[:retry_max]
|
130
132
|
end
|
131
133
|
|
134
|
+
def worker_set_jid jid
|
135
|
+
Thread.current.thread_variable_set :jid, jid
|
136
|
+
Thread.current.thread_variable_set :updated_at, Time.now
|
137
|
+
end
|
138
|
+
|
132
139
|
def tentatives job
|
133
140
|
"[#{job.tentatives}/#{@config[:retry_max]}]"
|
134
141
|
end
|
@@ -8,5 +8,19 @@ module RestFtpDaemon
|
|
8
8
|
add_transaction_tracer :worker_init, category: :task
|
9
9
|
add_transaction_tracer :worker_after, category: :task
|
10
10
|
add_transaction_tracer :worker_process, category: :task
|
11
|
+
|
12
|
+
protected
|
13
|
+
|
14
|
+
def log_context
|
15
|
+
{
|
16
|
+
wid: Thread.current.thread_variable_get(:wid),
|
17
|
+
jid: Thread.current.thread_variable_get(:jid),
|
18
|
+
}
|
19
|
+
end
|
20
|
+
|
21
|
+
def disabled? value
|
22
|
+
value.nil? || value === false || value == 0
|
23
|
+
end
|
24
|
+
|
11
25
|
end
|
12
26
|
end
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
|
4
4
|
# Project version
|
5
|
-
spec.version = "0.
|
5
|
+
spec.version = "0.435.0"
|
6
6
|
|
7
7
|
# Project description
|
8
8
|
spec.name = "rest-ftp-daemon"
|
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
|
|
32
32
|
spec.add_development_dependency "ruby-prof"
|
33
33
|
|
34
34
|
# Runtime dependencies
|
35
|
-
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.
|
35
|
+
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.4.1"
|
36
36
|
spec.add_runtime_dependency "json", "~> 1.8"
|
37
37
|
spec.add_runtime_dependency "thin", "~> 1.7"
|
38
38
|
spec.add_runtime_dependency "activesupport", "~> 4.2"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.435.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -114,14 +114,14 @@ dependencies:
|
|
114
114
|
requirements:
|
115
115
|
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.
|
117
|
+
version: 0.4.1
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.
|
124
|
+
version: 0.4.1
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: json
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|