rest-ftp-daemon 0.221.2 → 0.222.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +2 -0
- data/.rubocop.yml +859 -0
- data/Gemfile.lock +34 -3
- data/README.md +6 -1
- data/Rakefile +8 -1
- data/bin/rest-ftp-daemon +9 -9
- data/config.ru +3 -3
- data/lib/rest-ftp-daemon.rb +43 -43
- data/lib/rest-ftp-daemon/api/dashboard.rb +14 -4
- data/lib/rest-ftp-daemon/api/job_presenter.rb +1 -1
- data/lib/rest-ftp-daemon/api/jobs.rb +42 -42
- data/lib/rest-ftp-daemon/api/root.rb +37 -17
- data/lib/rest-ftp-daemon/constants.rb +8 -8
- data/lib/rest-ftp-daemon/helpers.rb +12 -13
- data/lib/rest-ftp-daemon/job.rb +24 -23
- data/lib/rest-ftp-daemon/job_queue.rb +8 -9
- data/lib/rest-ftp-daemon/logger.rb +2 -14
- data/lib/rest-ftp-daemon/logger_pool.rb +1 -1
- data/lib/rest-ftp-daemon/notification.rb +5 -5
- data/lib/rest-ftp-daemon/paginate.rb +4 -3
- data/lib/rest-ftp-daemon/settings.rb +11 -11
- data/lib/rest-ftp-daemon/uri.rb +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -1
- data/lib/rest-ftp-daemon/worker.rb +1 -1
- data/lib/rest-ftp-daemon/worker_conchita.rb +3 -3
- data/lib/rest-ftp-daemon/worker_job.rb +3 -3
- data/lib/rest-ftp-daemon/worker_pool.rb +6 -9
- data/rest-ftp-daemon.gemspec +3 -1
- data/rest-ftp-daemon.yml.sample +12 -11
- data/spec/rest-ftp-daemon/features/dashboard_spec.rb +23 -0
- data/spec/spec_helper.rb +62 -0
- data/spec/support/config.yml +26 -0
- metadata +35 -4
- data/lib/rest-ftp-daemon/api/debug.rb +0 -31
- data/lib/rest-ftp-daemon/api/status.rb +0 -33
@@ -12,11 +12,12 @@ module RestFtpDaemon
|
|
12
12
|
do_not_route_head!
|
13
13
|
do_not_route_options!
|
14
14
|
|
15
|
-
# FIXME
|
16
|
-
# add_swagger_documentation
|
17
|
-
# default_error_formatter :json
|
18
15
|
format :json
|
19
16
|
|
17
|
+
mount RestFtpDaemon::API::Jobs => "/jobs"
|
18
|
+
mount RestFtpDaemon::API::Dashbaord => "/"
|
19
|
+
|
20
|
+
|
20
21
|
|
21
22
|
####### INITIALIZATION
|
22
23
|
|
@@ -38,25 +39,44 @@ module RestFtpDaemon
|
|
38
39
|
Root.logger
|
39
40
|
end
|
40
41
|
|
41
|
-
|
42
|
-
# {}
|
43
|
-
# end
|
42
|
+
end
|
44
43
|
|
45
|
-
def api_error exception
|
46
|
-
{
|
47
|
-
:error => exception.message,
|
48
|
-
#:message => exception.backtrace.first,
|
49
|
-
}
|
50
|
-
end
|
51
44
|
|
52
|
-
|
53
|
-
template = File.read("#{APP_LIBS}/views/#{name.to_s}.haml")
|
54
|
-
haml_engine = Haml::Engine.new(template)
|
55
|
-
haml_engine.render(binding, values)
|
56
|
-
end
|
45
|
+
####### GET /routes
|
57
46
|
|
47
|
+
desc "show application routes"
|
48
|
+
get "/routes" do
|
49
|
+
log_info "GET /routes"
|
50
|
+
status 200
|
51
|
+
return RestFtpDaemon::API::Root.routes
|
58
52
|
end
|
59
53
|
|
54
|
+
|
55
|
+
####### GET /status
|
56
|
+
|
57
|
+
# Server global status
|
58
|
+
get "/status" do
|
59
|
+
log_info "GET /status"
|
60
|
+
mem = GetProcessMem.new
|
61
|
+
|
62
|
+
status 200
|
63
|
+
return {
|
64
|
+
hostname: `hostname`.chomp,
|
65
|
+
version: APP_VER,
|
66
|
+
started: APP_STARTED,
|
67
|
+
uptime: (Time.now - APP_STARTED).round(1),
|
68
|
+
counters: $queue.counters,
|
69
|
+
memory_bytes: mem.bytes.to_i,
|
70
|
+
memory_mb: mem.mb.round(0),
|
71
|
+
status: $queue.counts_by_status,
|
72
|
+
workers: $pool.worker_variables,
|
73
|
+
jobs_count: $queue.jobs_count,
|
74
|
+
jobs_queued: $queue.queued_ids,
|
75
|
+
config: Helpers.get_censored_config
|
76
|
+
}
|
77
|
+
end
|
78
|
+
|
79
|
+
|
60
80
|
end
|
61
81
|
end
|
62
82
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# Terrific constants
|
2
2
|
APP_NAME = "rest-ftp-daemon"
|
3
3
|
APP_NICK = "rftpd"
|
4
|
-
APP_VER = "0.
|
4
|
+
APP_VER = "0.222.0"
|
5
5
|
|
6
6
|
|
7
7
|
# Jobs and workers
|
@@ -21,8 +21,8 @@ LOG_COL_WID = 8
|
|
21
21
|
LOG_COL_JID = JOB_IDENT_LEN+3+2
|
22
22
|
LOG_COL_ID = 6
|
23
23
|
LOG_TRIM_LINE = 80
|
24
|
-
LOG_DUMPS = File.dirname(__FILE__) +
|
25
|
-
LOG_ROTATION =
|
24
|
+
LOG_DUMPS = File.dirname(__FILE__) + "/../../log/"
|
25
|
+
LOG_ROTATION = "daily"
|
26
26
|
LOG_FORMAT_TIME = "%Y-%m-%d %H:%M:%S"
|
27
27
|
LOG_FORMAT_PREFIX = "%s %s\t%-#{LOG_PIPE_LEN.to_i}s\t"
|
28
28
|
LOG_FORMAT_MESSAGE = "%#{-LOG_COL_WID.to_i}s\t%#{-LOG_COL_JID.to_i}s\t%#{-LOG_COL_ID.to_i}s"
|
@@ -43,11 +43,11 @@ JOB_STYLES = {
|
|
43
43
|
JOB_STATUS_RENAMING => :info,
|
44
44
|
}
|
45
45
|
WORKER_STYLES = {
|
46
|
-
:
|
47
|
-
:
|
48
|
-
:
|
49
|
-
:
|
50
|
-
:
|
46
|
+
waiting: :success,
|
47
|
+
working: :info,
|
48
|
+
crashed: :danger,
|
49
|
+
done: :success,
|
50
|
+
dead: :danger
|
51
51
|
}
|
52
52
|
PAGINATE_MAX = 30
|
53
53
|
|
@@ -16,12 +16,13 @@ module RestFtpDaemon
|
|
16
16
|
converted = number.to_f / ( 1024 ** index )
|
17
17
|
|
18
18
|
truncated = converted.round(decimals)
|
19
|
+
|
19
20
|
"#{truncated} #{units[index]}#{unit}"
|
20
21
|
end
|
21
22
|
|
22
23
|
def self.text_or_empty text
|
23
24
|
return "-" if text.nil? || text.empty?
|
24
|
-
|
25
|
+
|
25
26
|
text
|
26
27
|
end
|
27
28
|
|
@@ -37,10 +38,6 @@ module RestFtpDaemon
|
|
37
38
|
path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
|
38
39
|
end
|
39
40
|
|
40
|
-
# def self.hide_password url
|
41
|
-
# path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
|
42
|
-
# end
|
43
|
-
|
44
41
|
def self.extract_filename path
|
45
42
|
# match everything that's after a slash at the end of the string
|
46
43
|
m = path.match /\/?([^\/]+)$/
|
@@ -60,10 +57,10 @@ module RestFtpDaemon
|
|
60
57
|
end
|
61
58
|
|
62
59
|
def self.local_port_used? port
|
63
|
-
ip =
|
60
|
+
ip = "127.0.0.1"
|
64
61
|
timeout = 1
|
65
62
|
begin
|
66
|
-
Timeout
|
63
|
+
Timeout.timeout(timeout) do
|
67
64
|
begin
|
68
65
|
TCPSocket.new(ip, port).close
|
69
66
|
true
|
@@ -82,13 +79,13 @@ module RestFtpDaemon
|
|
82
79
|
return if method.nil?
|
83
80
|
klass = case method
|
84
81
|
when :file
|
85
|
-
|
82
|
+
"label-primary"
|
86
83
|
when :ftp
|
87
|
-
|
84
|
+
"label-warning"
|
88
85
|
when :ftps, :ftpes
|
89
|
-
|
86
|
+
"label-success"
|
90
87
|
else
|
91
|
-
|
88
|
+
"label-default"
|
92
89
|
end
|
93
90
|
"<div class=\"transfer-method label #{klass}\">#{method.upcase}</div>"
|
94
91
|
end
|
@@ -96,7 +93,8 @@ module RestFtpDaemon
|
|
96
93
|
# Dates and times: date with time generator
|
97
94
|
def self.datetime_full datetime
|
98
95
|
return "-" if datetime.nil?
|
99
|
-
|
96
|
+
|
97
|
+
datetime.to_datetime.strftime("%d.%m.%Y %H:%M:%S")
|
100
98
|
end
|
101
99
|
|
102
100
|
def self.datetime_short datetime
|
@@ -104,7 +102,8 @@ module RestFtpDaemon
|
|
104
102
|
return "-" if datetime.nil?
|
105
103
|
return "?" unless datetime.respond_to? :to_date
|
106
104
|
return datetime.to_datetime.strftime("%H:%M:%S") if datetime.to_date == Time.now.to_date
|
107
|
-
|
105
|
+
|
106
|
+
datetime.to_datetime.strftime("%d/%m %H:%M:%S")
|
108
107
|
end
|
109
108
|
|
110
109
|
def self.hide_credentials_from_url url
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -48,7 +48,7 @@ module RestFtpDaemon
|
|
48
48
|
|
49
49
|
# Import query params
|
50
50
|
FIELDS.each do |name|
|
51
|
-
instance_variable_set "@#{name
|
51
|
+
instance_variable_set "@#{name}", params[name]
|
52
52
|
end
|
53
53
|
|
54
54
|
# Set super-default flags
|
@@ -106,7 +106,7 @@ module RestFtpDaemon
|
|
106
106
|
# rescue RestFtpDaemon::RestFtpDaemonException => exception
|
107
107
|
# return oops :started, exception, :prepare_failed, true
|
108
108
|
|
109
|
-
# rescue
|
109
|
+
# rescue StandardError => exception
|
110
110
|
# return oops :started, exception, :prepare_unhandled, true
|
111
111
|
|
112
112
|
else
|
@@ -182,7 +182,7 @@ module RestFtpDaemon
|
|
182
182
|
# rescue RestFtpDaemon::RestFtpDaemonException => exception
|
183
183
|
# return oops :ended, exception, :transfer_failed, true
|
184
184
|
|
185
|
-
# rescue
|
185
|
+
# rescue StandardError => exception
|
186
186
|
# return oops :ended, exception, :transfer_unhandled, true
|
187
187
|
|
188
188
|
else
|
@@ -216,11 +216,11 @@ module RestFtpDaemon
|
|
216
216
|
end
|
217
217
|
|
218
218
|
def oops_after_crash exception
|
219
|
-
|
219
|
+
oops :ended, exception, :crashed
|
220
220
|
end
|
221
221
|
|
222
222
|
def oops_you_stop_now exception
|
223
|
-
|
223
|
+
oops :ended, exception, :timeout
|
224
224
|
end
|
225
225
|
|
226
226
|
protected
|
@@ -243,7 +243,7 @@ module RestFtpDaemon
|
|
243
243
|
end
|
244
244
|
|
245
245
|
def expand_url path
|
246
|
-
URI
|
246
|
+
URI.parse replace_tokens(path)
|
247
247
|
end
|
248
248
|
|
249
249
|
def contains_brackets(item)
|
@@ -256,7 +256,7 @@ module RestFtpDaemon
|
|
256
256
|
vectors = Settings.endpoints.clone
|
257
257
|
|
258
258
|
# Stack RANDOM into tokens
|
259
|
-
vectors[
|
259
|
+
vectors["RANDOM"] = SecureRandom.hex(JOB_RANDOM_LEN)
|
260
260
|
|
261
261
|
# Replace endpoints defined in config
|
262
262
|
newpath = path.clone
|
@@ -269,7 +269,7 @@ module RestFtpDaemon
|
|
269
269
|
raise RestFtpDaemon::JobUnresolvedTokens if contains_brackets newpath
|
270
270
|
|
271
271
|
# All OK, return this URL stripping multiple slashes
|
272
|
-
|
272
|
+
newpath.gsub(/([^:])\/\//, '\1/')
|
273
273
|
end
|
274
274
|
|
275
275
|
def prepare
|
@@ -293,11 +293,11 @@ module RestFtpDaemon
|
|
293
293
|
@target_url = expand_url @target
|
294
294
|
set :target_url, @target_url.to_s
|
295
295
|
|
296
|
-
if @target_url.
|
296
|
+
if @target_url.is_a? URI::FTP
|
297
297
|
@target_method = :ftp
|
298
|
-
elsif @target_url.
|
298
|
+
elsif @target_url.is_a? URI::FTPES
|
299
299
|
@target_method = :ftps
|
300
|
-
elsif @target_url.
|
300
|
+
elsif @target_url.is_a? URI::FTPS
|
301
301
|
@target_method = :ftps
|
302
302
|
end
|
303
303
|
set :target_method, @target_method
|
@@ -309,7 +309,7 @@ module RestFtpDaemon
|
|
309
309
|
|
310
310
|
def transfer
|
311
311
|
# Update job status
|
312
|
-
newstatus
|
312
|
+
newstatus :checking_source
|
313
313
|
@started_at = Time.now
|
314
314
|
|
315
315
|
# Method assertions and init
|
@@ -390,7 +390,7 @@ module RestFtpDaemon
|
|
390
390
|
|
391
391
|
def flag_default name, default
|
392
392
|
# build the flag instance var name
|
393
|
-
variable = "@#{name
|
393
|
+
variable = "@#{name}"
|
394
394
|
|
395
395
|
# If it's true or false, that's ok
|
396
396
|
return if [true, false].include? instance_variable_get(variable)
|
@@ -413,7 +413,7 @@ module RestFtpDaemon
|
|
413
413
|
@ftp = Net::FTP.new
|
414
414
|
when :ftps
|
415
415
|
@ftp = DoubleBagFTPS.new
|
416
|
-
@ftp.ssl_context = DoubleBagFTPS.create_ssl_context(:
|
416
|
+
@ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
|
417
417
|
@ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
|
418
418
|
else
|
419
419
|
log_info "Job.ftp_init unknown scheme [#{@target_url.scheme}]"
|
@@ -513,7 +513,7 @@ module RestFtpDaemon
|
|
513
513
|
# Try to chdir in this directory
|
514
514
|
@ftp.chdir(path)
|
515
515
|
|
516
|
-
rescue Net::FTPPermError
|
516
|
+
rescue Net::FTPPermError
|
517
517
|
# If not possible because the directory is missing
|
518
518
|
parent = Helpers.extract_parent(path)
|
519
519
|
log_info "#{pref} chdir failed - parent [#{parent}]"
|
@@ -553,7 +553,8 @@ module RestFtpDaemon
|
|
553
553
|
|
554
554
|
# Result can be nil or a list of files
|
555
555
|
return false if results.nil?
|
556
|
-
|
556
|
+
|
557
|
+
results.count >0
|
557
558
|
end
|
558
559
|
|
559
560
|
def ftp_transfer source_filename, target_name = nil
|
@@ -668,7 +669,7 @@ module RestFtpDaemon
|
|
668
669
|
payload[:event] = event
|
669
670
|
RestFtpDaemon::Notification.new @notify, payload
|
670
671
|
|
671
|
-
rescue
|
672
|
+
rescue StandardError => ex
|
672
673
|
log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
|
673
674
|
end
|
674
675
|
|
@@ -684,7 +685,7 @@ module RestFtpDaemon
|
|
684
685
|
# Log this error
|
685
686
|
error = exception.class if error.nil?
|
686
687
|
|
687
|
-
message = "Job.oops event[#{event
|
688
|
+
message = "Job.oops event[#{event}] error[#{error}] ex[#{exception.class}] #{exception.message}"
|
688
689
|
if include_backtrace
|
689
690
|
log_error message, exception.backtrace
|
690
691
|
else
|
@@ -715,14 +716,14 @@ module RestFtpDaemon
|
|
715
716
|
|
716
717
|
# Prepare notification if signal given
|
717
718
|
return unless event
|
718
|
-
client_notify event, error: error, status: notif_status, message: "#{exception.class
|
719
|
+
client_notify event, error: error, status: notif_status, message: "#{exception.class} | #{exception.message}"
|
719
720
|
end
|
720
721
|
|
721
722
|
if Settings.newrelic_enabled?
|
722
|
-
add_transaction_tracer :prepare,
|
723
|
-
add_transaction_tracer :transfer,
|
724
|
-
add_transaction_tracer :client_notify,
|
725
|
-
add_transaction_tracer :initialize,
|
723
|
+
add_transaction_tracer :prepare, category: :task
|
724
|
+
add_transaction_tracer :transfer, category: :task
|
725
|
+
add_transaction_tracer :client_notify, category: :task
|
726
|
+
add_transaction_tracer :initialize, category: :task
|
726
727
|
end
|
727
728
|
|
728
729
|
end
|
@@ -6,7 +6,6 @@ module RestFtpDaemon
|
|
6
6
|
attr_reader :queue
|
7
7
|
attr_reader :jobs
|
8
8
|
|
9
|
-
|
10
9
|
if Settings.newrelic_enabled?
|
11
10
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
12
11
|
end
|
@@ -18,7 +17,7 @@ module RestFtpDaemon
|
|
18
17
|
@waiting = []
|
19
18
|
@queue.taint # enable tainted communication
|
20
19
|
@waiting.taint
|
21
|
-
|
20
|
+
taint
|
22
21
|
@mutex = Mutex.new
|
23
22
|
|
24
23
|
# Logger
|
@@ -186,7 +185,7 @@ module RestFtpDaemon
|
|
186
185
|
|
187
186
|
# Ok, we have to clean it up ..
|
188
187
|
log_info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
|
189
|
-
log_info "
|
188
|
+
log_info " & unqueued" if @queue.delete(job)
|
190
189
|
|
191
190
|
true
|
192
191
|
end
|
@@ -208,12 +207,12 @@ module RestFtpDaemon
|
|
208
207
|
end
|
209
208
|
|
210
209
|
if Settings.newrelic_enabled?
|
211
|
-
add_transaction_tracer :push,
|
212
|
-
add_transaction_tracer :pop,
|
213
|
-
add_transaction_tracer :sort_queue!,
|
214
|
-
add_transaction_tracer :expire,
|
215
|
-
add_transaction_tracer :counts_by_status, :
|
216
|
-
add_transaction_tracer :filter_jobs,
|
210
|
+
add_transaction_tracer :push, category: :task
|
211
|
+
add_transaction_tracer :pop, category: :task
|
212
|
+
add_transaction_tracer :sort_queue!, category: :task
|
213
|
+
add_transaction_tracer :expire, category: :task
|
214
|
+
add_transaction_tracer :counts_by_status, category: :task
|
215
|
+
add_transaction_tracer :filter_jobs, category: :task
|
217
216
|
end
|
218
217
|
|
219
218
|
end
|
@@ -7,20 +7,8 @@ class Logger
|
|
7
7
|
|
8
8
|
# Build prefixes depending on this context
|
9
9
|
prefix1 = build_prefix(context)
|
10
|
-
prefix2 = build_prefix() +
|
10
|
+
prefix2 = build_prefix() + " | "
|
11
11
|
|
12
|
-
# # Build output lines
|
13
|
-
# output = []
|
14
|
-
# output << prefix1 + message.strip
|
15
|
-
|
16
|
-
# # Add optional lines
|
17
|
-
# context[:lines].each do |line|
|
18
|
-
# # line.strip!
|
19
|
-
# # next if line.empty?
|
20
|
-
# output << prefix2 + line[0..LOG_TRIM_LINE]
|
21
|
-
# end if context[:lines].is_a? Enumerable
|
22
|
-
|
23
|
-
# Use "context[:lines]" according to its type
|
24
12
|
lines = context[:lines]
|
25
13
|
|
26
14
|
if lines.is_a? Hash
|
@@ -47,7 +35,7 @@ class Logger
|
|
47
35
|
context[:wid].to_s,
|
48
36
|
context[:jid].to_s,
|
49
37
|
context[:id].to_s,
|
50
|
-
context[:level].to_i+1
|
38
|
+
context[:level].to_i+1
|
51
39
|
]
|
52
40
|
end
|
53
41
|
|
@@ -31,7 +31,7 @@ module RestFtpDaemon
|
|
31
31
|
# Build body and extract job ID if provided
|
32
32
|
body = {
|
33
33
|
id: params[:id].to_s,
|
34
|
-
signal: "#{NOTIFY_PREFIX}.#{params[:event]
|
34
|
+
signal: "#{NOTIFY_PREFIX}.#{params[:event]}",
|
35
35
|
error: params[:error],
|
36
36
|
host: Settings.host.to_s,
|
37
37
|
}
|
@@ -42,13 +42,13 @@ module RestFtpDaemon
|
|
42
42
|
|
43
43
|
|
44
44
|
# Send message in a thread
|
45
|
-
Thread.new do
|
45
|
+
Thread.new do
|
46
46
|
# Prepare query
|
47
47
|
uri = URI(url)
|
48
48
|
headers = {
|
49
|
-
|
50
|
-
|
51
|
-
|
49
|
+
"Content-Type" => "application/json",
|
50
|
+
"Accept" => "application/json",
|
51
|
+
"User-Agent" => "#{APP_NAME} - #{APP_VER}"
|
52
52
|
}
|
53
53
|
data = body.to_json
|
54
54
|
log_info "sending #{data}"
|