rest-ftp-daemon 0.250.5 → 0.300.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +19 -14
- data/README.md +12 -3
- data/bin/rest-ftp-daemon +102 -96
- data/config.ru +5 -5
- data/defaults.yml +61 -0
- data/lib/rest-ftp-daemon.rb +10 -4
- data/lib/rest-ftp-daemon/api/config.rb +3 -2
- data/lib/rest-ftp-daemon/api/dashboard.rb +1 -4
- data/lib/rest-ftp-daemon/api/debug.rb +30 -17
- data/lib/rest-ftp-daemon/api/job_presenter.rb +0 -2
- data/lib/rest-ftp-daemon/api/jobs.rb +4 -3
- data/lib/rest-ftp-daemon/api/root.rb +7 -10
- data/lib/rest-ftp-daemon/api/status.rb +7 -13
- data/lib/rest-ftp-daemon/constants.rb +27 -45
- data/lib/rest-ftp-daemon/counters.rb +0 -4
- data/lib/rest-ftp-daemon/helpers.rb +3 -18
- data/lib/rest-ftp-daemon/job.rb +16 -21
- data/lib/rest-ftp-daemon/job_queue.rb +21 -14
- data/lib/rest-ftp-daemon/launcher.rb +26 -0
- data/lib/rest-ftp-daemon/logger_pool.rb +9 -19
- data/lib/rest-ftp-daemon/metrics.rb +41 -0
- data/lib/rest-ftp-daemon/notification.rb +7 -10
- data/lib/rest-ftp-daemon/remote.rb +4 -4
- data/lib/rest-ftp-daemon/remote_ftp.rb +10 -10
- data/lib/rest-ftp-daemon/remote_sftp.rb +13 -24
- data/lib/rest-ftp-daemon/views/dashboard.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_footer.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_header.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +2 -2
- data/lib/rest-ftp-daemon/worker.rb +43 -12
- data/lib/rest-ftp-daemon/worker_conchita.rb +15 -28
- data/lib/rest-ftp-daemon/worker_job.rb +30 -21
- data/lib/rest-ftp-daemon/worker_pool.rb +59 -50
- data/lib/rest-ftp-daemon/worker_reporter.rb +70 -0
- data/lib/shared/conf.rb +195 -0
- data/lib/shared/logger_formatter.rb +31 -0
- data/lib/shared/logger_helper.rb +78 -0
- data/rest-ftp-daemon.gemspec +23 -22
- data/{rest-ftp-daemon.yml.sample → rest-ftp-daemon.sample.yml} +10 -7
- data/spec/spec_helper.rb +1 -1
- metadata +30 -12
- data/lib/rest-ftp-daemon/logger.rb +0 -57
- data/lib/rest-ftp-daemon/logger_helper.rb +0 -36
- data/lib/rest-ftp-daemon/settings.rb +0 -57
@@ -2,16 +2,10 @@ module RestFtpDaemon
|
|
2
2
|
|
3
3
|
# Queue that stores all the Jobs waiting to be processed or fully processed
|
4
4
|
class JobQueue
|
5
|
-
include LoggerHelper
|
5
|
+
include Shared::LoggerHelper
|
6
6
|
attr_reader :logger
|
7
|
-
|
8
|
-
#attr_reader :queues
|
9
7
|
attr_reader :jobs
|
10
8
|
|
11
|
-
if Settings.newrelic_enabled?
|
12
|
-
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
13
|
-
end
|
14
|
-
|
15
9
|
def initialize
|
16
10
|
# Instance variables
|
17
11
|
@queues = {}
|
@@ -45,10 +39,18 @@ module RestFtpDaemon
|
|
45
39
|
@queues
|
46
40
|
end
|
47
41
|
|
42
|
+
def queued_by_pool
|
43
|
+
result = {}
|
44
|
+
@queues.each do |pool, jobs|
|
45
|
+
result[pool] = jobs.count
|
46
|
+
end
|
47
|
+
result
|
48
|
+
end
|
49
|
+
|
48
50
|
def rate_by method_name
|
49
51
|
# Init
|
50
52
|
result = {}
|
51
|
-
return unless Job.new(0, {}).respond_to? method_name
|
53
|
+
#return unless Job.new(0, {}).respond_to? method_name
|
52
54
|
|
53
55
|
# Select only running jobs
|
54
56
|
@jobs.each do |job|
|
@@ -70,9 +72,7 @@ module RestFtpDaemon
|
|
70
72
|
|
71
73
|
# Add its current rate
|
72
74
|
result[group] ||= 0
|
73
|
-
#log_info " 2: #{result.inspect} (rate: #{rate})"
|
74
75
|
result[group] += rate
|
75
|
-
#log_info " 3: #{result.inspect}"
|
76
76
|
end
|
77
77
|
|
78
78
|
# Return the rate
|
@@ -109,7 +109,6 @@ module RestFtpDaemon
|
|
109
109
|
log_info "find_by_id (#{id}, #{prefixed}) > #{id}"
|
110
110
|
|
111
111
|
# Search in jobs queues
|
112
|
-
#@jobs.reverse.find { |item| item.id == id }
|
113
112
|
@jobs.find { |item| item.id == id }
|
114
113
|
end
|
115
114
|
|
@@ -201,7 +200,7 @@ module RestFtpDaemon
|
|
201
200
|
@mutex.synchronize do
|
202
201
|
# Delete jobs from the queue when they match status and age limits
|
203
202
|
@jobs.delete_if do |job|
|
204
|
-
#
|
203
|
+
# log_debug "testing job [#{job.id}] updated_at [#{job.updated_at}]"
|
205
204
|
|
206
205
|
# Skip if wrong status, updated_at invalid, or updated since time_limit
|
207
206
|
next unless job.status == status
|
@@ -213,7 +212,7 @@ module RestFtpDaemon
|
|
213
212
|
|
214
213
|
# From any queues, remove it
|
215
214
|
@queues.each do |pool, jobs|
|
216
|
-
|
215
|
+
log_debug "#{LOG_INDENT}unqueued from [#{pool}]" if jobs.delete(job)
|
217
216
|
end
|
218
217
|
|
219
218
|
# Remember we have to delete the original job !
|
@@ -225,11 +224,19 @@ module RestFtpDaemon
|
|
225
224
|
|
226
225
|
protected
|
227
226
|
|
227
|
+
def log_prefix
|
228
|
+
[nil, nil, nil]
|
229
|
+
end
|
230
|
+
|
228
231
|
def prefixed_id id
|
229
232
|
"#{@prefix}.#{id}"
|
230
233
|
end
|
231
234
|
|
232
|
-
|
235
|
+
private
|
236
|
+
|
237
|
+
# NewRelic instrumentation
|
238
|
+
if Conf.newrelic_enabled?
|
239
|
+
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
233
240
|
add_transaction_tracer :push, category: :task
|
234
241
|
add_transaction_tracer :pop, category: :task
|
235
242
|
add_transaction_tracer :expire, category: :task
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
class Launcher
|
3
|
+
LAUNCHER_PORT_TIMEOUT = 3
|
4
|
+
LAUNCHER_PORT_LOCALHOST = "127.0.0.1"
|
5
|
+
|
6
|
+
class << self
|
7
|
+
|
8
|
+
def local_port_used? port
|
9
|
+
Timeout.timeout(LAUNCHER_PORT_TIMEOUT) do
|
10
|
+
begin
|
11
|
+
TCPSocket.new(LAUNCHER_PORT_LOCALHOST, port).close
|
12
|
+
true
|
13
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
|
14
|
+
false
|
15
|
+
rescue Errno::EADDRNOTAVAIL
|
16
|
+
"local_port_used: Errno::EADDRNOTAVAIL"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
rescue Timeout::Error
|
20
|
+
false
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -16,29 +16,19 @@ module RestFtpDaemon
|
|
16
16
|
|
17
17
|
def create pipe
|
18
18
|
# Compute file path / STDERR
|
19
|
-
logfile =
|
19
|
+
logfile = Conf[:logs][pipe] if Conf[:logs].is_a? Hash
|
20
20
|
logfile ||= STDERR
|
21
21
|
|
22
|
+
# Check if we can write to than file
|
23
|
+
unless File.writable?(logfile)
|
24
|
+
puts "LoggerPool [#{pipe}] logging to hyperspace (cannot write to #{logfile})"
|
25
|
+
logfile = nil
|
26
|
+
end
|
27
|
+
|
22
28
|
# Create the logger and return it
|
23
29
|
logger = Logger.new(logfile, LOG_ROTATION) #, 10, 1024000)
|
24
|
-
logger.progname = pipe.to_s.
|
25
|
-
|
26
|
-
# And the formatter
|
27
|
-
logger.formatter = proc do |severity, datetime, progname, messages|
|
28
|
-
# Build common line prefix
|
29
|
-
prefix = LOG_FORMAT_PREFIX % [
|
30
|
-
datetime.strftime(LOG_FORMAT_TIME),
|
31
|
-
severity,
|
32
|
-
progname,
|
33
|
-
]
|
34
|
-
|
35
|
-
# If we have a bunch of lines, prefix them and send them together
|
36
|
-
if messages.is_a? Array
|
37
|
-
messages.map { |line| prefix + line + LOG_NEWLINE}.join
|
38
|
-
else
|
39
|
-
prefix + messages.to_s + LOG_NEWLINE
|
40
|
-
end
|
41
|
-
end
|
30
|
+
logger.progname = pipe.to_s.downcase
|
31
|
+
logger.formatter = Shared::LoggerFormatter
|
42
32
|
|
43
33
|
# Finally return this logger
|
44
34
|
logger
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
class Metrics
|
3
|
+
|
4
|
+
def self.sample
|
5
|
+
# Prepare external deps
|
6
|
+
mem = GetProcessMem.new
|
7
|
+
|
8
|
+
# Build final value
|
9
|
+
return {
|
10
|
+
system: {
|
11
|
+
uptime: (Time.now - Conf.app_started).round(1),
|
12
|
+
memory: mem.bytes.to_i,
|
13
|
+
threads: Thread.list.count,
|
14
|
+
},
|
15
|
+
jobs_by_status: $queue.jobs_by_status,
|
16
|
+
rate_by_pool: $queue.rate_by(:pool),
|
17
|
+
rate_by_targethost: $queue.rate_by(:targethost),
|
18
|
+
queued_by_pool: $queue.queued_by_pool,
|
19
|
+
workers_by_status: self.workers_count_by_status,
|
20
|
+
}
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
# Collect: workers by status
|
26
|
+
def self.workers_count_by_status
|
27
|
+
# Init
|
28
|
+
counts = {}
|
29
|
+
|
30
|
+
$pool.worker_variables.group_by do |wid, vars|
|
31
|
+
vars[:status]
|
32
|
+
end.each do |status, workers|
|
33
|
+
counts[status] = workers.count
|
34
|
+
end
|
35
|
+
|
36
|
+
# Return count
|
37
|
+
counts
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -2,7 +2,7 @@ module RestFtpDaemon
|
|
2
2
|
|
3
3
|
# Handles a notification POST using a dedicated thread
|
4
4
|
class Notification
|
5
|
-
include LoggerHelper
|
5
|
+
include Shared::LoggerHelper
|
6
6
|
attr_reader :logger
|
7
7
|
|
8
8
|
attr_accessor :job_id
|
@@ -35,10 +35,10 @@ module RestFtpDaemon
|
|
35
35
|
def process
|
36
36
|
# Check context
|
37
37
|
if @url.nil?
|
38
|
-
|
38
|
+
log_error "skipping (missing url)", params
|
39
39
|
return
|
40
40
|
elsif @params[:event].nil?
|
41
|
-
|
41
|
+
log_error "skipping (missing event)", params
|
42
42
|
return
|
43
43
|
end
|
44
44
|
|
@@ -47,7 +47,7 @@ module RestFtpDaemon
|
|
47
47
|
id: @params[:id].to_s,
|
48
48
|
signal: "#{NOTIFY_PREFIX}.#{@params[:event]}",
|
49
49
|
error: @params[:error],
|
50
|
-
host:
|
50
|
+
host: Conf.host.to_s,
|
51
51
|
}
|
52
52
|
flags[:status] = @params[:status] if @params[:status].is_a? Enumerable
|
53
53
|
flags[:message] = @params[:message].to_s unless @params[:message].nil?
|
@@ -64,7 +64,7 @@ module RestFtpDaemon
|
|
64
64
|
headers = {
|
65
65
|
"Content-Type" => "application/json",
|
66
66
|
"Accept" => "application/json",
|
67
|
-
"User-Agent" =>
|
67
|
+
"User-Agent" => "#{Conf.app_name}/v#{Conf.app_ver}"
|
68
68
|
}
|
69
69
|
data = flags.to_json
|
70
70
|
|
@@ -96,11 +96,8 @@ module RestFtpDaemon
|
|
96
96
|
log_error "UNHANDLED EXCEPTION: #{ex.inspect}"
|
97
97
|
end
|
98
98
|
|
99
|
-
def
|
100
|
-
|
101
|
-
id: @id,
|
102
|
-
jid: @jid
|
103
|
-
}
|
99
|
+
def log_prefix
|
100
|
+
[nil, @jid, @id]
|
104
101
|
end
|
105
102
|
|
106
103
|
end
|
@@ -2,13 +2,13 @@ module RestFtpDaemon
|
|
2
2
|
|
3
3
|
# Handles transfers for Job class
|
4
4
|
class Remote
|
5
|
-
include LoggerHelper
|
5
|
+
include Shared::LoggerHelper
|
6
6
|
attr_reader :logger
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :log_prefix
|
8
8
|
|
9
|
-
def initialize url,
|
9
|
+
def initialize url, log_prefix, options = {}
|
10
10
|
# Logger
|
11
|
-
@
|
11
|
+
@log_prefix = log_prefix || {}
|
12
12
|
@logger = RestFtpDaemon::LoggerPool.instance.get :jobs
|
13
13
|
|
14
14
|
# Extract URL parts
|
@@ -6,12 +6,12 @@ module RestFtpDaemon
|
|
6
6
|
class RemoteFTP < Remote
|
7
7
|
attr_reader :ftp
|
8
8
|
|
9
|
-
def initialize url,
|
9
|
+
def initialize url, log_prefix, options = {}
|
10
10
|
# Call super
|
11
11
|
super
|
12
12
|
|
13
13
|
# Use debug ?
|
14
|
-
@debug = (
|
14
|
+
@debug = (Conf.at :debug, :ftp) == true
|
15
15
|
|
16
16
|
# Create FTP object
|
17
17
|
if options[:ftpes]
|
@@ -26,7 +26,7 @@ module RestFtpDaemon
|
|
26
26
|
@chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
|
27
27
|
|
28
28
|
# Announce object
|
29
|
-
|
29
|
+
log_debug "RemoteFTP.initialize chunk_size:#{@chunk_size}"
|
30
30
|
end
|
31
31
|
|
32
32
|
def connect
|
@@ -40,7 +40,7 @@ module RestFtpDaemon
|
|
40
40
|
|
41
41
|
def present? target
|
42
42
|
size = @ftp.size target.full
|
43
|
-
|
43
|
+
log_debug "RemoteFTP.present? [#{target.name}]"
|
44
44
|
|
45
45
|
rescue Net::FTPPermError
|
46
46
|
return false
|
@@ -49,16 +49,16 @@ module RestFtpDaemon
|
|
49
49
|
end
|
50
50
|
|
51
51
|
def remove! target
|
52
|
-
|
52
|
+
log_debug "RemoteFTP.remove! [#{target.name}]"
|
53
53
|
@ftp.delete target.full
|
54
54
|
rescue Net::FTPPermError
|
55
|
-
|
55
|
+
log_debug "#{LOG_INDENT}[#{target.name}] file not found"
|
56
56
|
else
|
57
|
-
|
57
|
+
log_debug "#{LOG_INDENT}[#{target.name}] removed"
|
58
58
|
end
|
59
59
|
|
60
60
|
def mkdir directory
|
61
|
-
|
61
|
+
log_debug "RemoteFTP.mkdir [#{directory}]"
|
62
62
|
@ftp.mkdir directory
|
63
63
|
|
64
64
|
rescue
|
@@ -67,7 +67,7 @@ module RestFtpDaemon
|
|
67
67
|
|
68
68
|
def chdir_or_create directory, mkdir = false
|
69
69
|
# Init, extract my parent name and my own name
|
70
|
-
|
70
|
+
log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
71
71
|
parent, _current = Helpers.extract_parent(directory)
|
72
72
|
|
73
73
|
# Access this directory
|
@@ -98,7 +98,7 @@ module RestFtpDaemon
|
|
98
98
|
destination.name = tempname if tempname
|
99
99
|
|
100
100
|
# Do the transfer
|
101
|
-
|
101
|
+
log_debug "RemoteFTP.push to [#{destination.name}]"
|
102
102
|
|
103
103
|
@ftp.putbinaryfile source.full, target.name, @chunk_size do |data|
|
104
104
|
# Update job status after this block transfer
|
@@ -5,21 +5,21 @@ module RestFtpDaemon
|
|
5
5
|
class RemoteSFTP < Remote
|
6
6
|
attr_reader :sftp
|
7
7
|
|
8
|
-
def initialize url,
|
8
|
+
def initialize url, log_prefix, options = {}
|
9
9
|
# Call super
|
10
10
|
super
|
11
11
|
|
12
12
|
# Use debug ?
|
13
|
-
@debug = (
|
13
|
+
@debug = (Conf.at :debug, :sftp) == true
|
14
14
|
|
15
15
|
# Announce object
|
16
|
-
|
16
|
+
log_debug "RemoteSFTP.initialize"
|
17
17
|
end
|
18
18
|
|
19
19
|
def connect
|
20
20
|
# Connect init
|
21
21
|
super
|
22
|
-
|
22
|
+
log_debug "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
|
23
23
|
|
24
24
|
# Debug level
|
25
25
|
verbosity = @debug ? Logger::INFO : false
|
@@ -35,7 +35,7 @@ module RestFtpDaemon
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def present? target
|
38
|
-
|
38
|
+
log_debug "RemoteSFTP.present? [#{target.name}]"
|
39
39
|
stat = @sftp.stat! target.full
|
40
40
|
|
41
41
|
rescue Net::SFTP::StatusException
|
@@ -44,23 +44,18 @@ module RestFtpDaemon
|
|
44
44
|
return stat.size
|
45
45
|
end
|
46
46
|
|
47
|
-
# def remove target
|
48
|
-
# log_info "RemoteSFTP.remove [#{target.name}]"
|
49
|
-
# @sftp.remove target.full
|
50
|
-
# end
|
51
|
-
|
52
47
|
def remove! target
|
53
|
-
|
48
|
+
log_debug "RemoteSFTP.remove! [#{target.name}]"
|
54
49
|
@sftp.remove target.full
|
55
50
|
|
56
51
|
rescue Net::SFTP::StatusException
|
57
|
-
|
52
|
+
log_debug "#{LOG_INDENT}[#{target.name}] file not found"
|
58
53
|
else
|
59
|
-
|
54
|
+
log_debug "#{LOG_INDENT}[#{target.name}] removed"
|
60
55
|
end
|
61
56
|
|
62
57
|
def mkdir directory
|
63
|
-
|
58
|
+
log_debug "RemoteSFTP.mkdir [#{directory}]"
|
64
59
|
@sftp.mkdir! directory
|
65
60
|
|
66
61
|
rescue
|
@@ -69,12 +64,12 @@ module RestFtpDaemon
|
|
69
64
|
|
70
65
|
def chdir_or_create directory, mkdir = false
|
71
66
|
# Init, extract my parent name and my own name
|
72
|
-
|
67
|
+
log_debug "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
73
68
|
parent, _current = Helpers.extract_parent(directory)
|
74
69
|
|
75
70
|
# Access this directory
|
76
71
|
begin
|
77
|
-
|
72
|
+
log_debug "chdir [/#{directory}]"
|
78
73
|
@sftp.opendir! "./#{directory}"
|
79
74
|
|
80
75
|
rescue Net::SFTP::StatusException => _e
|
@@ -97,12 +92,6 @@ module RestFtpDaemon
|
|
97
92
|
raise JobTargetShouldBeDirectory
|
98
93
|
end
|
99
94
|
|
100
|
-
# def dir_contents directory
|
101
|
-
# # Access this directory
|
102
|
-
# handle = @sftp.opendir! directory
|
103
|
-
# @sftp.readdir! handle
|
104
|
-
# end
|
105
|
-
|
106
95
|
def push source, target, tempname = nil, &callback
|
107
96
|
# Push init
|
108
97
|
raise RestFtpDaemon::JobAssertionFailed, "push/1" if @sftp.nil?
|
@@ -112,7 +101,7 @@ module RestFtpDaemon
|
|
112
101
|
destination.name = tempname if tempname
|
113
102
|
|
114
103
|
# Do the transfer
|
115
|
-
|
104
|
+
log_debug "RemoteSFTP.push [#{destination.full}]"
|
116
105
|
@sftp.upload! source.full, destination.full do |event, _uploader, *args|
|
117
106
|
case event
|
118
107
|
when :open then
|
@@ -140,7 +129,7 @@ module RestFtpDaemon
|
|
140
129
|
|
141
130
|
# Rename if needed
|
142
131
|
if tempname
|
143
|
-
|
132
|
+
log_debug "RemoteSFTP.push rename to\t[#{target.name}]"
|
144
133
|
@sftp.rename! destination.full, target.full, flags
|
145
134
|
end
|
146
135
|
|