rest-ftp-daemon 0.250.5 → 0.300.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +19 -14
  3. data/README.md +12 -3
  4. data/bin/rest-ftp-daemon +102 -96
  5. data/config.ru +5 -5
  6. data/defaults.yml +61 -0
  7. data/lib/rest-ftp-daemon.rb +10 -4
  8. data/lib/rest-ftp-daemon/api/config.rb +3 -2
  9. data/lib/rest-ftp-daemon/api/dashboard.rb +1 -4
  10. data/lib/rest-ftp-daemon/api/debug.rb +30 -17
  11. data/lib/rest-ftp-daemon/api/job_presenter.rb +0 -2
  12. data/lib/rest-ftp-daemon/api/jobs.rb +4 -3
  13. data/lib/rest-ftp-daemon/api/root.rb +7 -10
  14. data/lib/rest-ftp-daemon/api/status.rb +7 -13
  15. data/lib/rest-ftp-daemon/constants.rb +27 -45
  16. data/lib/rest-ftp-daemon/counters.rb +0 -4
  17. data/lib/rest-ftp-daemon/helpers.rb +3 -18
  18. data/lib/rest-ftp-daemon/job.rb +16 -21
  19. data/lib/rest-ftp-daemon/job_queue.rb +21 -14
  20. data/lib/rest-ftp-daemon/launcher.rb +26 -0
  21. data/lib/rest-ftp-daemon/logger_pool.rb +9 -19
  22. data/lib/rest-ftp-daemon/metrics.rb +41 -0
  23. data/lib/rest-ftp-daemon/notification.rb +7 -10
  24. data/lib/rest-ftp-daemon/remote.rb +4 -4
  25. data/lib/rest-ftp-daemon/remote_ftp.rb +10 -10
  26. data/lib/rest-ftp-daemon/remote_sftp.rb +13 -24
  27. data/lib/rest-ftp-daemon/views/dashboard.haml +2 -2
  28. data/lib/rest-ftp-daemon/views/dashboard_footer.haml +2 -2
  29. data/lib/rest-ftp-daemon/views/dashboard_header.haml +2 -2
  30. data/lib/rest-ftp-daemon/views/dashboard_workers.haml +2 -2
  31. data/lib/rest-ftp-daemon/worker.rb +43 -12
  32. data/lib/rest-ftp-daemon/worker_conchita.rb +15 -28
  33. data/lib/rest-ftp-daemon/worker_job.rb +30 -21
  34. data/lib/rest-ftp-daemon/worker_pool.rb +59 -50
  35. data/lib/rest-ftp-daemon/worker_reporter.rb +70 -0
  36. data/lib/shared/conf.rb +195 -0
  37. data/lib/shared/logger_formatter.rb +31 -0
  38. data/lib/shared/logger_helper.rb +78 -0
  39. data/rest-ftp-daemon.gemspec +23 -22
  40. data/{rest-ftp-daemon.yml.sample → rest-ftp-daemon.sample.yml} +10 -7
  41. data/spec/spec_helper.rb +1 -1
  42. metadata +30 -12
  43. data/lib/rest-ftp-daemon/logger.rb +0 -57
  44. data/lib/rest-ftp-daemon/logger_helper.rb +0 -36
  45. data/lib/rest-ftp-daemon/settings.rb +0 -57
@@ -2,16 +2,10 @@ module RestFtpDaemon
2
2
 
3
3
  # Queue that stores all the Jobs waiting to be processed or fully processed
4
4
  class JobQueue
5
- include LoggerHelper
5
+ include Shared::LoggerHelper
6
6
  attr_reader :logger
7
-
8
- #attr_reader :queues
9
7
  attr_reader :jobs
10
8
 
11
- if Settings.newrelic_enabled?
12
- include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
13
- end
14
-
15
9
  def initialize
16
10
  # Instance variables
17
11
  @queues = {}
@@ -45,10 +39,18 @@ module RestFtpDaemon
45
39
  @queues
46
40
  end
47
41
 
42
+ def queued_by_pool
43
+ result = {}
44
+ @queues.each do |pool, jobs|
45
+ result[pool] = jobs.count
46
+ end
47
+ result
48
+ end
49
+
48
50
  def rate_by method_name
49
51
  # Init
50
52
  result = {}
51
- return unless Job.new(0, {}).respond_to? method_name
53
+ #return unless Job.new(0, {}).respond_to? method_name
52
54
 
53
55
  # Select only running jobs
54
56
  @jobs.each do |job|
@@ -70,9 +72,7 @@ module RestFtpDaemon
70
72
 
71
73
  # Add its current rate
72
74
  result[group] ||= 0
73
- #log_info " 2: #{result.inspect} (rate: #{rate})"
74
75
  result[group] += rate
75
- #log_info " 3: #{result.inspect}"
76
76
  end
77
77
 
78
78
  # Return the rate
@@ -109,7 +109,6 @@ module RestFtpDaemon
109
109
  log_info "find_by_id (#{id}, #{prefixed}) > #{id}"
110
110
 
111
111
  # Search in jobs queues
112
- #@jobs.reverse.find { |item| item.id == id }
113
112
  @jobs.find { |item| item.id == id }
114
113
  end
115
114
 
@@ -201,7 +200,7 @@ module RestFtpDaemon
201
200
  @mutex.synchronize do
202
201
  # Delete jobs from the queue when they match status and age limits
203
202
  @jobs.delete_if do |job|
204
- # log_info "testing job [#{job.id}] updated_at [#{job.updated_at}]"
203
+ # log_debug "testing job [#{job.id}] updated_at [#{job.updated_at}]"
205
204
 
206
205
  # Skip if wrong status, updated_at invalid, or updated since time_limit
207
206
  next unless job.status == status
@@ -213,7 +212,7 @@ module RestFtpDaemon
213
212
 
214
213
  # From any queues, remove it
215
214
  @queues.each do |pool, jobs|
216
- log_info "#{LOG_INDENT}unqueued from [#{pool}]" if jobs.delete(job)
215
+ log_debug "#{LOG_INDENT}unqueued from [#{pool}]" if jobs.delete(job)
217
216
  end
218
217
 
219
218
  # Remember we have to delete the original job !
@@ -225,11 +224,19 @@ module RestFtpDaemon
225
224
 
226
225
  protected
227
226
 
227
+ def log_prefix
228
+ [nil, nil, nil]
229
+ end
230
+
228
231
  def prefixed_id id
229
232
  "#{@prefix}.#{id}"
230
233
  end
231
234
 
232
- if Settings.newrelic_enabled?
235
+ private
236
+
237
+ # NewRelic instrumentation
238
+ if Conf.newrelic_enabled?
239
+ include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
233
240
  add_transaction_tracer :push, category: :task
234
241
  add_transaction_tracer :pop, category: :task
235
242
  add_transaction_tracer :expire, category: :task
@@ -0,0 +1,26 @@
1
+ module RestFtpDaemon
2
+ class Launcher
3
+ LAUNCHER_PORT_TIMEOUT = 3
4
+ LAUNCHER_PORT_LOCALHOST = "127.0.0.1"
5
+
6
+ class << self
7
+
8
+ def local_port_used? port
9
+ Timeout.timeout(LAUNCHER_PORT_TIMEOUT) do
10
+ begin
11
+ TCPSocket.new(LAUNCHER_PORT_LOCALHOST, port).close
12
+ true
13
+ rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
14
+ false
15
+ rescue Errno::EADDRNOTAVAIL
16
+ "local_port_used: Errno::EADDRNOTAVAIL"
17
+ end
18
+ end
19
+ rescue Timeout::Error
20
+ false
21
+ end
22
+
23
+ end
24
+
25
+ end
26
+ end
@@ -16,29 +16,19 @@ module RestFtpDaemon
16
16
 
17
17
  def create pipe
18
18
  # Compute file path / STDERR
19
- logfile = Settings.logs[pipe] if Settings.logs.is_a? Hash
19
+ logfile = Conf[:logs][pipe] if Conf[:logs].is_a? Hash
20
20
  logfile ||= STDERR
21
21
 
22
+ # Check if we can write to than file
23
+ unless File.writable?(logfile)
24
+ puts "LoggerPool [#{pipe}] logging to hyperspace (cannot write to #{logfile})"
25
+ logfile = nil
26
+ end
27
+
22
28
  # Create the logger and return it
23
29
  logger = Logger.new(logfile, LOG_ROTATION) #, 10, 1024000)
24
- logger.progname = pipe.to_s.upcase
25
-
26
- # And the formatter
27
- logger.formatter = proc do |severity, datetime, progname, messages|
28
- # Build common line prefix
29
- prefix = LOG_FORMAT_PREFIX % [
30
- datetime.strftime(LOG_FORMAT_TIME),
31
- severity,
32
- progname,
33
- ]
34
-
35
- # If we have a bunch of lines, prefix them and send them together
36
- if messages.is_a? Array
37
- messages.map { |line| prefix + line + LOG_NEWLINE}.join
38
- else
39
- prefix + messages.to_s + LOG_NEWLINE
40
- end
41
- end
30
+ logger.progname = pipe.to_s.downcase
31
+ logger.formatter = Shared::LoggerFormatter
42
32
 
43
33
  # Finally return this logger
44
34
  logger
@@ -0,0 +1,41 @@
1
+ module RestFtpDaemon
2
+ class Metrics
3
+
4
+ def self.sample
5
+ # Prepare external deps
6
+ mem = GetProcessMem.new
7
+
8
+ # Build final value
9
+ return {
10
+ system: {
11
+ uptime: (Time.now - Conf.app_started).round(1),
12
+ memory: mem.bytes.to_i,
13
+ threads: Thread.list.count,
14
+ },
15
+ jobs_by_status: $queue.jobs_by_status,
16
+ rate_by_pool: $queue.rate_by(:pool),
17
+ rate_by_targethost: $queue.rate_by(:targethost),
18
+ queued_by_pool: $queue.queued_by_pool,
19
+ workers_by_status: self.workers_count_by_status,
20
+ }
21
+ end
22
+
23
+ private
24
+
25
+ # Collect: workers by status
26
+ def self.workers_count_by_status
27
+ # Init
28
+ counts = {}
29
+
30
+ $pool.worker_variables.group_by do |wid, vars|
31
+ vars[:status]
32
+ end.each do |status, workers|
33
+ counts[status] = workers.count
34
+ end
35
+
36
+ # Return count
37
+ counts
38
+ end
39
+
40
+ end
41
+ end
@@ -2,7 +2,7 @@ module RestFtpDaemon
2
2
 
3
3
  # Handles a notification POST using a dedicated thread
4
4
  class Notification
5
- include LoggerHelper
5
+ include Shared::LoggerHelper
6
6
  attr_reader :logger
7
7
 
8
8
  attr_accessor :job_id
@@ -35,10 +35,10 @@ module RestFtpDaemon
35
35
  def process
36
36
  # Check context
37
37
  if @url.nil?
38
- log_info "skipping (missing url): #{@params.inspect}"
38
+ log_error "skipping (missing url)", params
39
39
  return
40
40
  elsif @params[:event].nil?
41
- log_info "skipping (missing event): #{@params.inspect}"
41
+ log_error "skipping (missing event)", params
42
42
  return
43
43
  end
44
44
 
@@ -47,7 +47,7 @@ module RestFtpDaemon
47
47
  id: @params[:id].to_s,
48
48
  signal: "#{NOTIFY_PREFIX}.#{@params[:event]}",
49
49
  error: @params[:error],
50
- host: Settings.host.to_s,
50
+ host: Conf.host.to_s,
51
51
  }
52
52
  flags[:status] = @params[:status] if @params[:status].is_a? Enumerable
53
53
  flags[:message] = @params[:message].to_s unless @params[:message].nil?
@@ -64,7 +64,7 @@ module RestFtpDaemon
64
64
  headers = {
65
65
  "Content-Type" => "application/json",
66
66
  "Accept" => "application/json",
67
- "User-Agent" => NOTIFY_USERAGENT
67
+ "User-Agent" => "#{Conf.app_name}/v#{Conf.app_ver}"
68
68
  }
69
69
  data = flags.to_json
70
70
 
@@ -96,11 +96,8 @@ module RestFtpDaemon
96
96
  log_error "UNHANDLED EXCEPTION: #{ex.inspect}"
97
97
  end
98
98
 
99
- def log_context
100
- {
101
- id: @id,
102
- jid: @jid
103
- }
99
+ def log_prefix
100
+ [nil, @jid, @id]
104
101
  end
105
102
 
106
103
  end
@@ -2,13 +2,13 @@ module RestFtpDaemon
2
2
 
3
3
  # Handles transfers for Job class
4
4
  class Remote
5
- include LoggerHelper
5
+ include Shared::LoggerHelper
6
6
  attr_reader :logger
7
- attr_reader :log_context
7
+ attr_reader :log_prefix
8
8
 
9
- def initialize url, log_context, options = {}
9
+ def initialize url, log_prefix, options = {}
10
10
  # Logger
11
- @log_context = log_context || {}
11
+ @log_prefix = log_prefix || {}
12
12
  @logger = RestFtpDaemon::LoggerPool.instance.get :jobs
13
13
 
14
14
  # Extract URL parts
@@ -6,12 +6,12 @@ module RestFtpDaemon
6
6
  class RemoteFTP < Remote
7
7
  attr_reader :ftp
8
8
 
9
- def initialize url, log_context, options = {}
9
+ def initialize url, log_prefix, options = {}
10
10
  # Call super
11
11
  super
12
12
 
13
13
  # Use debug ?
14
- @debug = (Settings.at :debug, :ftp) == true
14
+ @debug = (Conf.at :debug, :ftp) == true
15
15
 
16
16
  # Create FTP object
17
17
  if options[:ftpes]
@@ -26,7 +26,7 @@ module RestFtpDaemon
26
26
  @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
27
27
 
28
28
  # Announce object
29
- log_info "RemoteFTP.initialize chunk_size:#{@chunk_size}"
29
+ log_debug "RemoteFTP.initialize chunk_size:#{@chunk_size}"
30
30
  end
31
31
 
32
32
  def connect
@@ -40,7 +40,7 @@ module RestFtpDaemon
40
40
 
41
41
  def present? target
42
42
  size = @ftp.size target.full
43
- log_info "RemoteFTP.present? [#{target.name}]"
43
+ log_debug "RemoteFTP.present? [#{target.name}]"
44
44
 
45
45
  rescue Net::FTPPermError
46
46
  return false
@@ -49,16 +49,16 @@ module RestFtpDaemon
49
49
  end
50
50
 
51
51
  def remove! target
52
- log_info "RemoteFTP.remove! [#{target.name}]"
52
+ log_debug "RemoteFTP.remove! [#{target.name}]"
53
53
  @ftp.delete target.full
54
54
  rescue Net::FTPPermError
55
- log_info "#{LOG_INDENT}[#{target.name}] file not found"
55
+ log_debug "#{LOG_INDENT}[#{target.name}] file not found"
56
56
  else
57
- log_info "#{LOG_INDENT}[#{target.name}] removed"
57
+ log_debug "#{LOG_INDENT}[#{target.name}] removed"
58
58
  end
59
59
 
60
60
  def mkdir directory
61
- log_info "RemoteFTP.mkdir [#{directory}]"
61
+ log_debug "RemoteFTP.mkdir [#{directory}]"
62
62
  @ftp.mkdir directory
63
63
 
64
64
  rescue
@@ -67,7 +67,7 @@ module RestFtpDaemon
67
67
 
68
68
  def chdir_or_create directory, mkdir = false
69
69
  # Init, extract my parent name and my own name
70
- log_info "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
70
+ log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
71
71
  parent, _current = Helpers.extract_parent(directory)
72
72
 
73
73
  # Access this directory
@@ -98,7 +98,7 @@ module RestFtpDaemon
98
98
  destination.name = tempname if tempname
99
99
 
100
100
  # Do the transfer
101
- log_info "RemoteFTP.push to [#{destination.name}]"
101
+ log_debug "RemoteFTP.push to [#{destination.name}]"
102
102
 
103
103
  @ftp.putbinaryfile source.full, target.name, @chunk_size do |data|
104
104
  # Update job status after this block transfer
@@ -5,21 +5,21 @@ module RestFtpDaemon
5
5
  class RemoteSFTP < Remote
6
6
  attr_reader :sftp
7
7
 
8
- def initialize url, log_context, options = {}
8
+ def initialize url, log_prefix, options = {}
9
9
  # Call super
10
10
  super
11
11
 
12
12
  # Use debug ?
13
- @debug = (Settings.at :debug, :sftp) == true
13
+ @debug = (Conf.at :debug, :sftp) == true
14
14
 
15
15
  # Announce object
16
- log_info "RemoteSFTP.initialize"
16
+ log_debug "RemoteSFTP.initialize"
17
17
  end
18
18
 
19
19
  def connect
20
20
  # Connect init
21
21
  super
22
- log_info "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
22
+ log_debug "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
23
23
 
24
24
  # Debug level
25
25
  verbosity = @debug ? Logger::INFO : false
@@ -35,7 +35,7 @@ module RestFtpDaemon
35
35
  end
36
36
 
37
37
  def present? target
38
- log_info "RemoteSFTP.present? [#{target.name}]"
38
+ log_debug "RemoteSFTP.present? [#{target.name}]"
39
39
  stat = @sftp.stat! target.full
40
40
 
41
41
  rescue Net::SFTP::StatusException
@@ -44,23 +44,18 @@ module RestFtpDaemon
44
44
  return stat.size
45
45
  end
46
46
 
47
- # def remove target
48
- # log_info "RemoteSFTP.remove [#{target.name}]"
49
- # @sftp.remove target.full
50
- # end
51
-
52
47
  def remove! target
53
- log_info "RemoteSFTP.remove! [#{target.name}]"
48
+ log_debug "RemoteSFTP.remove! [#{target.name}]"
54
49
  @sftp.remove target.full
55
50
 
56
51
  rescue Net::SFTP::StatusException
57
- log_info "#{LOG_INDENT}[#{target.name}] file not found"
52
+ log_debug "#{LOG_INDENT}[#{target.name}] file not found"
58
53
  else
59
- log_info "#{LOG_INDENT}[#{target.name}] removed"
54
+ log_debug "#{LOG_INDENT}[#{target.name}] removed"
60
55
  end
61
56
 
62
57
  def mkdir directory
63
- log_info "RemoteSFTP.mkdir [#{directory}]"
58
+ log_debug "RemoteSFTP.mkdir [#{directory}]"
64
59
  @sftp.mkdir! directory
65
60
 
66
61
  rescue
@@ -69,12 +64,12 @@ module RestFtpDaemon
69
64
 
70
65
  def chdir_or_create directory, mkdir = false
71
66
  # Init, extract my parent name and my own name
72
- log_info "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
67
+ log_debug "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
73
68
  parent, _current = Helpers.extract_parent(directory)
74
69
 
75
70
  # Access this directory
76
71
  begin
77
- # log_info " chdir [/#{directory}]"
72
+ log_debug "chdir [/#{directory}]"
78
73
  @sftp.opendir! "./#{directory}"
79
74
 
80
75
  rescue Net::SFTP::StatusException => _e
@@ -97,12 +92,6 @@ module RestFtpDaemon
97
92
  raise JobTargetShouldBeDirectory
98
93
  end
99
94
 
100
- # def dir_contents directory
101
- # # Access this directory
102
- # handle = @sftp.opendir! directory
103
- # @sftp.readdir! handle
104
- # end
105
-
106
95
  def push source, target, tempname = nil, &callback
107
96
  # Push init
108
97
  raise RestFtpDaemon::JobAssertionFailed, "push/1" if @sftp.nil?
@@ -112,7 +101,7 @@ module RestFtpDaemon
112
101
  destination.name = tempname if tempname
113
102
 
114
103
  # Do the transfer
115
- log_info "RemoteSFTP.push [#{destination.full}]"
104
+ log_debug "RemoteSFTP.push [#{destination.full}]"
116
105
  @sftp.upload! source.full, destination.full do |event, _uploader, *args|
117
106
  case event
118
107
  when :open then
@@ -140,7 +129,7 @@ module RestFtpDaemon
140
129
 
141
130
  # Rename if needed
142
131
  if tempname
143
- log_info "RemoteSFTP.push rename to\t[#{target.name}]"
132
+ log_debug "RemoteSFTP.push rename to\t[#{target.name}]"
144
133
  @sftp.rename! destination.full, target.full, flags
145
134
  end
146
135