rest-ftp-daemon 0.250.5 → 0.300.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +19 -14
  3. data/README.md +12 -3
  4. data/bin/rest-ftp-daemon +102 -96
  5. data/config.ru +5 -5
  6. data/defaults.yml +61 -0
  7. data/lib/rest-ftp-daemon.rb +10 -4
  8. data/lib/rest-ftp-daemon/api/config.rb +3 -2
  9. data/lib/rest-ftp-daemon/api/dashboard.rb +1 -4
  10. data/lib/rest-ftp-daemon/api/debug.rb +30 -17
  11. data/lib/rest-ftp-daemon/api/job_presenter.rb +0 -2
  12. data/lib/rest-ftp-daemon/api/jobs.rb +4 -3
  13. data/lib/rest-ftp-daemon/api/root.rb +7 -10
  14. data/lib/rest-ftp-daemon/api/status.rb +7 -13
  15. data/lib/rest-ftp-daemon/constants.rb +27 -45
  16. data/lib/rest-ftp-daemon/counters.rb +0 -4
  17. data/lib/rest-ftp-daemon/helpers.rb +3 -18
  18. data/lib/rest-ftp-daemon/job.rb +16 -21
  19. data/lib/rest-ftp-daemon/job_queue.rb +21 -14
  20. data/lib/rest-ftp-daemon/launcher.rb +26 -0
  21. data/lib/rest-ftp-daemon/logger_pool.rb +9 -19
  22. data/lib/rest-ftp-daemon/metrics.rb +41 -0
  23. data/lib/rest-ftp-daemon/notification.rb +7 -10
  24. data/lib/rest-ftp-daemon/remote.rb +4 -4
  25. data/lib/rest-ftp-daemon/remote_ftp.rb +10 -10
  26. data/lib/rest-ftp-daemon/remote_sftp.rb +13 -24
  27. data/lib/rest-ftp-daemon/views/dashboard.haml +2 -2
  28. data/lib/rest-ftp-daemon/views/dashboard_footer.haml +2 -2
  29. data/lib/rest-ftp-daemon/views/dashboard_header.haml +2 -2
  30. data/lib/rest-ftp-daemon/views/dashboard_workers.haml +2 -2
  31. data/lib/rest-ftp-daemon/worker.rb +43 -12
  32. data/lib/rest-ftp-daemon/worker_conchita.rb +15 -28
  33. data/lib/rest-ftp-daemon/worker_job.rb +30 -21
  34. data/lib/rest-ftp-daemon/worker_pool.rb +59 -50
  35. data/lib/rest-ftp-daemon/worker_reporter.rb +70 -0
  36. data/lib/shared/conf.rb +195 -0
  37. data/lib/shared/logger_formatter.rb +31 -0
  38. data/lib/shared/logger_helper.rb +78 -0
  39. data/rest-ftp-daemon.gemspec +23 -22
  40. data/{rest-ftp-daemon.yml.sample → rest-ftp-daemon.sample.yml} +10 -7
  41. data/spec/spec_helper.rb +1 -1
  42. metadata +30 -12
  43. data/lib/rest-ftp-daemon/logger.rb +0 -57
  44. data/lib/rest-ftp-daemon/logger_helper.rb +0 -36
  45. data/lib/rest-ftp-daemon/settings.rb +0 -57
@@ -6,7 +6,7 @@
6
6
  %link{ href:"/css/bootstrap.css" , rel: "stylesheet"}
7
7
  %link{ href:"/css/main.css" , rel: "stylesheet"}
8
8
  -# %link{ href:"http://fonts.googleapis.com/css?family=Inconsolata:400,700" , rel: "stylesheet"}
9
- %title="#{Settings.host} [#{Settings.namespace}] #{APP_NAME}"
9
+ %title="#{Conf.host} [#{Conf.app_env}] #{Conf.app_name}"
10
10
 
11
11
  %body
12
12
 
@@ -21,7 +21,7 @@
21
21
 
22
22
  .row
23
23
  #box-tokens.col-md-6
24
- = render :dashboard_tokens, {tokens: Settings.endpoints || {}}
24
+ = render :dashboard_tokens, {tokens: Conf[:endpoints] || {}}
25
25
 
26
26
  #box-workers.col-md-3
27
27
  = render :dashboard_workers
@@ -17,7 +17,7 @@
17
17
 
18
18
  .btn-group.btn-group-sm
19
19
  .btn.btn-default.btn-info Host
20
- .btn.btn-default= Settings.host
20
+ .btn.btn-default= Conf.host
21
21
 
22
22
  .btn-group.btn-group-sm
23
23
  .btn.btn-default.btn-info IP
@@ -29,4 +29,4 @@
29
29
 
30
30
  .btn-group.btn-group-sm
31
31
  .btn.btn-default.btn-info Started
32
- .btn.btn-default= Helpers.datetime_short(APP_STARTED)
32
+ .btn.btn-default= Helpers.datetime_short(Conf.app_started)
@@ -10,9 +10,9 @@
10
10
  .navbar-header
11
11
 
12
12
  %h1
13
- = APP_NAME
13
+ = Conf.app_name
14
14
  %small
15
- = "v#{APP_VER} [#{Settings.namespace}]"
15
+ = "v#{Conf.app_ver} [#{Conf.app_env}]"
16
16
 
17
17
 
18
18
  .header-indicators.navbar-header.pull-right
@@ -14,8 +14,8 @@
14
14
  %th.text-right seen
15
15
 
16
16
  %tbody
17
- - variables.each do |vars|
18
- - wid = vars[:wid]
17
+ - variables.each do |wid, vars|
18
+ -# wid = vars[:wid]
19
19
  - status = vars[:status]
20
20
  - alive = $pool.worker_alive? wid
21
21
  - trclass = WORKER_STYLES[status]
@@ -1,12 +1,8 @@
1
1
  module RestFtpDaemon
2
2
  class Worker
3
- include LoggerHelper
3
+ include Shared::LoggerHelper
4
4
  attr_reader :logger
5
5
 
6
- if Settings.newrelic_enabled?
7
- include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
8
- end
9
-
10
6
  def initialize wid, pool = nil
11
7
  # Logger
12
8
  @logger = RestFtpDaemon::LoggerPool.instance.get :workers
@@ -20,15 +16,28 @@ module RestFtpDaemon
20
16
  Thread.current.thread_variable_set :wid, wid
21
17
  Thread.current.thread_variable_set :started_at, Time.now
22
18
  worker_status WORKER_STATUS_STARTING
19
+
20
+ # Load corker conf
21
+ load_config wid
23
22
  end
24
23
 
25
24
  protected
26
25
 
27
- def log_context
28
- {
29
- wid: Thread.current.thread_variable_get(:wid),
30
- jid: Thread.current.thread_variable_get(:jid),
31
- }
26
+ def wait_according_to_config
27
+ # Sleep for a few seconds
28
+ worker_status WORKER_STATUS_WAITING
29
+ sleep @config[:timer] if @config.is_a? Hash
30
+ end
31
+
32
+ def log_prefix
33
+ [
34
+ Thread.current.thread_variable_get(:wid),
35
+ Thread.current.thread_variable_get(:jid),
36
+ nil
37
+ ]
38
+ end
39
+
40
+ def work
32
41
  end
33
42
 
34
43
  def start
@@ -52,9 +61,9 @@ module RestFtpDaemon
52
61
 
53
62
  # Log this status change
54
63
  if job.is_a?(Job)
55
- log_info "#{status} - job[#{job.id}] status[#{job.status}] error[#{job.error}]"
64
+ log_info "status [#{status}] on job[#{job.id}] status[#{job.status}] error[#{job.error}]"
56
65
  else
57
- log_info "#{status}"
66
+ log_info "status [#{status}]"
58
67
  end
59
68
  end
60
69
 
@@ -63,5 +72,27 @@ module RestFtpDaemon
63
72
  Thread.current.thread_variable_set :updated_at, Time.now
64
73
  end
65
74
 
75
+ private
76
+
77
+ def load_config wid
78
+ # My debug
79
+ @debug = (Conf.at :debug, wid) == true
80
+ @log_worker_status_changes = @debug
81
+
82
+ # My configuration
83
+ @config = Conf[wid]
84
+ if !@config.is_a? Hash
85
+ return log_info "#{self.class.name}: missing #{wid}/* configuration"
86
+ elsif @config[:timer].nil?
87
+ return log_info "#{self.class.name}: missing #{wid}/timer value"
88
+ end
89
+ end
90
+
91
+ # NewRelic instrumentation
92
+ if Conf.newrelic_enabled?
93
+ include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
94
+ add_transaction_tracer :work, category: :task
95
+ end
96
+
66
97
  end
67
98
  end
@@ -3,29 +3,25 @@ module RestFtpDaemon
3
3
  # Worker used to clean up the queue deleting expired jobs
4
4
  class ConchitaWorker < Worker
5
5
 
6
- def initialize wid = :conchita
7
- # Generic worker initialize
6
+ def initialize wid, pool = nil
7
+ # Call dady and load my conf
8
8
  super
9
9
 
10
- # Use debug ?
11
- @debug = (Settings.at :debug, :conchita) == true
12
- @log_worker_status_changes = @debug
13
-
14
- # Conchita configuration
15
- @conchita = Settings.conchita
16
- if !@conchita.is_a? Hash
17
- return log_info "ConchitaWorker: missing conchita.* configuration"
18
- elsif @conchita[:timer].nil?
19
- return log_info "ConchitaWorker: missing conchita.timer value"
20
- end
21
-
22
10
  # Start main loop
23
- log_info "ConchitaWorker starting", @conchita
11
+ log_info "#{self.class.name} starting", @config
24
12
  start
25
13
  end
26
14
 
27
15
  protected
28
16
 
17
+ # def log_prefix
18
+ # [
19
+ # Thread.current.thread_variable_get(:wid),
20
+ # nil,
21
+ # nil
22
+ # ]
23
+ # end
24
+
29
25
  def work
30
26
  # Announce we are working
31
27
  worker_status WORKER_STATUS_CLEANING
@@ -36,26 +32,17 @@ module RestFtpDaemon
36
32
  $queue.expire JOB_STATUS_QUEUED, maxage(JOB_STATUS_QUEUED), @debug
37
33
 
38
34
  # Force garbage collector
39
- GC.start if @conchita["garbage_collector"]
35
+ GC.start if @config["garbage_collector"]
40
36
 
41
37
  rescue StandardError => e
42
- log_error "CONCHITA EXCEPTION: #{e.inspect}"
38
+ log_error "EXCEPTION: #{e.inspect}"
43
39
  sleep 1
44
40
  else
45
- # Restore previous status
46
- worker_status WORKER_STATUS_WAITING
47
-
48
- # Sleep for a few seconds
49
- sleep @conchita[:timer]
41
+ wait_according_to_config
50
42
  end
51
43
 
52
44
  def maxage status
53
- @conchita["clean_#{status}"] || 0
54
- end
55
-
56
-
57
- if Settings.newrelic_enabled?
58
- add_transaction_tracer :work, category: :task
45
+ @config["clean_#{status}"] || 0
59
46
  end
60
47
 
61
48
  end
@@ -2,22 +2,40 @@ module RestFtpDaemon
2
2
 
3
3
  # Worker used to process Jobs
4
4
  class JobWorker < Worker
5
- #attr_reader :pool
6
5
 
7
6
  def initialize wid, pool
8
- # Generic worker initialize
7
+ # Call dady and load my conf
9
8
  super
10
9
 
11
- # Timeout config
12
- @timeout = (Settings.transfer.timeout rescue nil) || DEFAULT_WORKER_TIMEOUT
10
+ # Timeout and retry config
11
+ @timeout = (Conf.at(:transfer, :timeout) rescue nil)
12
+ @retry = (Conf.at(:retry) rescue {})
13
+
14
+ # Retry config
15
+ @retry_on_errors = Conf.at(:retry, :on_errors)
16
+ @retry_max_age = Conf.at(:retry, :max_age)
17
+ @retry_max_runs = Conf.at(:retry, :max_runs)
18
+ @retry_delay = Conf.at(:retry, :delay)
19
+
13
20
 
14
21
  # Start main loop
15
- log_info "JobWorker initializing", ["wid: #{wid}", "pool: #{pool}", "timeout: #{@timeout}"]
22
+ log_info "JobWorker initializing", {
23
+ wid: wid,
24
+ pool: pool,
25
+ timeout: @timeout
26
+ }
16
27
  start
17
28
  end
18
29
 
19
30
  protected
20
31
 
32
+
33
+ def load_config wid
34
+ # Do nothing
35
+ end
36
+
37
+ private
38
+
21
39
  def work
22
40
  # Wait for a job to be available in the queue
23
41
  worker_status WORKER_STATUS_WAITING
@@ -31,26 +49,21 @@ module RestFtpDaemon
31
49
  #sleep 1
32
50
 
33
51
  # If job status requires a retry, just restack it
34
- on_errors = Settings.at(:retry, :on_errors)
35
- max_age = Settings.at(:retry, :max_age)
36
- max_runs = Settings.at(:retry, :max_runs)
37
- delay = Settings.at(:retry, :delay)
38
-
39
52
  if !job.error
40
53
  #log_info "job succeeded"
41
54
 
42
- elsif !(on_errors.is_a?(Enumerable) && on_errors.include?(job.error))
55
+ elsif !(@retry_on_errors.is_a?(Enumerable) && @retry_on_errors.include?(job.error))
43
56
  log_error "not retrying: error not eligible"
44
57
 
45
- elsif max_age && (job.age >= max_age)
46
- log_error "not retrying: max_age reached (#{max_age} s)"
58
+ elsif @retry_max_age && (job.age >= @retry_max_age)
59
+ log_error "not retrying: max_age reached (#{@retry_max_age} s)"
47
60
 
48
- elsif max_runs && (job.runs >= max_runs)
49
- log_error "not retrying: max_runs reached (#{max_runs} tries)"
61
+ elsif @retry_max_runs && (job.runs >= @retry_max_runs)
62
+ log_error "not retrying: max_runs reached (#{@retry_max_runs} tries)"
50
63
 
51
64
  else
52
65
  # Delay cannot be negative, and will be 1s minimum
53
- retry_after = [delay || DEFAULT_RETRY_DELAY, 1].max
66
+ retry_after = [@retry_delay || DEFAULT_RETRY_DELAY, 1].max
54
67
  log_info "retrying job: waiting for #{retry_after} seconds"
55
68
 
56
69
  # Wait !
@@ -71,8 +84,8 @@ module RestFtpDaemon
71
84
 
72
85
  def work_on_job job
73
86
  # Prepare job and worker for processing
74
- worker_status WORKER_STATUS_RUNNING, job
75
87
  worker_jid job.id
88
+ worker_status WORKER_STATUS_RUNNING, job
76
89
  job.wid = Thread.current.thread_variable_get :wid
77
90
 
78
91
  # Processs this job protected by a timeout
@@ -101,9 +114,5 @@ module RestFtpDaemon
101
114
  job.oops_after_crash ex unless job.nil?
102
115
  end
103
116
 
104
- if Settings.newrelic_enabled?
105
- add_transaction_tracer :work, category: :task
106
- end
107
-
108
117
  end
109
118
  end
@@ -2,14 +2,10 @@ module RestFtpDaemon
2
2
 
3
3
  # Handles a pool of Worker objects
4
4
  class WorkerPool
5
- include LoggerHelper
5
+ include Shared::LoggerHelper
6
6
  attr_reader :logger
7
7
  attr_reader :wid
8
8
 
9
- if Settings.newrelic_enabled?
10
- include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
11
- end
12
-
13
9
  def initialize
14
10
  # Logger
15
11
  @logger = RestFtpDaemon::LoggerPool.instance.get :workers
@@ -17,93 +13,106 @@ module RestFtpDaemon
17
13
  # Prepare status hash and vars
18
14
  @statuses = {}
19
15
  @workers = {}
20
- @conchita = nil
21
16
  @mutex = Mutex.new
22
17
 
23
18
  # Identifiers generator
24
- @last_id = 0
19
+ @last_worker_id = 0
25
20
 
26
21
  # Create worker threads
27
22
  create_threads
28
23
  end
29
24
 
30
25
  def worker_variables
31
- @workers.collect do |_wid, worker|
32
- vars = {}
33
- worker.thread_variables.each do |var|
34
- vars[var] = worker.thread_variable_get var
35
- end
36
- vars
26
+ vars = {}
27
+ @workers.collect do |wid, worker|
28
+ vars[wid] = thread_variables worker
37
29
  end
30
+ vars
38
31
  end
39
32
 
40
33
  def worker_alive? wid
41
34
  @workers[wid] && @workers[wid].alive?
42
35
  end
43
36
 
37
+ protected
38
+
39
+ def log_prefix
40
+ [nil, nil, nil]
41
+ end
42
+
44
43
  private
45
44
 
46
- def generate_id
45
+ def thread_variables thread
46
+ vars = {}
47
+ thread.thread_variables.each do |var|
48
+ vars[var] = thread.thread_variable_get var
49
+ end
50
+ vars
51
+ end
52
+
53
+ def next_worker_id
47
54
  @mutex.synchronize do
48
- @last_id += 1
55
+ @last_worker_id += 1
49
56
  end
50
- "w#{@last_id}"
57
+ "w#{@last_worker_id}"
51
58
  end
52
59
 
53
60
  def create_threads
54
- # Read configuration
55
- pools = (Settings.pools || {})
61
+ # Read configuration or initialize with empty hash
62
+ pools = Conf[:pools]
63
+ pools = {} unless pools.is_a? Hash
56
64
 
57
65
  # Minimum one worker on DEFAULT_POOL
58
66
  if !(pools.is_a? Hash)
59
- log_error "create_threads: one worker is the minimum possible number (#{pools.inspect}"
67
+ log_error "create_threads: one JobWorker is the minimum (#{pools.inspect}"
60
68
  end
61
- pools[DEFAULT_POOL] ||= 1
69
+ log_info "WorkerPool creating workers - JobWorker #{pools.to_hash.inspect}"
62
70
 
63
- # Create workers
64
- log_info "WorkerPool creating JobWorker's #{pools.inspect} + ConchitaWorker"
71
+ # Start ConchitaWorker and ReporterWorker
72
+ create_thread :conchita, ConchitaWorker
73
+ create_thread :reporter, ReporterWorker
74
+
75
+ # Start JobWorkers threads, ensure we have at least one worker in default pool
76
+ pools[DEFAULT_POOL] ||= 1
65
77
  pools.each do |pool, count|
66
- # Start worker threads for each pool
67
- log_info "WorkerPool creating JobWorker [#{pool}] x#{count}"
68
78
  count.times do
69
- wid = generate_id
70
- @workers[wid] = create_worker_thread wid, pool
79
+ wid = next_worker_id
80
+ @workers[wid] = create_thread wid, JobWorker, pool
71
81
  end
72
82
  end
73
83
 
74
- # Start conchita thread
75
- log_info "WorkerPool creating ConchitaWorker"
76
- @conchita = create_conchita_thread
77
-
78
84
  rescue StandardError => ex
79
- log_error "UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
80
- end
81
-
82
- def create_worker_thread wid, pool
83
- Thread.new wid do
84
- begin
85
- worker = JobWorker.new wid, pool
86
- log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
87
- rescue StandardError => ex
88
- log_error "JobWorker EXCEPTION: #{ex.message}"
89
- end
90
- end
85
+ log_error "EXCEPTION: #{ex.message}", ex.backtrace
91
86
  end
92
87
 
93
- def create_conchita_thread
94
- Thread.new do
88
+ # def create_worker_thread wid, pool
89
+ # Thread.new wid do
90
+ # begin
91
+ # worker = JobWorker.new wid, pool
92
+ # #log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
93
+ # rescue StandardError => ex
94
+ # log_error "JobWorker EXCEPTION: #{ex.message} #{e.backtrace}"
95
+ # end
96
+ # end
97
+ # end
98
+
99
+ def create_thread wid, klass, pool = nil
100
+ # Spawn thread and add it to my index
101
+ log_info "spawning #{klass.name} [#{wid}] [#{pool}]"
102
+ @workers[wid] = Thread.new do
95
103
  begin
96
- worker = ConchitaWorker.new :conchita
97
- log_info "ConchitaWorker: #{worker}"
104
+ worker = klass.new wid, pool
98
105
  rescue StandardError => ex
99
- log_error "ConchitaWorker EXCEPTION: #{ex.message}"
106
+ log_error "#{klass.name} EXCEPTION: #{ex.message}"
100
107
  end
101
108
  end
102
109
  end
103
110
 
104
- if Settings.newrelic_enabled?
105
- add_transaction_tracer :create_conchita_thread, category: :task
106
- add_transaction_tracer :create_worker_thread, category: :task
111
+ # NewRelic instrumentation
112
+ if Conf.newrelic_enabled?
113
+ include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
114
+ # add_transaction_tracer :create_conchita_thread, category: :task
115
+ add_transaction_tracer :create_thread, category: :task
107
116
  end
108
117
 
109
118
  end