rest-ftp-daemon 0.222.0 → 0.230.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CODE_OF_CONDUCT.md +13 -0
  4. data/Gemfile.lock +47 -20
  5. data/README.md +160 -94
  6. data/Rakefile +7 -1
  7. data/bin/rest-ftp-daemon +22 -3
  8. data/lib/rest-ftp-daemon.rb +25 -21
  9. data/lib/rest-ftp-daemon/constants.rb +19 -5
  10. data/lib/rest-ftp-daemon/exceptions.rb +2 -1
  11. data/lib/rest-ftp-daemon/helpers.rb +10 -5
  12. data/lib/rest-ftp-daemon/job.rb +181 -304
  13. data/lib/rest-ftp-daemon/job_queue.rb +5 -3
  14. data/lib/rest-ftp-daemon/logger.rb +4 -3
  15. data/lib/rest-ftp-daemon/logger_helper.rb +14 -10
  16. data/lib/rest-ftp-daemon/notification.rb +54 -43
  17. data/lib/rest-ftp-daemon/paginate.rb +2 -2
  18. data/lib/rest-ftp-daemon/path.rb +43 -0
  19. data/lib/rest-ftp-daemon/remote.rb +57 -0
  20. data/lib/rest-ftp-daemon/remote_ftp.rb +141 -0
  21. data/lib/rest-ftp-daemon/remote_sftp.rb +160 -0
  22. data/lib/rest-ftp-daemon/uri.rb +11 -4
  23. data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -1
  24. data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
  25. data/lib/rest-ftp-daemon/worker.rb +10 -2
  26. data/lib/rest-ftp-daemon/worker_conchita.rb +12 -6
  27. data/lib/rest-ftp-daemon/worker_job.rb +8 -11
  28. data/rest-ftp-daemon.gemspec +6 -1
  29. data/rest-ftp-daemon.yml.sample +4 -2
  30. data/spec/rest-ftp-daemon/features/dashboard_spec.rb +8 -4
  31. data/spec/rest-ftp-daemon/features/jobs_spec.rb +68 -0
  32. data/spec/rest-ftp-daemon/features/routes_spec.rb +20 -0
  33. data/spec/rest-ftp-daemon/features/status_spec.rb +19 -0
  34. data/spec/spec_helper.rb +6 -2
  35. data/spec/support/config.yml +0 -1
  36. data/spec/support/request_helpers.rb +22 -0
  37. metadata +53 -3
  38. data/.ruby-version +0 -1
data/Rakefile CHANGED
@@ -7,4 +7,10 @@ RSpec::Core::RakeTask.new(:spec)
7
7
 
8
8
  # Run specs by default
9
9
  desc 'Run all tests'
10
- task :default => :spec
10
+
11
+ require 'rubocop/rake_task'
12
+ RuboCop::RakeTask.new(:rubocop) do |task|
13
+ task.fail_on_error = false
14
+ end
15
+
16
+ task :default => [:spec, :rubocop]
data/bin/rest-ftp-daemon CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  # Try to load external libs
4
- app_root = File.dirname(__FILE__) + "/../"
4
+ app_root = File.expand_path File.dirname(__FILE__) + "/../"
5
5
  begin
6
6
  require "thin"
7
7
  require "optparse"
@@ -20,6 +20,18 @@ end
20
20
  puts
21
21
 
22
22
 
23
+ # Provide default config file information
24
+ DEFAULT_CONFIG_PATH = File.expand_path "/etc/#{APP_NAME}.yml"
25
+ SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/rest-ftp-daemon.yml.sample")
26
+ #SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/#{APP_NAME}.yml.sample")
27
+ TAIL_MESSAGE = <<EOD
28
+
29
+ A default configuration is available here: #{SAMPLE_CONFIG_FILE}.
30
+ You should copy it to the expected location #{DEFAULT_CONFIG_PATH}:
31
+
32
+ sudo cp #{SAMPLE_CONFIG_FILE} #{DEFAULT_CONFIG_PATH}
33
+ EOD
34
+
23
35
  # Detect options from ARGV
24
36
  options = {}
25
37
  parser = OptionParser.new do |opts|
@@ -35,11 +47,17 @@ parser = OptionParser.new do |opts|
35
47
  opts.on("-u", "--user NAME", "User to run daemon as (use with -g)") { |user| options["user"] = user }
36
48
  opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)"){ |group| options["group"] = group }
37
49
 
38
- opts.on_tail("-h", "--help", "Show this message") { puts opts; exit }
50
+ opts.separator ""
51
+ opts.on_tail("-h", "--help", "Show this message") do
52
+ puts opts
53
+ puts TAIL_MESSAGE unless File.exists?(DEFAULT_CONFIG_PATH)
54
+ exit
55
+ end
39
56
  opts.on_tail("-v", "--version", "Show version (#{APP_VER})") { puts APP_VER; exit }
40
57
  end
41
58
 
42
59
 
60
+
43
61
  # Parse options and check compliance
44
62
  begin
45
63
  parser.order!(ARGV)
@@ -52,7 +70,8 @@ end
52
70
 
53
71
 
54
72
  # Load config, and merge options from ARGV into settings
55
- APP_CONF ||= File.expand_path "/etc/#{APP_NAME}.yml"
73
+ # FIXME: file configuration detection could reside in settings.rb
74
+ APP_CONF ||= DEFAULT_CONFIG_PATH
56
75
  abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exists? APP_CONF
57
76
  begin
58
77
  # Import settings
@@ -12,6 +12,7 @@ require "timeout"
12
12
  require "sys/cpu"
13
13
  require "syslog"
14
14
  require "net/ftp"
15
+ require "net/sftp"
15
16
  require "net/http"
16
17
  require "double_bag_ftps"
17
18
  require "thread"
@@ -24,26 +25,29 @@ require "get_process_mem"
24
25
 
25
26
 
26
27
  # Project's libs
27
- require "rest-ftp-daemon/constants"
28
- require "rest-ftp-daemon/settings"
29
- require "rest-ftp-daemon/exceptions"
30
- require "rest-ftp-daemon/helpers"
31
- require "rest-ftp-daemon/logger_helper"
32
- require "rest-ftp-daemon/logger_pool"
33
- require "rest-ftp-daemon/logger"
34
- require "rest-ftp-daemon/paginate"
35
- require "rest-ftp-daemon/uri"
36
- require "rest-ftp-daemon/job_queue"
37
- require "rest-ftp-daemon/worker"
38
- require "rest-ftp-daemon/worker_conchita"
39
- require "rest-ftp-daemon/worker_job"
40
- require "rest-ftp-daemon/worker_pool"
41
- require "rest-ftp-daemon/job"
42
- require "rest-ftp-daemon/notification"
28
+ require_relative "rest-ftp-daemon/constants"
29
+ require_relative "rest-ftp-daemon/settings"
30
+ require_relative "rest-ftp-daemon/exceptions"
31
+ require_relative "rest-ftp-daemon/helpers"
32
+ require_relative "rest-ftp-daemon/logger_helper"
33
+ require_relative "rest-ftp-daemon/logger_pool"
34
+ require_relative "rest-ftp-daemon/logger"
35
+ require_relative "rest-ftp-daemon/paginate"
36
+ require_relative "rest-ftp-daemon/uri"
37
+ require_relative "rest-ftp-daemon/job_queue"
38
+ require_relative "rest-ftp-daemon/worker"
39
+ require_relative "rest-ftp-daemon/worker_conchita"
40
+ require_relative "rest-ftp-daemon/worker_job"
41
+ require_relative "rest-ftp-daemon/worker_pool"
42
+ require_relative "rest-ftp-daemon/job"
43
+ require_relative "rest-ftp-daemon/notification"
43
44
 
44
- require "rest-ftp-daemon/api/job_presenter"
45
- require "rest-ftp-daemon/api/jobs"
46
- require "rest-ftp-daemon/api/dashboard"
47
-
48
- require "rest-ftp-daemon/api/root"
45
+ require_relative "rest-ftp-daemon/path"
46
+ require_relative "rest-ftp-daemon/remote"
47
+ require_relative "rest-ftp-daemon/remote_ftp"
48
+ require_relative "rest-ftp-daemon/remote_sftp"
49
49
 
50
+ require_relative "rest-ftp-daemon/api/job_presenter"
51
+ require_relative "rest-ftp-daemon/api/jobs"
52
+ require_relative "rest-ftp-daemon/api/dashboard"
53
+ require_relative "rest-ftp-daemon/api/root"
@@ -1,19 +1,29 @@
1
1
  # Terrific constants
2
2
  APP_NAME = "rest-ftp-daemon"
3
3
  APP_NICK = "rftpd"
4
- APP_VER = "0.222.0"
4
+ APP_VER = "0.230.0"
5
5
 
6
6
 
7
7
  # Jobs and workers
8
8
  JOB_RANDOM_LEN = 8
9
9
  JOB_IDENT_LEN = 4
10
10
  JOB_TEMPFILE_LEN = 8
11
+ JOB_UPDATE_INTERVAL = 1
12
+
11
13
  JOB_STATUS_UPLOADING = :uploading
12
14
  JOB_STATUS_RENAMING = :renaming
15
+ JOB_STATUS_PREPARED = :prepared
13
16
  JOB_STATUS_FINISHED = :finished
14
17
  JOB_STATUS_FAILED = :failed
15
18
  JOB_STATUS_QUEUED = :queued
16
19
 
20
+ WORKER_STATUS_STARTING = :starting
21
+ WORKER_STATUS_WAITING = :waiting
22
+ WORKER_STATUS_RUNNING = :running
23
+ WORKER_STATUS_FINISHED = :finished
24
+ WORKER_STATUS_TIMEOUT = :timeout
25
+ WORKER_STATUS_CRASHED = :crashed
26
+ WORKER_STATUS_CLEANING = :cleaning
17
27
 
18
28
  # Logging and startup
19
29
  LOG_PIPE_LEN = 10
@@ -28,6 +38,10 @@ LOG_FORMAT_PREFIX = "%s %s\t%-#{LOG_PIPE_LEN.to_i}s\t"
28
38
  LOG_FORMAT_MESSAGE = "%#{-LOG_COL_WID.to_i}s\t%#{-LOG_COL_JID.to_i}s\t%#{-LOG_COL_ID.to_i}s"
29
39
  LOG_NEWLINE = "\n"
30
40
 
41
+ LOG_INDENT = "\t"
42
+
43
+
44
+
31
45
  # Notifications
32
46
  NOTIFY_PREFIX = "rftpd"
33
47
  NOTIFY_USERAGENT = "#{APP_NAME} - #{APP_VER}"
@@ -35,26 +49,26 @@ NOTIFY_IDENTIFIER_LEN = 4
35
49
 
36
50
 
37
51
  # Dashboard row styles
38
- JOB_STYLES = {
52
+ DASHBOARD_JOB_STYLES = {
39
53
  JOB_STATUS_QUEUED => :active,
40
54
  JOB_STATUS_FAILED => :warning,
41
55
  JOB_STATUS_FINISHED => :success,
42
56
  JOB_STATUS_UPLOADING => :info,
43
57
  JOB_STATUS_RENAMING => :info,
44
58
  }
45
- WORKER_STYLES = {
59
+ DASHBOARD_WORKER_STYLES = {
46
60
  waiting: :success,
47
61
  working: :info,
48
62
  crashed: :danger,
49
63
  done: :success,
50
64
  dead: :danger
51
65
  }
52
- PAGINATE_MAX = 30
53
66
 
54
67
 
55
68
  # Configuration defaults
56
69
  DEFAULT_WORKER_TIMEOUT = 3600
57
- DEFAULT_FTP_CHUNK = 2048
70
+ DEFAULT_FTP_CHUNK = 512
71
+ DEFAULT_PAGE_SIZE = 40
58
72
 
59
73
 
60
74
  # Initialize defaults
@@ -19,7 +19,8 @@ module RestFtpDaemon
19
19
  class JobTargetUnsupported < RestFtpDaemonException; end
20
20
  class JobTargetUnparseable < RestFtpDaemonException; end
21
21
  class JobTargetFileExists < RestFtpDaemonException; end
22
- class JobTargetShouldBeDirectory< RestFtpDaemonException; end
22
+ class JobTargetDirectoryError < RestFtpDaemonException; end
23
+ class JobTargetPermissionError < RestFtpDaemonException; end
23
24
  class JobTooManyOpenFiles < RestFtpDaemonException; end
24
25
 
25
26
  end
@@ -31,29 +31,33 @@ module RestFtpDaemon
31
31
  end
32
32
 
33
33
  def self.tokenize item
34
+ return unless item.is_a? String
34
35
  "[#{item}]"
35
36
  end
36
37
 
37
38
  def self.highlight_tokens path
39
+ return unless path.is_a? String
38
40
  path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
39
41
  end
40
42
 
41
43
  def self.extract_filename path
44
+ return unless path.is_a? String
42
45
  # match everything that's after a slash at the end of the string
43
46
  m = path.match /\/?([^\/]+)$/
44
47
  return m[1] unless m.nil?
45
48
  end
46
49
 
47
50
  def self.extract_dirname path
51
+ return unless path.is_a? String
48
52
  # match all the beginning of the string up to the last slash
49
53
  m = path.match(/^(.*)\/[^\/]*$/)
50
54
  return "/#{m[1]}" unless m.nil?
51
55
  end
52
56
 
53
57
  def self.extract_parent path
54
- # m = path.match(/^(.*\/)[^\/]*\/+$/)
55
- m = path.match(/^(.*\/)[^\/]+\/?$/)
56
- return m[1] unless m.nil?
58
+ return unless path.is_a? String
59
+ m = path.match(/^(.*)\/([^\/]+)\/?$/)
60
+ return m[1], m[2] unless m.nil?
57
61
  end
58
62
 
59
63
  def self.local_port_used? port
@@ -106,8 +110,9 @@ module RestFtpDaemon
106
110
  datetime.to_datetime.strftime("%d/%m %H:%M:%S")
107
111
  end
108
112
 
109
- def self.hide_credentials_from_url url
110
- url.sub(/([a-z]+:\/\/[^\/]+):[^\/]+\@/, '\1@' )
113
+ def self.hide_credentials_from_url path
114
+ return unless path.is_a? String
115
+ path.sub(/([a-z]+:\/\/[^\/]+):[^\/]+\@/, '\1@' )
111
116
  end
112
117
 
113
118
  end
@@ -57,24 +57,22 @@ module RestFtpDaemon
57
57
  flag_default :tempfile, false
58
58
 
59
59
  # Read source file size and parameters
60
- @ftp_debug_enabled = (Settings.at :debug, :ftp) == true
61
- update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_FTP_CHUNK
62
60
  @notify_after_sec = Settings.transfer.notify_after_sec rescue nil
63
- @chunk_size = update_every_kb * 1024
61
+ @chunk_size = DEFAULT_FTP_CHUNK * 1024
64
62
 
65
63
  # Flag current job
66
64
  @queued_at = Time.now
67
65
  @updated_at = Time.now
68
66
 
69
67
  # Send first notification
70
- log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] update_every_kb[#{@update_every_kb}]"
68
+ log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] JOB_UPDATE_INTERVAL[#{JOB_UPDATE_INTERVAL}]"
71
69
  client_notify :queued
72
70
  end
73
71
 
74
72
  def process
75
73
  # Update job's status
76
74
  @error = nil
77
- log_info "Job.process starting"
75
+ log_info "Job.process"
78
76
 
79
77
  # Prepare job
80
78
  begin
@@ -99,20 +97,10 @@ module RestFtpDaemon
99
97
  rescue RestFtpDaemon::JobAssertionFailed => exception
100
98
  return oops :started, exception, :assertion_failed
101
99
 
102
- # rescue RestFtpDaemon::JobTimeout => exception
103
- # info "Job.process propagate JobTimeout to Worker"
104
- # raise RestFtpDaemon::JobTimeout
105
-
106
- # rescue RestFtpDaemon::RestFtpDaemonException => exception
107
- # return oops :started, exception, :prepare_failed, true
108
-
109
- # rescue StandardError => exception
110
- # return oops :started, exception, :prepare_unhandled, true
111
-
112
100
  else
113
101
  # Prepare done !
114
- newstatus :prepared
115
- log_info "Job.process notify[started]"
102
+ newstatus JOB_STATUS_PREPARED
103
+ log_info "Job.process notify [started]"
116
104
  client_notify :started
117
105
  end
118
106
 
@@ -154,6 +142,9 @@ module RestFtpDaemon
154
142
  rescue Net::FTPTempError => exception
155
143
  return oops :ended, exception, :net_temp_error
156
144
 
145
+ rescue Net::SFTP::StatusException => exception
146
+ return oops :ended, exception, :sftp_exception
147
+
157
148
  rescue Errno::EMFILE => exception
158
149
  return oops :ended, exception, :too_many_open_files
159
150
 
@@ -169,26 +160,19 @@ module RestFtpDaemon
169
160
  rescue RestFtpDaemon::JobTargetFileExists => exception
170
161
  return oops :ended, exception, :target_file_exists
171
162
 
172
- rescue RestFtpDaemon::JobTargetShouldBeDirectory => exception
173
- return oops :ended, exception, :target_not_directory
163
+ rescue RestFtpDaemon::JobTargetDirectoryError => exception
164
+ return oops :ended, exception, :target_directory_missing
165
+
166
+ rescue RestFtpDaemon::JobTargetPermissionError => exception
167
+ return oops :ended, exception, :target_permission_error
174
168
 
175
169
  rescue RestFtpDaemon::JobAssertionFailed => exception
176
170
  return oops :ended, exception, :assertion_failed
177
171
 
178
- # rescue RestFtpDaemon::JobTimeout => exception
179
- # info "Job.process propagate JobTimeout to Worker"
180
- # raise RestFtpDaemon::JobTimeout
181
-
182
- # rescue RestFtpDaemon::RestFtpDaemonException => exception
183
- # return oops :ended, exception, :transfer_failed, true
184
-
185
- # rescue StandardError => exception
186
- # return oops :ended, exception, :transfer_unhandled, true
187
-
188
172
  else
189
173
  # All done !
190
174
  newstatus JOB_STATUS_FINISHED
191
- log_info "Job.process notify[ended]"
175
+ log_info "Job.process notify [ended]"
192
176
  client_notify :ended
193
177
  end
194
178
 
@@ -274,103 +258,115 @@ module RestFtpDaemon
274
258
 
275
259
  def prepare
276
260
  # Update job status
277
- newstatus :preparing
261
+ newstatus :prepare
278
262
 
279
263
  # Init
280
264
  @source_method = :file
281
265
  @target_method = nil
282
266
  @source_path = nil
283
- @target_url = nil
284
267
 
285
- # Check source
268
+ # Prepare source
286
269
  raise RestFtpDaemon::JobMissingAttribute unless @source
287
270
  @source_path = expand_path @source
288
271
  set :source_path, @source_path
289
272
  set :source_method, :file
290
273
 
291
- # Check target
274
+ # Prepare target
292
275
  raise RestFtpDaemon::JobMissingAttribute unless @target
293
- @target_url = expand_url @target
294
- set :target_url, @target_url.to_s
295
-
296
- if @target_url.is_a? URI::FTP
297
- @target_method = :ftp
298
- elsif @target_url.is_a? URI::FTPES
299
- @target_method = :ftps
300
- elsif @target_url.is_a? URI::FTPS
301
- @target_method = :ftps
302
- end
303
- set :target_method, @target_method
276
+ target_uri = expand_url @target
277
+ set :target_uri, target_uri.to_s
278
+ @target_path = Path.new target_uri.path, true
279
+
280
+ #puts "@target_path: #{@target_path.inspect}"
281
+
282
+ # Prepare remote
283
+ newstatus :remote_init
284
+ #FIXME: use a "case" statement on @target_url.class
285
+
286
+ if target_uri.is_a? URI::FTP
287
+ log_info "Job.prepare target_method FTP"
288
+ set :target_method, :ftp
289
+ @remote = RemoteFTP.new target_uri, log_context
290
+
291
+ elsif (target_uri.is_a? URI::FTPES) || (target_uri.is_a? URI::FTPS)
292
+ log_info "Job.prepare target_method FTPES"
293
+ set :target_method, :ftpes
294
+ @remote = RemoteFTP.new target_uri, log_context, ftpes: true
304
295
 
305
- # Check compliance
306
- raise RestFtpDaemon::JobTargetUnparseable if @target_url.nil?
307
- raise RestFtpDaemon::JobTargetUnsupported if @target_method.nil?
296
+ elsif target_uri.is_a? URI::SFTP
297
+ log_info "Job.prepare target_method SFTP"
298
+ set :target_method, :sftp
299
+ @remote = RemoteSFTP.new target_uri, log_context
300
+
301
+ else
302
+ log_info "Job.prepare unknown scheme [#{target_uri.scheme}]"
303
+ raise RestFtpDaemon::JobTargetUnsupported
304
+
305
+ end
308
306
  end
309
307
 
310
308
  def transfer
311
309
  # Update job status
312
- newstatus :checking_source
310
+ #log_info "Job.transfer starting"
313
311
  @started_at = Time.now
314
312
 
315
313
  # Method assertions and init
316
314
  raise RestFtpDaemon::JobAssertionFailed, "transfer/1" unless @source_path
317
- raise RestFtpDaemon::JobAssertionFailed, "transfer/2" unless @target_url
315
+ raise RestFtpDaemon::JobAssertionFailed, "transfer/2" unless @target_path
318
316
  @transfer_sent = 0
319
317
  set :source_processed, 0
320
318
 
321
- # Guess source file names using Dir.glob
322
- source_matches = Dir.glob @source_path
323
-
324
- # Log detected names
325
- source_names = source_matches.map{ |s| Helpers.extract_filename s }
326
- log_info "Job.transfer sources #{source_names.inspect}"
327
-
328
- # Asserts and counters
329
- raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
330
- set :source_count, source_matches.count
331
- set :source_files, source_matches
319
+ # Guess source files from disk
320
+ newstatus :checking_source
321
+ sources = find_local @source_path
322
+ set :source_count, sources.count
323
+ set :source_files, sources.collect(&:full)
324
+ log_info "Job.transfer sources #{sources.collect(&:name)}"
325
+ #log_info "Job.transfer target #{target.full}"
326
+ raise RestFtpDaemon::JobSourceNotFound if sources.empty?
332
327
 
333
328
  # Guess target file name, and fail if present while we matched multiple sources
334
- target_name = Helpers.extract_filename @target_url.path
335
- raise RestFtpDaemon::JobTargetShouldBeDirectory if target_name && source_matches.count>1
336
-
337
- # Scheme-aware config
338
- ftp_init
329
+ raise RestFtpDaemon::JobTargetDirectoryError if @target_path.name && sources.count>1
339
330
 
340
331
  # Connect to remote server and login
341
- ftp_connect
342
- ftp_login
343
-
344
- # Change to the right path
345
- path = Helpers.extract_dirname(@target_url.path).to_s
346
- ftp_chdir_or_buildpath path
332
+ newstatus :remote_connect
333
+ #log_info "Job.remote_connect" # [#{host}] [#{login}]"
334
+ @remote.connect
347
335
 
348
- # Check source files presence and compute total size, they should be there, coming from Dir.glob()
349
- @transfer_total = 0
350
- source_matches.each do |filename|
351
- raise RestFtpDaemon::JobSourceNotReadable unless File.readable? filename
352
- raise RestFtpDaemon::JobSourceNotReadable unless File.file? filename
336
+ # Prepare target path or build it if asked
337
+ #log_info "Job.remote_chdir"
338
+ newstatus :remote_chdir
339
+ @remote.chdir_or_create @target_path.dir, @mkdir
353
340
 
354
- # Skip if not a flie
355
- next unless File.file? filename
356
-
357
- @transfer_total += File.size filename
358
- end
341
+ # Compute total files size
342
+ @transfer_total = sources.collect(&:size).sum
359
343
  set :transfer_total, @transfer_total
360
344
 
345
+ # Reset counters
346
+ @last_data = 0
347
+ @last_time = Time.now
348
+
361
349
  # Handle each source file matched, and start a transfer
362
- done = 0
363
- source_matches.each do |filename|
364
- # Do the transfer, only if it's a file
365
- ftp_transfer filename, target_name
350
+ source_processed = 0
351
+ sources.each do |source|
352
+ # Compute target filename
353
+ full_target = @target_path.clone
354
+
355
+ # Add the source file name if none found in the target path
356
+ unless full_target.name
357
+ full_target.name = source.name
358
+ end
359
+
360
+ # Do the transfer, for each file
361
+ #log_info "Job.remote_push"
362
+ remote_push source, full_target
366
363
 
367
364
  # Update counters
368
- done += 1
369
- set :source_processed, done
365
+ set :source_processed, source_processed += 1
370
366
  end
371
367
 
372
368
  # FTP transfer finished
373
- ftp_finished
369
+ finalize
374
370
  end
375
371
 
376
372
 
@@ -383,6 +379,18 @@ module RestFtpDaemon
383
379
  }
384
380
  end
385
381
 
382
+ def find_local path
383
+ Dir.glob(path).collect do |file|
384
+ next unless File.readable? file
385
+ next unless File.file? file
386
+ Path.new file
387
+ end
388
+ end
389
+
390
+ def worker_is_still_active
391
+ Thread.current.thread_variable_set :updted_at, Time.now
392
+ end
393
+
386
394
  def newstatus name
387
395
  @status = name
388
396
  worker_is_still_active
@@ -399,57 +407,13 @@ module RestFtpDaemon
399
407
  instance_variable_set variable, default
400
408
  end
401
409
 
402
- def ftp_init
403
- # Update job status
404
- newstatus :ftp_init
405
-
406
- # Method assertions
407
- raise RestFtpDaemon::JobAssertionFailed, "ftp_init/1" if @target_method.nil?
408
- raise RestFtpDaemon::JobAssertionFailed, "ftp_init/2" if @target_url.nil?
409
-
410
- log_info "Job.ftp_init target_method [#{@target_method}]"
411
- case @target_method
412
- when :ftp
413
- @ftp = Net::FTP.new
414
- when :ftps
415
- @ftp = DoubleBagFTPS.new
416
- @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
417
- @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
418
- else
419
- log_info "Job.ftp_init unknown scheme [#{@target_url.scheme}]"
420
- railse RestFtpDaemon::JobTargetUnsupported
421
- end
422
-
423
- # FTP debug mode ?
424
- if @ftp_debug_enabled
425
- # Output header to STDOUT
426
- puts
427
- puts "-------------------- FTP SESSION STARTING --------------------"
428
- puts "job id\t #{@id}"
429
- puts "source\t #{@source}"
430
- puts "target\t #{@target}"
431
- puts "host\t #{@target_url.host}"
432
- puts "user\t #{@target_url.user}"
433
- puts "--------------------------------------------------------------"
434
-
435
- # Set debug mode on connection
436
- @ftp.debug_mode = true
437
- end
438
-
439
- # Activate passive mode
440
- @ftp.passive = true
441
- end
442
-
443
- def ftp_finished
410
+ def finalize
444
411
  # Close FTP connexion and free up memory
445
- @ftp.close
446
- log_info "Job.ftp_finished closed"
447
- @ftp = nil
412
+ log_info "Job.finalize"
413
+ @remote.close
448
414
 
449
- # FTP debug mode ?
450
- if @ftp_debug_enabled
451
- puts "-------------------- FTP SESSION ENDED -----------------------"
452
- end
415
+ # Free-up remote object
416
+ @remote = nil
453
417
 
454
418
  # Update job status
455
419
  newstatus :disconnecting
@@ -460,206 +424,106 @@ module RestFtpDaemon
460
424
  $queue.counter_add :transferred, @transfer_total
461
425
  end
462
426
 
463
- def ftp_connect
464
- # Update job status
465
- newstatus :ftp_connect
466
-
467
- # Method assertions
468
- host = @target_url.host
469
- log_info "Job.ftp_connect [#{host}]"
470
- raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/1" if @ftp.nil?
471
- raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/2" if @target_url.nil?
472
-
473
- @ftp.connect(host)
474
- end
475
-
476
- def ftp_login
477
- # Update job status
478
- newstatus :ftp_login
479
-
480
- # Method assertions
481
- raise RestFtpDaemon::JobAssertionFailed, "ftp_login/1" if @ftp.nil?
482
-
483
- # use "anonymous" if user is empty
484
- login = @target_url.user || "anonymous"
485
- log_info "Job.ftp_login [#{login}]"
486
-
487
- @ftp.login login, @target_url.password
488
- end
489
-
490
- def ftp_chdir_or_buildpath path
491
- # Method assertions
492
- log_info "Job.ftp_chdir [#{path}] mkdir: #{@mkdir}"
493
- newstatus :ftp_chdir
494
- raise RestFtpDaemon::JobAssertionFailed, "ftp_chdir_or_buildpath/1" if path.nil?
495
-
496
- # Extract directory from path
497
- if @mkdir
498
- # Split dir in parts
499
- log_info "Job.ftp_chdir buildpath [#{path}]"
500
- ftp_buildpath path
501
- else
502
- # Directly chdir if not mkdir requested
503
- log_info "Job.ftp_chdir chdir [#{path}]"
504
- @ftp.chdir path
505
- end
506
- end
507
-
508
- def ftp_buildpath path
509
- # Init
510
- pref = "Job.ftp_buildpath [#{path}]"
511
-
512
- begin
513
- # Try to chdir in this directory
514
- @ftp.chdir(path)
515
-
516
- rescue Net::FTPPermError
517
- # If not possible because the directory is missing
518
- parent = Helpers.extract_parent(path)
519
- log_info "#{pref} chdir failed - parent [#{parent}]"
520
-
521
- # And only if we still have something to "dive up into"
522
- if parent
523
- # Do the same for the parent
524
- ftp_buildpath parent
525
-
526
- # Then finally create this dir and chdir
527
- log_info "#{pref} > now mkdir [#{path}]"
528
- @ftp.mkdir path
529
-
530
- # And get into it (this chdir is not rescue'd on purpose in order to throw the ex)
531
- log_info "#{pref} > now chdir [#{path}]"
532
- @ftp.chdir(path)
533
- end
534
-
535
- end
536
-
537
- # Now we were able to chdir inside, just tell it
538
- log_info "#{pref} > ftp.pwd [#{@ftp.pwd}]"
539
- end
540
-
541
- def ftp_presence target_name
542
- # Update job status
543
- newstatus :ftp_presence
544
- # FIXME / TODO: try with nlst
545
-
546
- # Method assertions
547
- raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/1" if @ftp.nil?
548
- raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/2" if @target_url.nil?
549
-
550
- # Get file list, sometimes the response can be an empty value
551
- results = @ftp.list(target_name) rescue nil
552
- log_info "Job.ftp_presence: #{results.inspect}"
553
-
554
- # Result can be nil or a list of files
555
- return false if results.nil?
556
-
557
- results.count >0
558
- end
559
-
560
- def ftp_transfer source_filename, target_name = nil
427
+ def remote_push source, target
561
428
  # Method assertions
562
- log_info "Job.ftp_transfer source: #{source_filename}"
563
- raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/1" if @ftp.nil?
564
- raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/2" if source_filename.nil?
429
+ raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/1" if @remote.nil?
430
+ raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/2" if source.nil?
431
+ raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/3" if target.nil?
565
432
 
566
433
  # Use source filename if target path provided none (typically with multiple sources)
567
- target_name ||= Helpers.extract_filename source_filename
568
- log_info "Job.ftp_transfer target: #{target_name}"
569
- set :source_current, target_name
570
-
571
- # Check for target file presence
572
- newstatus :checking_target
573
- present = ftp_presence target_name
574
- if present
575
- if @overwrite
576
- # delete it first
577
- log_info "Job.ftp_transfer removing target file"
578
- @ftp.delete(target_name)
579
- else
580
- # won't overwrite then stop here
581
- log_info "Job.ftp_transfer failed: target file exists"
582
- raise RestFtpDaemon::JobTargetFileExists
583
- end
584
- end
434
+ log_info "Job.remote_push [#{source.name}]: [#{source.full}] > [#{target.full}]"
435
+ set :source_current, source.name
585
436
 
586
437
  # Compute temp target name
587
- target_real = target_name
438
+ tempname = nil
588
439
  if @tempfile
589
- target_real = "#{target_name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
590
- log_info "Job.ftp_transfer tempfile: #{target_real}"
440
+ tempname = "#{target.name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
441
+ #log_info "Job.remote_push tempname [#{tempname}]"
442
+ else
443
+ end
444
+
445
+ # Remove any existing version if expected, or test its presence
446
+ if @overwrite
447
+ @remote.remove! target
448
+ elsif size = @remote.present?(target)
449
+ log_info "Job.remote_push existing (#{Helpers.format_bytes size, 'B'})"
450
+ raise RestFtpDaemon::JobTargetFileExists
591
451
  end
592
452
 
593
453
  # Start transfer
594
454
  transfer_started_at = Time.now
595
- @transfer_pointer_at = transfer_started_at
596
-
597
- @notified_at = Time.now
455
+ @progress_at = 0
456
+ @notified_at = transfer_started_at
598
457
  newstatus JOB_STATUS_UPLOADING
599
458
 
600
- @ftp.putbinaryfile(source_filename, target_real, @chunk_size) do |block|
601
- # Update the worker activity marker
602
- worker_is_still_active
603
-
604
- # Update job status after this block transfer
605
- ftp_transfer_block block
606
- end
459
+ # Start the transfer, update job status after each block transfer
460
+ newstatus :uploading
461
+ @remote.push source, target, tempname do |transferred, name|
462
+ # Update transfer statistics
463
+ progress transferred, name
607
464
 
608
- # Rename temp file to target_temp
609
- if @tempfile
610
- newstatus JOB_STATUS_RENAMING
611
- log_info "Job.ftp_transfer renaming: #{target_name}"
612
- @ftp.rename target_real, target_name
465
+ # Touch my worker status
466
+ worker_is_still_active
613
467
  end
614
468
 
615
469
  # Compute final bitrate
616
- set :transfer_bitrate, get_bitrate(@transfer_total, transfer_started_at).round(0)
470
+ global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
471
+ set :transfer_bitrate, global_transfer_bitrate.round(0)
617
472
 
618
473
  # Done
619
474
  set :source_current, nil
620
- log_info "Job.ftp_transfer finished"
475
+ #log_info "Job.remote_push finished"
621
476
  end
622
477
 
623
- def ftp_transfer_block block
478
+ def progress transferred, name = ""
479
+ # What's current time ?
480
+ now = Time.now
481
+
624
482
  # Update counters
625
- @transfer_sent += block.bytesize
483
+ @transfer_sent += transferred
626
484
  set :transfer_sent, @transfer_sent
627
485
 
628
- # Update bitrate
629
- #dt = Time.now - t0
630
- bitrate0 = get_bitrate(@chunk_size, @transfer_pointer_at).round(0)
631
- set :transfer_bitrate, bitrate0
632
-
633
486
  # Update job info
634
487
  percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
635
488
  set :progress, percent0
636
489
 
637
- # Log progress
638
- stack = []
639
- stack << "#{percent0} %"
640
- stack << (Helpers.format_bytes @transfer_sent, "B")
641
- stack << (Helpers.format_bytes @transfer_total, "B")
642
- stack << (Helpers.format_bytes bitrate0, "bps")
643
- stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
644
- log_info "Job.ftp_transfer #{stack2}"
645
-
646
- # Update time pointer
647
- @transfer_pointer_at = Time.now
490
+ # Update job status after each NOTIFY_UPADE_STATUS
491
+ progressed_ago = (now.to_f - @progress_at.to_f)
492
+ if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
493
+ @current_bitrate = running_bitrate @transfer_sent
494
+ set :transfer_bitrate, @current_bitrate.round(0)
495
+
496
+ # Log progress
497
+ stack = []
498
+ stack << "#{percent0} %"
499
+ stack << (Helpers.format_bytes @transfer_sent, "B")
500
+ stack << (Helpers.format_bytes @transfer_total, "B")
501
+ stack << (Helpers.format_bytes @current_bitrate.round(0), "bps")
502
+ stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
503
+ log_info "#{LOG_INDENT}progress #{stack2} \t#{name}"
504
+
505
+ # Remember when we last did it
506
+ @progress_at = now
507
+ end
648
508
 
649
509
  # Notify if requested
650
- # info "Job.ftp_transfer next notif (#{(@notified_at+@notify_after_sec).to_f}) now #{Time.now.to_f}"
651
- if @notify_after_sec.nil? || (Time.now > @notified_at + @notify_after_sec)
510
+ notified_ago = (now.to_f - @notified_at.to_f)
511
+ if (!@notify_after_sec.nil?) && (notified_ago > @notify_after_sec)
512
+ # Prepare and send notification
652
513
  notif_status = {
653
514
  progress: percent0,
654
515
  transfer_sent: @transfer_sent,
655
516
  transfer_total: @transfer_total,
656
- transfer_bitrate: bitrate0
517
+ transfer_bitrate: @current_bitrate
657
518
  }
658
519
  client_notify :progress, status: notif_status
659
- @notified_at = Time.now
520
+
521
+ # Remember when we last did it
522
+ @notified_at = now
660
523
  end
661
524
  end
662
525
 
526
+
663
527
  def client_notify event, payload = {}
664
528
  # Skip if no URL given
665
529
  return unless @notify
@@ -673,12 +537,25 @@ module RestFtpDaemon
673
537
  log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
674
538
  end
675
539
 
676
- def get_bitrate total, last_timestamp
677
- 8*total.to_f / (Time.now - last_timestamp)
540
+ def get_bitrate delta_data, delta_time
541
+ return nil if delta_time.nil? || delta_time.zero?
542
+ 8 * delta_data.to_f.to_f / delta_time
678
543
  end
679
544
 
680
- def worker_is_still_active
681
- Thread.current.thread_variable_set :updted_at, Time.now
545
+ def running_bitrate current_data
546
+ return if @last_time.nil?
547
+
548
+ # Compute deltas
549
+ @last_data ||= 0
550
+ delta_data = current_data - @last_data
551
+ delta_time = Time.now - @last_time
552
+
553
+ # Update counters
554
+ @last_time = Time.now
555
+ @last_data = current_data
556
+
557
+ # Return bitrate
558
+ get_bitrate delta_data, delta_time
682
559
  end
683
560
 
684
561
  def oops event, exception, error = nil, include_backtrace = false
@@ -693,7 +570,7 @@ module RestFtpDaemon
693
570
  end
694
571
 
695
572
  # Close ftp connexion if open
696
- @ftp.close unless @ftp.nil? || @ftp.welcome.nil?
573
+ @remote.close unless @remote.nil? || !@remote.connected?
697
574
 
698
575
  # Update job's internal status
699
576
  newstatus JOB_STATUS_FAILED