rest-ftp-daemon 0.222.0 → 0.230.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/Gemfile.lock +47 -20
- data/README.md +160 -94
- data/Rakefile +7 -1
- data/bin/rest-ftp-daemon +22 -3
- data/lib/rest-ftp-daemon.rb +25 -21
- data/lib/rest-ftp-daemon/constants.rb +19 -5
- data/lib/rest-ftp-daemon/exceptions.rb +2 -1
- data/lib/rest-ftp-daemon/helpers.rb +10 -5
- data/lib/rest-ftp-daemon/job.rb +181 -304
- data/lib/rest-ftp-daemon/job_queue.rb +5 -3
- data/lib/rest-ftp-daemon/logger.rb +4 -3
- data/lib/rest-ftp-daemon/logger_helper.rb +14 -10
- data/lib/rest-ftp-daemon/notification.rb +54 -43
- data/lib/rest-ftp-daemon/paginate.rb +2 -2
- data/lib/rest-ftp-daemon/path.rb +43 -0
- data/lib/rest-ftp-daemon/remote.rb +57 -0
- data/lib/rest-ftp-daemon/remote_ftp.rb +141 -0
- data/lib/rest-ftp-daemon/remote_sftp.rb +160 -0
- data/lib/rest-ftp-daemon/uri.rb +11 -4
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -1
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
- data/lib/rest-ftp-daemon/worker.rb +10 -2
- data/lib/rest-ftp-daemon/worker_conchita.rb +12 -6
- data/lib/rest-ftp-daemon/worker_job.rb +8 -11
- data/rest-ftp-daemon.gemspec +6 -1
- data/rest-ftp-daemon.yml.sample +4 -2
- data/spec/rest-ftp-daemon/features/dashboard_spec.rb +8 -4
- data/spec/rest-ftp-daemon/features/jobs_spec.rb +68 -0
- data/spec/rest-ftp-daemon/features/routes_spec.rb +20 -0
- data/spec/rest-ftp-daemon/features/status_spec.rb +19 -0
- data/spec/spec_helper.rb +6 -2
- data/spec/support/config.yml +0 -1
- data/spec/support/request_helpers.rb +22 -0
- metadata +53 -3
- data/.ruby-version +0 -1
data/Rakefile
CHANGED
@@ -7,4 +7,10 @@ RSpec::Core::RakeTask.new(:spec)
|
|
7
7
|
|
8
8
|
# Run specs by default
|
9
9
|
desc 'Run all tests'
|
10
|
-
|
10
|
+
|
11
|
+
require 'rubocop/rake_task'
|
12
|
+
RuboCop::RakeTask.new(:rubocop) do |task|
|
13
|
+
task.fail_on_error = false
|
14
|
+
end
|
15
|
+
|
16
|
+
task :default => [:spec, :rubocop]
|
data/bin/rest-ftp-daemon
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
# Try to load external libs
|
4
|
-
app_root = File.dirname(__FILE__) + "/../"
|
4
|
+
app_root = File.expand_path File.dirname(__FILE__) + "/../"
|
5
5
|
begin
|
6
6
|
require "thin"
|
7
7
|
require "optparse"
|
@@ -20,6 +20,18 @@ end
|
|
20
20
|
puts
|
21
21
|
|
22
22
|
|
23
|
+
# Provide default config file information
|
24
|
+
DEFAULT_CONFIG_PATH = File.expand_path "/etc/#{APP_NAME}.yml"
|
25
|
+
SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/rest-ftp-daemon.yml.sample")
|
26
|
+
#SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/#{APP_NAME}.yml.sample")
|
27
|
+
TAIL_MESSAGE = <<EOD
|
28
|
+
|
29
|
+
A default configuration is available here: #{SAMPLE_CONFIG_FILE}.
|
30
|
+
You should copy it to the expected location #{DEFAULT_CONFIG_PATH}:
|
31
|
+
|
32
|
+
sudo cp #{SAMPLE_CONFIG_FILE} #{DEFAULT_CONFIG_PATH}
|
33
|
+
EOD
|
34
|
+
|
23
35
|
# Detect options from ARGV
|
24
36
|
options = {}
|
25
37
|
parser = OptionParser.new do |opts|
|
@@ -35,11 +47,17 @@ parser = OptionParser.new do |opts|
|
|
35
47
|
opts.on("-u", "--user NAME", "User to run daemon as (use with -g)") { |user| options["user"] = user }
|
36
48
|
opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)"){ |group| options["group"] = group }
|
37
49
|
|
38
|
-
opts.
|
50
|
+
opts.separator ""
|
51
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
52
|
+
puts opts
|
53
|
+
puts TAIL_MESSAGE unless File.exists?(DEFAULT_CONFIG_PATH)
|
54
|
+
exit
|
55
|
+
end
|
39
56
|
opts.on_tail("-v", "--version", "Show version (#{APP_VER})") { puts APP_VER; exit }
|
40
57
|
end
|
41
58
|
|
42
59
|
|
60
|
+
|
43
61
|
# Parse options and check compliance
|
44
62
|
begin
|
45
63
|
parser.order!(ARGV)
|
@@ -52,7 +70,8 @@ end
|
|
52
70
|
|
53
71
|
|
54
72
|
# Load config, and merge options from ARGV into settings
|
55
|
-
|
73
|
+
# FIXME: file configuration detection could reside in settings.rb
|
74
|
+
APP_CONF ||= DEFAULT_CONFIG_PATH
|
56
75
|
abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exists? APP_CONF
|
57
76
|
begin
|
58
77
|
# Import settings
|
data/lib/rest-ftp-daemon.rb
CHANGED
@@ -12,6 +12,7 @@ require "timeout"
|
|
12
12
|
require "sys/cpu"
|
13
13
|
require "syslog"
|
14
14
|
require "net/ftp"
|
15
|
+
require "net/sftp"
|
15
16
|
require "net/http"
|
16
17
|
require "double_bag_ftps"
|
17
18
|
require "thread"
|
@@ -24,26 +25,29 @@ require "get_process_mem"
|
|
24
25
|
|
25
26
|
|
26
27
|
# Project's libs
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
28
|
+
require_relative "rest-ftp-daemon/constants"
|
29
|
+
require_relative "rest-ftp-daemon/settings"
|
30
|
+
require_relative "rest-ftp-daemon/exceptions"
|
31
|
+
require_relative "rest-ftp-daemon/helpers"
|
32
|
+
require_relative "rest-ftp-daemon/logger_helper"
|
33
|
+
require_relative "rest-ftp-daemon/logger_pool"
|
34
|
+
require_relative "rest-ftp-daemon/logger"
|
35
|
+
require_relative "rest-ftp-daemon/paginate"
|
36
|
+
require_relative "rest-ftp-daemon/uri"
|
37
|
+
require_relative "rest-ftp-daemon/job_queue"
|
38
|
+
require_relative "rest-ftp-daemon/worker"
|
39
|
+
require_relative "rest-ftp-daemon/worker_conchita"
|
40
|
+
require_relative "rest-ftp-daemon/worker_job"
|
41
|
+
require_relative "rest-ftp-daemon/worker_pool"
|
42
|
+
require_relative "rest-ftp-daemon/job"
|
43
|
+
require_relative "rest-ftp-daemon/notification"
|
43
44
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
require "rest-ftp-daemon/api/root"
|
45
|
+
require_relative "rest-ftp-daemon/path"
|
46
|
+
require_relative "rest-ftp-daemon/remote"
|
47
|
+
require_relative "rest-ftp-daemon/remote_ftp"
|
48
|
+
require_relative "rest-ftp-daemon/remote_sftp"
|
49
49
|
|
50
|
+
require_relative "rest-ftp-daemon/api/job_presenter"
|
51
|
+
require_relative "rest-ftp-daemon/api/jobs"
|
52
|
+
require_relative "rest-ftp-daemon/api/dashboard"
|
53
|
+
require_relative "rest-ftp-daemon/api/root"
|
@@ -1,19 +1,29 @@
|
|
1
1
|
# Terrific constants
|
2
2
|
APP_NAME = "rest-ftp-daemon"
|
3
3
|
APP_NICK = "rftpd"
|
4
|
-
APP_VER = "0.
|
4
|
+
APP_VER = "0.230.0"
|
5
5
|
|
6
6
|
|
7
7
|
# Jobs and workers
|
8
8
|
JOB_RANDOM_LEN = 8
|
9
9
|
JOB_IDENT_LEN = 4
|
10
10
|
JOB_TEMPFILE_LEN = 8
|
11
|
+
JOB_UPDATE_INTERVAL = 1
|
12
|
+
|
11
13
|
JOB_STATUS_UPLOADING = :uploading
|
12
14
|
JOB_STATUS_RENAMING = :renaming
|
15
|
+
JOB_STATUS_PREPARED = :prepared
|
13
16
|
JOB_STATUS_FINISHED = :finished
|
14
17
|
JOB_STATUS_FAILED = :failed
|
15
18
|
JOB_STATUS_QUEUED = :queued
|
16
19
|
|
20
|
+
WORKER_STATUS_STARTING = :starting
|
21
|
+
WORKER_STATUS_WAITING = :waiting
|
22
|
+
WORKER_STATUS_RUNNING = :running
|
23
|
+
WORKER_STATUS_FINISHED = :finished
|
24
|
+
WORKER_STATUS_TIMEOUT = :timeout
|
25
|
+
WORKER_STATUS_CRASHED = :crashed
|
26
|
+
WORKER_STATUS_CLEANING = :cleaning
|
17
27
|
|
18
28
|
# Logging and startup
|
19
29
|
LOG_PIPE_LEN = 10
|
@@ -28,6 +38,10 @@ LOG_FORMAT_PREFIX = "%s %s\t%-#{LOG_PIPE_LEN.to_i}s\t"
|
|
28
38
|
LOG_FORMAT_MESSAGE = "%#{-LOG_COL_WID.to_i}s\t%#{-LOG_COL_JID.to_i}s\t%#{-LOG_COL_ID.to_i}s"
|
29
39
|
LOG_NEWLINE = "\n"
|
30
40
|
|
41
|
+
LOG_INDENT = "\t"
|
42
|
+
|
43
|
+
|
44
|
+
|
31
45
|
# Notifications
|
32
46
|
NOTIFY_PREFIX = "rftpd"
|
33
47
|
NOTIFY_USERAGENT = "#{APP_NAME} - #{APP_VER}"
|
@@ -35,26 +49,26 @@ NOTIFY_IDENTIFIER_LEN = 4
|
|
35
49
|
|
36
50
|
|
37
51
|
# Dashboard row styles
|
38
|
-
|
52
|
+
DASHBOARD_JOB_STYLES = {
|
39
53
|
JOB_STATUS_QUEUED => :active,
|
40
54
|
JOB_STATUS_FAILED => :warning,
|
41
55
|
JOB_STATUS_FINISHED => :success,
|
42
56
|
JOB_STATUS_UPLOADING => :info,
|
43
57
|
JOB_STATUS_RENAMING => :info,
|
44
58
|
}
|
45
|
-
|
59
|
+
DASHBOARD_WORKER_STYLES = {
|
46
60
|
waiting: :success,
|
47
61
|
working: :info,
|
48
62
|
crashed: :danger,
|
49
63
|
done: :success,
|
50
64
|
dead: :danger
|
51
65
|
}
|
52
|
-
PAGINATE_MAX = 30
|
53
66
|
|
54
67
|
|
55
68
|
# Configuration defaults
|
56
69
|
DEFAULT_WORKER_TIMEOUT = 3600
|
57
|
-
DEFAULT_FTP_CHUNK =
|
70
|
+
DEFAULT_FTP_CHUNK = 512
|
71
|
+
DEFAULT_PAGE_SIZE = 40
|
58
72
|
|
59
73
|
|
60
74
|
# Initialize defaults
|
@@ -19,7 +19,8 @@ module RestFtpDaemon
|
|
19
19
|
class JobTargetUnsupported < RestFtpDaemonException; end
|
20
20
|
class JobTargetUnparseable < RestFtpDaemonException; end
|
21
21
|
class JobTargetFileExists < RestFtpDaemonException; end
|
22
|
-
class
|
22
|
+
class JobTargetDirectoryError < RestFtpDaemonException; end
|
23
|
+
class JobTargetPermissionError < RestFtpDaemonException; end
|
23
24
|
class JobTooManyOpenFiles < RestFtpDaemonException; end
|
24
25
|
|
25
26
|
end
|
@@ -31,29 +31,33 @@ module RestFtpDaemon
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def self.tokenize item
|
34
|
+
return unless item.is_a? String
|
34
35
|
"[#{item}]"
|
35
36
|
end
|
36
37
|
|
37
38
|
def self.highlight_tokens path
|
39
|
+
return unless path.is_a? String
|
38
40
|
path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
|
39
41
|
end
|
40
42
|
|
41
43
|
def self.extract_filename path
|
44
|
+
return unless path.is_a? String
|
42
45
|
# match everything that's after a slash at the end of the string
|
43
46
|
m = path.match /\/?([^\/]+)$/
|
44
47
|
return m[1] unless m.nil?
|
45
48
|
end
|
46
49
|
|
47
50
|
def self.extract_dirname path
|
51
|
+
return unless path.is_a? String
|
48
52
|
# match all the beginning of the string up to the last slash
|
49
53
|
m = path.match(/^(.*)\/[^\/]*$/)
|
50
54
|
return "/#{m[1]}" unless m.nil?
|
51
55
|
end
|
52
56
|
|
53
57
|
def self.extract_parent path
|
54
|
-
|
55
|
-
m = path.match(/^(
|
56
|
-
return m[1] unless m.nil?
|
58
|
+
return unless path.is_a? String
|
59
|
+
m = path.match(/^(.*)\/([^\/]+)\/?$/)
|
60
|
+
return m[1], m[2] unless m.nil?
|
57
61
|
end
|
58
62
|
|
59
63
|
def self.local_port_used? port
|
@@ -106,8 +110,9 @@ module RestFtpDaemon
|
|
106
110
|
datetime.to_datetime.strftime("%d/%m %H:%M:%S")
|
107
111
|
end
|
108
112
|
|
109
|
-
def self.hide_credentials_from_url
|
110
|
-
|
113
|
+
def self.hide_credentials_from_url path
|
114
|
+
return unless path.is_a? String
|
115
|
+
path.sub(/([a-z]+:\/\/[^\/]+):[^\/]+\@/, '\1@' )
|
111
116
|
end
|
112
117
|
|
113
118
|
end
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -57,24 +57,22 @@ module RestFtpDaemon
|
|
57
57
|
flag_default :tempfile, false
|
58
58
|
|
59
59
|
# Read source file size and parameters
|
60
|
-
@ftp_debug_enabled = (Settings.at :debug, :ftp) == true
|
61
|
-
update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_FTP_CHUNK
|
62
60
|
@notify_after_sec = Settings.transfer.notify_after_sec rescue nil
|
63
|
-
@chunk_size =
|
61
|
+
@chunk_size = DEFAULT_FTP_CHUNK * 1024
|
64
62
|
|
65
63
|
# Flag current job
|
66
64
|
@queued_at = Time.now
|
67
65
|
@updated_at = Time.now
|
68
66
|
|
69
67
|
# Send first notification
|
70
|
-
log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}]
|
68
|
+
log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] JOB_UPDATE_INTERVAL[#{JOB_UPDATE_INTERVAL}]"
|
71
69
|
client_notify :queued
|
72
70
|
end
|
73
71
|
|
74
72
|
def process
|
75
73
|
# Update job's status
|
76
74
|
@error = nil
|
77
|
-
log_info "Job.process
|
75
|
+
log_info "Job.process"
|
78
76
|
|
79
77
|
# Prepare job
|
80
78
|
begin
|
@@ -99,20 +97,10 @@ module RestFtpDaemon
|
|
99
97
|
rescue RestFtpDaemon::JobAssertionFailed => exception
|
100
98
|
return oops :started, exception, :assertion_failed
|
101
99
|
|
102
|
-
# rescue RestFtpDaemon::JobTimeout => exception
|
103
|
-
# info "Job.process propagate JobTimeout to Worker"
|
104
|
-
# raise RestFtpDaemon::JobTimeout
|
105
|
-
|
106
|
-
# rescue RestFtpDaemon::RestFtpDaemonException => exception
|
107
|
-
# return oops :started, exception, :prepare_failed, true
|
108
|
-
|
109
|
-
# rescue StandardError => exception
|
110
|
-
# return oops :started, exception, :prepare_unhandled, true
|
111
|
-
|
112
100
|
else
|
113
101
|
# Prepare done !
|
114
|
-
newstatus
|
115
|
-
log_info "Job.process notify[started]"
|
102
|
+
newstatus JOB_STATUS_PREPARED
|
103
|
+
log_info "Job.process notify [started]"
|
116
104
|
client_notify :started
|
117
105
|
end
|
118
106
|
|
@@ -154,6 +142,9 @@ module RestFtpDaemon
|
|
154
142
|
rescue Net::FTPTempError => exception
|
155
143
|
return oops :ended, exception, :net_temp_error
|
156
144
|
|
145
|
+
rescue Net::SFTP::StatusException => exception
|
146
|
+
return oops :ended, exception, :sftp_exception
|
147
|
+
|
157
148
|
rescue Errno::EMFILE => exception
|
158
149
|
return oops :ended, exception, :too_many_open_files
|
159
150
|
|
@@ -169,26 +160,19 @@ module RestFtpDaemon
|
|
169
160
|
rescue RestFtpDaemon::JobTargetFileExists => exception
|
170
161
|
return oops :ended, exception, :target_file_exists
|
171
162
|
|
172
|
-
rescue RestFtpDaemon::
|
173
|
-
return oops :ended, exception, :
|
163
|
+
rescue RestFtpDaemon::JobTargetDirectoryError => exception
|
164
|
+
return oops :ended, exception, :target_directory_missing
|
165
|
+
|
166
|
+
rescue RestFtpDaemon::JobTargetPermissionError => exception
|
167
|
+
return oops :ended, exception, :target_permission_error
|
174
168
|
|
175
169
|
rescue RestFtpDaemon::JobAssertionFailed => exception
|
176
170
|
return oops :ended, exception, :assertion_failed
|
177
171
|
|
178
|
-
# rescue RestFtpDaemon::JobTimeout => exception
|
179
|
-
# info "Job.process propagate JobTimeout to Worker"
|
180
|
-
# raise RestFtpDaemon::JobTimeout
|
181
|
-
|
182
|
-
# rescue RestFtpDaemon::RestFtpDaemonException => exception
|
183
|
-
# return oops :ended, exception, :transfer_failed, true
|
184
|
-
|
185
|
-
# rescue StandardError => exception
|
186
|
-
# return oops :ended, exception, :transfer_unhandled, true
|
187
|
-
|
188
172
|
else
|
189
173
|
# All done !
|
190
174
|
newstatus JOB_STATUS_FINISHED
|
191
|
-
log_info "Job.process notify[ended]"
|
175
|
+
log_info "Job.process notify [ended]"
|
192
176
|
client_notify :ended
|
193
177
|
end
|
194
178
|
|
@@ -274,103 +258,115 @@ module RestFtpDaemon
|
|
274
258
|
|
275
259
|
def prepare
|
276
260
|
# Update job status
|
277
|
-
newstatus :
|
261
|
+
newstatus :prepare
|
278
262
|
|
279
263
|
# Init
|
280
264
|
@source_method = :file
|
281
265
|
@target_method = nil
|
282
266
|
@source_path = nil
|
283
|
-
@target_url = nil
|
284
267
|
|
285
|
-
#
|
268
|
+
# Prepare source
|
286
269
|
raise RestFtpDaemon::JobMissingAttribute unless @source
|
287
270
|
@source_path = expand_path @source
|
288
271
|
set :source_path, @source_path
|
289
272
|
set :source_method, :file
|
290
273
|
|
291
|
-
#
|
274
|
+
# Prepare target
|
292
275
|
raise RestFtpDaemon::JobMissingAttribute unless @target
|
293
|
-
|
294
|
-
set :
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
276
|
+
target_uri = expand_url @target
|
277
|
+
set :target_uri, target_uri.to_s
|
278
|
+
@target_path = Path.new target_uri.path, true
|
279
|
+
|
280
|
+
#puts "@target_path: #{@target_path.inspect}"
|
281
|
+
|
282
|
+
# Prepare remote
|
283
|
+
newstatus :remote_init
|
284
|
+
#FIXME: use a "case" statement on @target_url.class
|
285
|
+
|
286
|
+
if target_uri.is_a? URI::FTP
|
287
|
+
log_info "Job.prepare target_method FTP"
|
288
|
+
set :target_method, :ftp
|
289
|
+
@remote = RemoteFTP.new target_uri, log_context
|
290
|
+
|
291
|
+
elsif (target_uri.is_a? URI::FTPES) || (target_uri.is_a? URI::FTPS)
|
292
|
+
log_info "Job.prepare target_method FTPES"
|
293
|
+
set :target_method, :ftpes
|
294
|
+
@remote = RemoteFTP.new target_uri, log_context, ftpes: true
|
304
295
|
|
305
|
-
|
306
|
-
|
307
|
-
|
296
|
+
elsif target_uri.is_a? URI::SFTP
|
297
|
+
log_info "Job.prepare target_method SFTP"
|
298
|
+
set :target_method, :sftp
|
299
|
+
@remote = RemoteSFTP.new target_uri, log_context
|
300
|
+
|
301
|
+
else
|
302
|
+
log_info "Job.prepare unknown scheme [#{target_uri.scheme}]"
|
303
|
+
raise RestFtpDaemon::JobTargetUnsupported
|
304
|
+
|
305
|
+
end
|
308
306
|
end
|
309
307
|
|
310
308
|
def transfer
|
311
309
|
# Update job status
|
312
|
-
|
310
|
+
#log_info "Job.transfer starting"
|
313
311
|
@started_at = Time.now
|
314
312
|
|
315
313
|
# Method assertions and init
|
316
314
|
raise RestFtpDaemon::JobAssertionFailed, "transfer/1" unless @source_path
|
317
|
-
raise RestFtpDaemon::JobAssertionFailed, "transfer/2" unless @
|
315
|
+
raise RestFtpDaemon::JobAssertionFailed, "transfer/2" unless @target_path
|
318
316
|
@transfer_sent = 0
|
319
317
|
set :source_processed, 0
|
320
318
|
|
321
|
-
# Guess source
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
log_info "Job.transfer sources #{
|
327
|
-
|
328
|
-
|
329
|
-
raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
|
330
|
-
set :source_count, source_matches.count
|
331
|
-
set :source_files, source_matches
|
319
|
+
# Guess source files from disk
|
320
|
+
newstatus :checking_source
|
321
|
+
sources = find_local @source_path
|
322
|
+
set :source_count, sources.count
|
323
|
+
set :source_files, sources.collect(&:full)
|
324
|
+
log_info "Job.transfer sources #{sources.collect(&:name)}"
|
325
|
+
#log_info "Job.transfer target #{target.full}"
|
326
|
+
raise RestFtpDaemon::JobSourceNotFound if sources.empty?
|
332
327
|
|
333
328
|
# Guess target file name, and fail if present while we matched multiple sources
|
334
|
-
|
335
|
-
raise RestFtpDaemon::JobTargetShouldBeDirectory if target_name && source_matches.count>1
|
336
|
-
|
337
|
-
# Scheme-aware config
|
338
|
-
ftp_init
|
329
|
+
raise RestFtpDaemon::JobTargetDirectoryError if @target_path.name && sources.count>1
|
339
330
|
|
340
331
|
# Connect to remote server and login
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
# Change to the right path
|
345
|
-
path = Helpers.extract_dirname(@target_url.path).to_s
|
346
|
-
ftp_chdir_or_buildpath path
|
332
|
+
newstatus :remote_connect
|
333
|
+
#log_info "Job.remote_connect" # [#{host}] [#{login}]"
|
334
|
+
@remote.connect
|
347
335
|
|
348
|
-
#
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
raise RestFtpDaemon::JobSourceNotReadable unless File.file? filename
|
336
|
+
# Prepare target path or build it if asked
|
337
|
+
#log_info "Job.remote_chdir"
|
338
|
+
newstatus :remote_chdir
|
339
|
+
@remote.chdir_or_create @target_path.dir, @mkdir
|
353
340
|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
@transfer_total += File.size filename
|
358
|
-
end
|
341
|
+
# Compute total files size
|
342
|
+
@transfer_total = sources.collect(&:size).sum
|
359
343
|
set :transfer_total, @transfer_total
|
360
344
|
|
345
|
+
# Reset counters
|
346
|
+
@last_data = 0
|
347
|
+
@last_time = Time.now
|
348
|
+
|
361
349
|
# Handle each source file matched, and start a transfer
|
362
|
-
|
363
|
-
|
364
|
-
#
|
365
|
-
|
350
|
+
source_processed = 0
|
351
|
+
sources.each do |source|
|
352
|
+
# Compute target filename
|
353
|
+
full_target = @target_path.clone
|
354
|
+
|
355
|
+
# Add the source file name if none found in the target path
|
356
|
+
unless full_target.name
|
357
|
+
full_target.name = source.name
|
358
|
+
end
|
359
|
+
|
360
|
+
# Do the transfer, for each file
|
361
|
+
#log_info "Job.remote_push"
|
362
|
+
remote_push source, full_target
|
366
363
|
|
367
364
|
# Update counters
|
368
|
-
|
369
|
-
set :source_processed, done
|
365
|
+
set :source_processed, source_processed += 1
|
370
366
|
end
|
371
367
|
|
372
368
|
# FTP transfer finished
|
373
|
-
|
369
|
+
finalize
|
374
370
|
end
|
375
371
|
|
376
372
|
|
@@ -383,6 +379,18 @@ module RestFtpDaemon
|
|
383
379
|
}
|
384
380
|
end
|
385
381
|
|
382
|
+
def find_local path
|
383
|
+
Dir.glob(path).collect do |file|
|
384
|
+
next unless File.readable? file
|
385
|
+
next unless File.file? file
|
386
|
+
Path.new file
|
387
|
+
end
|
388
|
+
end
|
389
|
+
|
390
|
+
def worker_is_still_active
|
391
|
+
Thread.current.thread_variable_set :updted_at, Time.now
|
392
|
+
end
|
393
|
+
|
386
394
|
def newstatus name
|
387
395
|
@status = name
|
388
396
|
worker_is_still_active
|
@@ -399,57 +407,13 @@ module RestFtpDaemon
|
|
399
407
|
instance_variable_set variable, default
|
400
408
|
end
|
401
409
|
|
402
|
-
def
|
403
|
-
# Update job status
|
404
|
-
newstatus :ftp_init
|
405
|
-
|
406
|
-
# Method assertions
|
407
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_init/1" if @target_method.nil?
|
408
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_init/2" if @target_url.nil?
|
409
|
-
|
410
|
-
log_info "Job.ftp_init target_method [#{@target_method}]"
|
411
|
-
case @target_method
|
412
|
-
when :ftp
|
413
|
-
@ftp = Net::FTP.new
|
414
|
-
when :ftps
|
415
|
-
@ftp = DoubleBagFTPS.new
|
416
|
-
@ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
|
417
|
-
@ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
|
418
|
-
else
|
419
|
-
log_info "Job.ftp_init unknown scheme [#{@target_url.scheme}]"
|
420
|
-
railse RestFtpDaemon::JobTargetUnsupported
|
421
|
-
end
|
422
|
-
|
423
|
-
# FTP debug mode ?
|
424
|
-
if @ftp_debug_enabled
|
425
|
-
# Output header to STDOUT
|
426
|
-
puts
|
427
|
-
puts "-------------------- FTP SESSION STARTING --------------------"
|
428
|
-
puts "job id\t #{@id}"
|
429
|
-
puts "source\t #{@source}"
|
430
|
-
puts "target\t #{@target}"
|
431
|
-
puts "host\t #{@target_url.host}"
|
432
|
-
puts "user\t #{@target_url.user}"
|
433
|
-
puts "--------------------------------------------------------------"
|
434
|
-
|
435
|
-
# Set debug mode on connection
|
436
|
-
@ftp.debug_mode = true
|
437
|
-
end
|
438
|
-
|
439
|
-
# Activate passive mode
|
440
|
-
@ftp.passive = true
|
441
|
-
end
|
442
|
-
|
443
|
-
def ftp_finished
|
410
|
+
def finalize
|
444
411
|
# Close FTP connexion and free up memory
|
445
|
-
|
446
|
-
|
447
|
-
@ftp = nil
|
412
|
+
log_info "Job.finalize"
|
413
|
+
@remote.close
|
448
414
|
|
449
|
-
#
|
450
|
-
|
451
|
-
puts "-------------------- FTP SESSION ENDED -----------------------"
|
452
|
-
end
|
415
|
+
# Free-up remote object
|
416
|
+
@remote = nil
|
453
417
|
|
454
418
|
# Update job status
|
455
419
|
newstatus :disconnecting
|
@@ -460,206 +424,106 @@ module RestFtpDaemon
|
|
460
424
|
$queue.counter_add :transferred, @transfer_total
|
461
425
|
end
|
462
426
|
|
463
|
-
def
|
464
|
-
# Update job status
|
465
|
-
newstatus :ftp_connect
|
466
|
-
|
467
|
-
# Method assertions
|
468
|
-
host = @target_url.host
|
469
|
-
log_info "Job.ftp_connect [#{host}]"
|
470
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/1" if @ftp.nil?
|
471
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/2" if @target_url.nil?
|
472
|
-
|
473
|
-
@ftp.connect(host)
|
474
|
-
end
|
475
|
-
|
476
|
-
def ftp_login
|
477
|
-
# Update job status
|
478
|
-
newstatus :ftp_login
|
479
|
-
|
480
|
-
# Method assertions
|
481
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_login/1" if @ftp.nil?
|
482
|
-
|
483
|
-
# use "anonymous" if user is empty
|
484
|
-
login = @target_url.user || "anonymous"
|
485
|
-
log_info "Job.ftp_login [#{login}]"
|
486
|
-
|
487
|
-
@ftp.login login, @target_url.password
|
488
|
-
end
|
489
|
-
|
490
|
-
def ftp_chdir_or_buildpath path
|
491
|
-
# Method assertions
|
492
|
-
log_info "Job.ftp_chdir [#{path}] mkdir: #{@mkdir}"
|
493
|
-
newstatus :ftp_chdir
|
494
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_chdir_or_buildpath/1" if path.nil?
|
495
|
-
|
496
|
-
# Extract directory from path
|
497
|
-
if @mkdir
|
498
|
-
# Split dir in parts
|
499
|
-
log_info "Job.ftp_chdir buildpath [#{path}]"
|
500
|
-
ftp_buildpath path
|
501
|
-
else
|
502
|
-
# Directly chdir if not mkdir requested
|
503
|
-
log_info "Job.ftp_chdir chdir [#{path}]"
|
504
|
-
@ftp.chdir path
|
505
|
-
end
|
506
|
-
end
|
507
|
-
|
508
|
-
def ftp_buildpath path
|
509
|
-
# Init
|
510
|
-
pref = "Job.ftp_buildpath [#{path}]"
|
511
|
-
|
512
|
-
begin
|
513
|
-
# Try to chdir in this directory
|
514
|
-
@ftp.chdir(path)
|
515
|
-
|
516
|
-
rescue Net::FTPPermError
|
517
|
-
# If not possible because the directory is missing
|
518
|
-
parent = Helpers.extract_parent(path)
|
519
|
-
log_info "#{pref} chdir failed - parent [#{parent}]"
|
520
|
-
|
521
|
-
# And only if we still have something to "dive up into"
|
522
|
-
if parent
|
523
|
-
# Do the same for the parent
|
524
|
-
ftp_buildpath parent
|
525
|
-
|
526
|
-
# Then finally create this dir and chdir
|
527
|
-
log_info "#{pref} > now mkdir [#{path}]"
|
528
|
-
@ftp.mkdir path
|
529
|
-
|
530
|
-
# And get into it (this chdir is not rescue'd on purpose in order to throw the ex)
|
531
|
-
log_info "#{pref} > now chdir [#{path}]"
|
532
|
-
@ftp.chdir(path)
|
533
|
-
end
|
534
|
-
|
535
|
-
end
|
536
|
-
|
537
|
-
# Now we were able to chdir inside, just tell it
|
538
|
-
log_info "#{pref} > ftp.pwd [#{@ftp.pwd}]"
|
539
|
-
end
|
540
|
-
|
541
|
-
def ftp_presence target_name
|
542
|
-
# Update job status
|
543
|
-
newstatus :ftp_presence
|
544
|
-
# FIXME / TODO: try with nlst
|
545
|
-
|
546
|
-
# Method assertions
|
547
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/1" if @ftp.nil?
|
548
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/2" if @target_url.nil?
|
549
|
-
|
550
|
-
# Get file list, sometimes the response can be an empty value
|
551
|
-
results = @ftp.list(target_name) rescue nil
|
552
|
-
log_info "Job.ftp_presence: #{results.inspect}"
|
553
|
-
|
554
|
-
# Result can be nil or a list of files
|
555
|
-
return false if results.nil?
|
556
|
-
|
557
|
-
results.count >0
|
558
|
-
end
|
559
|
-
|
560
|
-
def ftp_transfer source_filename, target_name = nil
|
427
|
+
def remote_push source, target
|
561
428
|
# Method assertions
|
562
|
-
|
563
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/
|
564
|
-
raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/
|
429
|
+
raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/1" if @remote.nil?
|
430
|
+
raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/2" if source.nil?
|
431
|
+
raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/3" if target.nil?
|
565
432
|
|
566
433
|
# Use source filename if target path provided none (typically with multiple sources)
|
567
|
-
|
568
|
-
|
569
|
-
set :source_current, target_name
|
570
|
-
|
571
|
-
# Check for target file presence
|
572
|
-
newstatus :checking_target
|
573
|
-
present = ftp_presence target_name
|
574
|
-
if present
|
575
|
-
if @overwrite
|
576
|
-
# delete it first
|
577
|
-
log_info "Job.ftp_transfer removing target file"
|
578
|
-
@ftp.delete(target_name)
|
579
|
-
else
|
580
|
-
# won't overwrite then stop here
|
581
|
-
log_info "Job.ftp_transfer failed: target file exists"
|
582
|
-
raise RestFtpDaemon::JobTargetFileExists
|
583
|
-
end
|
584
|
-
end
|
434
|
+
log_info "Job.remote_push [#{source.name}]: [#{source.full}] > [#{target.full}]"
|
435
|
+
set :source_current, source.name
|
585
436
|
|
586
437
|
# Compute temp target name
|
587
|
-
|
438
|
+
tempname = nil
|
588
439
|
if @tempfile
|
589
|
-
|
590
|
-
log_info "Job.
|
440
|
+
tempname = "#{target.name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
|
441
|
+
#log_info "Job.remote_push tempname [#{tempname}]"
|
442
|
+
else
|
443
|
+
end
|
444
|
+
|
445
|
+
# Remove any existing version if expected, or test its presence
|
446
|
+
if @overwrite
|
447
|
+
@remote.remove! target
|
448
|
+
elsif size = @remote.present?(target)
|
449
|
+
log_info "Job.remote_push existing (#{Helpers.format_bytes size, 'B'})"
|
450
|
+
raise RestFtpDaemon::JobTargetFileExists
|
591
451
|
end
|
592
452
|
|
593
453
|
# Start transfer
|
594
454
|
transfer_started_at = Time.now
|
595
|
-
@
|
596
|
-
|
597
|
-
@notified_at = Time.now
|
455
|
+
@progress_at = 0
|
456
|
+
@notified_at = transfer_started_at
|
598
457
|
newstatus JOB_STATUS_UPLOADING
|
599
458
|
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
ftp_transfer_block block
|
606
|
-
end
|
459
|
+
# Start the transfer, update job status after each block transfer
|
460
|
+
newstatus :uploading
|
461
|
+
@remote.push source, target, tempname do |transferred, name|
|
462
|
+
# Update transfer statistics
|
463
|
+
progress transferred, name
|
607
464
|
|
608
|
-
|
609
|
-
|
610
|
-
newstatus JOB_STATUS_RENAMING
|
611
|
-
log_info "Job.ftp_transfer renaming: #{target_name}"
|
612
|
-
@ftp.rename target_real, target_name
|
465
|
+
# Touch my worker status
|
466
|
+
worker_is_still_active
|
613
467
|
end
|
614
468
|
|
615
469
|
# Compute final bitrate
|
616
|
-
|
470
|
+
global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
|
471
|
+
set :transfer_bitrate, global_transfer_bitrate.round(0)
|
617
472
|
|
618
473
|
# Done
|
619
474
|
set :source_current, nil
|
620
|
-
log_info "Job.
|
475
|
+
#log_info "Job.remote_push finished"
|
621
476
|
end
|
622
477
|
|
623
|
-
def
|
478
|
+
def progress transferred, name = ""
|
479
|
+
# What's current time ?
|
480
|
+
now = Time.now
|
481
|
+
|
624
482
|
# Update counters
|
625
|
-
@transfer_sent +=
|
483
|
+
@transfer_sent += transferred
|
626
484
|
set :transfer_sent, @transfer_sent
|
627
485
|
|
628
|
-
# Update bitrate
|
629
|
-
#dt = Time.now - t0
|
630
|
-
bitrate0 = get_bitrate(@chunk_size, @transfer_pointer_at).round(0)
|
631
|
-
set :transfer_bitrate, bitrate0
|
632
|
-
|
633
486
|
# Update job info
|
634
487
|
percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
|
635
488
|
set :progress, percent0
|
636
489
|
|
637
|
-
#
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
490
|
+
# Update job status after each NOTIFY_UPADE_STATUS
|
491
|
+
progressed_ago = (now.to_f - @progress_at.to_f)
|
492
|
+
if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
|
493
|
+
@current_bitrate = running_bitrate @transfer_sent
|
494
|
+
set :transfer_bitrate, @current_bitrate.round(0)
|
495
|
+
|
496
|
+
# Log progress
|
497
|
+
stack = []
|
498
|
+
stack << "#{percent0} %"
|
499
|
+
stack << (Helpers.format_bytes @transfer_sent, "B")
|
500
|
+
stack << (Helpers.format_bytes @transfer_total, "B")
|
501
|
+
stack << (Helpers.format_bytes @current_bitrate.round(0), "bps")
|
502
|
+
stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
|
503
|
+
log_info "#{LOG_INDENT}progress #{stack2} \t#{name}"
|
504
|
+
|
505
|
+
# Remember when we last did it
|
506
|
+
@progress_at = now
|
507
|
+
end
|
648
508
|
|
649
509
|
# Notify if requested
|
650
|
-
|
651
|
-
if
|
510
|
+
notified_ago = (now.to_f - @notified_at.to_f)
|
511
|
+
if (!@notify_after_sec.nil?) && (notified_ago > @notify_after_sec)
|
512
|
+
# Prepare and send notification
|
652
513
|
notif_status = {
|
653
514
|
progress: percent0,
|
654
515
|
transfer_sent: @transfer_sent,
|
655
516
|
transfer_total: @transfer_total,
|
656
|
-
transfer_bitrate:
|
517
|
+
transfer_bitrate: @current_bitrate
|
657
518
|
}
|
658
519
|
client_notify :progress, status: notif_status
|
659
|
-
|
520
|
+
|
521
|
+
# Remember when we last did it
|
522
|
+
@notified_at = now
|
660
523
|
end
|
661
524
|
end
|
662
525
|
|
526
|
+
|
663
527
|
def client_notify event, payload = {}
|
664
528
|
# Skip if no URL given
|
665
529
|
return unless @notify
|
@@ -673,12 +537,25 @@ module RestFtpDaemon
|
|
673
537
|
log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
|
674
538
|
end
|
675
539
|
|
676
|
-
def get_bitrate
|
677
|
-
|
540
|
+
def get_bitrate delta_data, delta_time
|
541
|
+
return nil if delta_time.nil? || delta_time.zero?
|
542
|
+
8 * delta_data.to_f.to_f / delta_time
|
678
543
|
end
|
679
544
|
|
680
|
-
def
|
681
|
-
|
545
|
+
def running_bitrate current_data
|
546
|
+
return if @last_time.nil?
|
547
|
+
|
548
|
+
# Compute deltas
|
549
|
+
@last_data ||= 0
|
550
|
+
delta_data = current_data - @last_data
|
551
|
+
delta_time = Time.now - @last_time
|
552
|
+
|
553
|
+
# Update counters
|
554
|
+
@last_time = Time.now
|
555
|
+
@last_data = current_data
|
556
|
+
|
557
|
+
# Return bitrate
|
558
|
+
get_bitrate delta_data, delta_time
|
682
559
|
end
|
683
560
|
|
684
561
|
def oops event, exception, error = nil, include_backtrace = false
|
@@ -693,7 +570,7 @@ module RestFtpDaemon
|
|
693
570
|
end
|
694
571
|
|
695
572
|
# Close ftp connexion if open
|
696
|
-
@
|
573
|
+
@remote.close unless @remote.nil? || !@remote.connected?
|
697
574
|
|
698
575
|
# Update job's internal status
|
699
576
|
newstatus JOB_STATUS_FAILED
|