rest-ftp-daemon 0.202.2 → 0.210.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -2
- data/lib/rest-ftp-daemon.rb +8 -1
- data/lib/rest-ftp-daemon/api/dashboard.rb +14 -4
- data/lib/rest-ftp-daemon/api/jobs.rb +6 -4
- data/lib/rest-ftp-daemon/api/root.rb +4 -9
- data/lib/rest-ftp-daemon/api/status.rb +3 -4
- data/lib/rest-ftp-daemon/constants.rb +41 -19
- data/lib/rest-ftp-daemon/exceptions.rb +1 -0
- data/lib/rest-ftp-daemon/helpers.rb +3 -5
- data/lib/rest-ftp-daemon/job.rb +110 -92
- data/lib/rest-ftp-daemon/job_queue.rb +88 -80
- data/lib/rest-ftp-daemon/logger.rb +5 -3
- data/lib/rest-ftp-daemon/logger_pool.rb +3 -2
- data/lib/rest-ftp-daemon/notification.rb +13 -18
- data/lib/rest-ftp-daemon/views/dashboard.haml +1 -1
- data/lib/rest-ftp-daemon/views/dashboard_jobs.haml +18 -6
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +5 -11
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +14 -12
- data/lib/rest-ftp-daemon/worker_pool.rb +73 -83
- data/rest-ftp-daemon.yml.sample +6 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad1215dadf62110ff9e20ee3e5100c7af016c258
|
4
|
+
data.tar.gz: e5e14d4cd2f4bf5170915e4a8ab0f2fde3046c59
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0f67f2a19ef8e17abc200bce126192127595c8c8e3c712840e23fffbf462bd0b06221f1e3c828c4fe46c334f834bc8312de60b42d9cdedd02f25ac63d0028dd2
|
7
|
+
data.tar.gz: af5f5a20d55f85cbe5213f2db1f6ed5593c538283c2add01608a34eff6a20e322620f0123b365457859f9d731bf6a6a0efdb6c2a08556ac4bf39fe64ce921a89
|
data/.gitignore
CHANGED
data/lib/rest-ftp-daemon.rb
CHANGED
@@ -5,17 +5,24 @@ require 'grape'
|
|
5
5
|
require 'grape-entity'
|
6
6
|
require 'haml'
|
7
7
|
require 'facter'
|
8
|
+
require 'uri'
|
9
|
+
require 'securerandom'
|
10
|
+
require 'timeout'
|
8
11
|
require 'sys/cpu'
|
9
12
|
require 'syslog'
|
13
|
+
require 'net/ftp'
|
14
|
+
require 'net/http'
|
15
|
+
require 'double_bag_ftps'
|
10
16
|
|
11
17
|
|
12
|
-
#
|
18
|
+
# Project's libs
|
13
19
|
require 'rest-ftp-daemon/constants'
|
14
20
|
require 'rest-ftp-daemon/settings'
|
15
21
|
require 'rest-ftp-daemon/exceptions'
|
16
22
|
require 'rest-ftp-daemon/helpers'
|
17
23
|
require 'rest-ftp-daemon/uri'
|
18
24
|
require 'rest-ftp-daemon/job_queue'
|
25
|
+
require 'rest-ftp-daemon/worker'
|
19
26
|
require 'rest-ftp-daemon/worker_pool'
|
20
27
|
require 'rest-ftp-daemon/logger'
|
21
28
|
require 'rest-ftp-daemon/logger_pool'
|
@@ -3,6 +3,12 @@ module RestFtpDaemon
|
|
3
3
|
class Root < Grape::API
|
4
4
|
|
5
5
|
|
6
|
+
####### HELPERS
|
7
|
+
|
8
|
+
helpers do
|
9
|
+
end
|
10
|
+
|
11
|
+
|
6
12
|
####### DASHBOARD - GET /
|
7
13
|
|
8
14
|
# Server global status
|
@@ -15,14 +21,18 @@ module RestFtpDaemon
|
|
15
21
|
# Detect QS filters
|
16
22
|
only = params["only"].to_s
|
17
23
|
|
18
|
-
# Get jobs
|
19
|
-
|
24
|
+
# Get jobs for this view, order jobs by their weights
|
25
|
+
current = $queue.filter_jobs only
|
26
|
+
|
27
|
+
# Provide queue only if no filtering set
|
28
|
+
queue = []
|
29
|
+
queue = $queue.queue if only.empty?
|
20
30
|
|
21
31
|
# Get workers status
|
22
|
-
@
|
32
|
+
@worker_variables = $pool.worker_variables
|
23
33
|
|
24
34
|
# Compile haml template
|
25
|
-
output = render :dashboard, {
|
35
|
+
output = render :dashboard, {queue: queue, current: current, only: only}
|
26
36
|
|
27
37
|
# Send response
|
28
38
|
env['api.format'] = :html
|
@@ -13,7 +13,8 @@ module RestFtpDaemon
|
|
13
13
|
|
14
14
|
begin
|
15
15
|
# Get job to display
|
16
|
-
|
16
|
+
raise RestFtpDaemon::JobNotFound if params[:id].nil?
|
17
|
+
job = $queue.find_by_id(params[:id]) || $queue.find_by_id(params[:id], true)
|
17
18
|
raise RestFtpDaemon::JobNotFound if job.nil?
|
18
19
|
|
19
20
|
rescue RestFtpDaemon::JobNotFound => exception
|
@@ -48,7 +49,8 @@ module RestFtpDaemon
|
|
48
49
|
only = params["only"].to_s
|
49
50
|
|
50
51
|
# Get jobs to display
|
51
|
-
jobs = $queue.sorted_by_status(only)
|
52
|
+
# jobs = $queue.sorted_by_status(only)
|
53
|
+
jobs = $queue.jobs
|
52
54
|
|
53
55
|
rescue RestFtpDaemonException => exception
|
54
56
|
info "EXCEPTION: RestFtpDaemonException: #{exception.message}"
|
@@ -86,8 +88,8 @@ module RestFtpDaemon
|
|
86
88
|
end
|
87
89
|
|
88
90
|
post '/jobs/' do
|
89
|
-
info "POST /jobs #{
|
90
|
-
|
91
|
+
info "POST /jobs", params.collect {|name, value| "#{name}: #{value.inspect}"}
|
92
|
+
|
91
93
|
begin
|
92
94
|
|
93
95
|
# Create a new job
|
@@ -33,8 +33,10 @@ module RestFtpDaemon
|
|
33
33
|
|
34
34
|
helpers do
|
35
35
|
|
36
|
-
def info message,
|
37
|
-
Root.logger.info_with_id message,
|
36
|
+
def info message, lines = []
|
37
|
+
Root.logger.info_with_id message,
|
38
|
+
lines: lines,
|
39
|
+
origin: self.class.to_s
|
38
40
|
end
|
39
41
|
|
40
42
|
def api_error exception
|
@@ -50,13 +52,6 @@ module RestFtpDaemon
|
|
50
52
|
haml_engine.render(binding, values)
|
51
53
|
end
|
52
54
|
|
53
|
-
def job_find job_id
|
54
|
-
return nil if ($queue.all_size==0)
|
55
|
-
|
56
|
-
# Find a job with exactly this id, or prefixed if not found
|
57
|
-
$queue.find_by_id(job_id) || $queue.find_by_id(job_id, true)
|
58
|
-
end
|
59
|
-
|
60
55
|
end
|
61
56
|
|
62
57
|
end
|
@@ -17,10 +17,9 @@ module RestFtpDaemon
|
|
17
17
|
uptime: (Time.now - APP_STARTED).round(1),
|
18
18
|
counters: $queue.counters,
|
19
19
|
status: $queue.counts_by_status,
|
20
|
-
|
21
|
-
jobs_count: $queue.
|
22
|
-
jobs_queued: $queue.
|
23
|
-
jobs_popped: $queue.popped.collect(&:id),
|
20
|
+
workers: $pool.worker_variables,
|
21
|
+
jobs_count: $queue.jobs_count,
|
22
|
+
jobs_queued: $queue.queued_ids
|
24
23
|
#routes: RestFtpDaemon::API::Root::routes,
|
25
24
|
}
|
26
25
|
end
|
@@ -1,31 +1,53 @@
|
|
1
1
|
# Terrific constants
|
2
2
|
APP_NAME = "rest-ftp-daemon"
|
3
|
-
APP_VER = "0.
|
3
|
+
APP_VER = "0.210.0"
|
4
4
|
|
5
5
|
|
6
|
-
#
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
# Jobs and workers
|
7
|
+
JOB_RANDOM_LEN = 8
|
8
|
+
JOB_IDENT_LEN = 4
|
9
|
+
JOB_TEMPFILE_LEN = 8
|
10
|
+
JOB_STATUS_UPLOADING = :uploading
|
11
|
+
JOB_STATUS_RENAMING = :renaming
|
12
|
+
JOB_STATUS_FINISHED = :finished
|
13
|
+
JOB_STATUS_FAILED = :failed
|
14
|
+
JOB_STATUS_QUEUED = :queued
|
10
15
|
|
11
16
|
|
12
|
-
#
|
13
|
-
|
14
|
-
|
15
|
-
|
17
|
+
# Logging and startup
|
18
|
+
LOG_PIPE_LEN = 10
|
19
|
+
LOG_COL_WID = 4
|
20
|
+
LOG_COL_JID = JOB_IDENT_LEN+3+2
|
21
|
+
LOG_COL_ID = 6
|
22
|
+
LOG_TRIM_LINE = 80
|
16
23
|
|
17
24
|
|
18
|
-
# Jobs
|
19
|
-
JOB_UPDATE_KB = 2048
|
20
|
-
JOB_STATUS_UPLOADING = :uploading
|
21
|
-
JOB_STATUS_FINISHED = :finished
|
22
|
-
JOB_STATUS_QUEUED = :queued
|
23
|
-
JOB_WEIGHTS = {queued: -10, uploading: 10, finished: 50}
|
24
|
-
|
25
25
|
# Notifications
|
26
|
-
NOTIFY_PREFIX
|
27
|
-
NOTIFY_USERAGENT
|
28
|
-
NOTIFY_IDENTIFIER_LEN
|
26
|
+
NOTIFY_PREFIX = "rftpd"
|
27
|
+
NOTIFY_USERAGENT = "#{APP_NAME} - #{APP_VER}"
|
28
|
+
NOTIFY_IDENTIFIER_LEN = 4
|
29
|
+
|
30
|
+
|
31
|
+
# Dashboard row styles
|
32
|
+
JOB_STYLES = {
|
33
|
+
JOB_STATUS_QUEUED => :active,
|
34
|
+
JOB_STATUS_FAILED => :warning,
|
35
|
+
JOB_STATUS_FINISHED => :success,
|
36
|
+
JOB_STATUS_UPLOADING => :info,
|
37
|
+
JOB_STATUS_RENAMING => :info,
|
38
|
+
}
|
39
|
+
WORKER_STYLES = {
|
40
|
+
:waiting => :success,
|
41
|
+
:processing => :info,
|
42
|
+
:crashed => :danger,
|
43
|
+
:done => :success,
|
44
|
+
:dead => :danger
|
45
|
+
}
|
46
|
+
|
47
|
+
|
48
|
+
# Configuration defaults
|
49
|
+
DEFAULT_WORKER_TIMEOUT = 3600
|
50
|
+
DEFAULT_FTP_CHUNK = 2048
|
29
51
|
|
30
52
|
|
31
53
|
# Initialize defaults
|
@@ -8,6 +8,7 @@ module RestFtpDaemon
|
|
8
8
|
class MissingPool < RestFtpDaemonException; end
|
9
9
|
|
10
10
|
class JobException < RestFtpDaemonException; end
|
11
|
+
class JobTimeout < RestFtpDaemonException; end
|
11
12
|
class JobNotFound < RestFtpDaemonException; end
|
12
13
|
class JobUnresolvedTokens < RestFtpDaemonException; end
|
13
14
|
class JobAssertionFailed < RestFtpDaemonException; end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'securerandom'
|
2
|
-
|
3
1
|
module RestFtpDaemon
|
4
2
|
class Helpers
|
5
3
|
|
@@ -96,15 +94,15 @@ module RestFtpDaemon
|
|
96
94
|
# Dates and times: date with time generator
|
97
95
|
def self.datetime_full datetime
|
98
96
|
return "-" if datetime.nil?
|
99
|
-
return datetime.to_datetime.strftime("%d.%m.%Y %H:%M")
|
97
|
+
return datetime.to_datetime.strftime("%d.%m.%Y %H:%M:%S")
|
100
98
|
end
|
101
99
|
|
102
100
|
def self.datetime_short datetime
|
103
101
|
# return param.class
|
104
102
|
return "-" if datetime.nil?
|
105
103
|
return "?" unless datetime.respond_to? :to_date
|
106
|
-
return datetime.to_datetime.strftime("%H:%M") if datetime.to_date == Time.now.to_date
|
107
|
-
return datetime.to_datetime.strftime("%d/%m %H:%M")
|
104
|
+
return datetime.to_datetime.strftime("%H:%M:%S") if datetime.to_date == Time.now.to_date
|
105
|
+
return datetime.to_datetime.strftime("%d/%m %H:%M:%S")
|
108
106
|
end
|
109
107
|
|
110
108
|
def self.hide_credentials_from_url url
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -1,8 +1,3 @@
|
|
1
|
-
require 'uri'
|
2
|
-
require 'net/ftp'
|
3
|
-
require 'double_bag_ftps'
|
4
|
-
require 'timeout'
|
5
|
-
|
6
1
|
module RestFtpDaemon
|
7
2
|
class Job
|
8
3
|
|
@@ -40,11 +35,8 @@ module RestFtpDaemon
|
|
40
35
|
@status = nil
|
41
36
|
@wid = nil
|
42
37
|
|
43
|
-
# Debug mode
|
44
|
-
@ftp_debug_enabled = (Settings.at :debug, :ftp) == true
|
45
|
-
|
46
38
|
# Logger
|
47
|
-
@logger = RestFtpDaemon::LoggerPool.instance.get :
|
39
|
+
@logger = RestFtpDaemon::LoggerPool.instance.get :jobs
|
48
40
|
|
49
41
|
# Protect with a mutex
|
50
42
|
@mutex = Mutex.new
|
@@ -59,22 +51,29 @@ module RestFtpDaemon
|
|
59
51
|
flag_default :overwrite, false
|
60
52
|
flag_default :tempfile, false
|
61
53
|
|
54
|
+
# Read source file size and parameters
|
55
|
+
@ftp_debug_enabled = (Settings.at :debug, :ftp) == true
|
56
|
+
update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_FTP_CHUNK
|
57
|
+
@notify_after_sec = Settings.transfer.notify_after_sec rescue nil
|
58
|
+
@chunk_size = update_every_kb * 1024
|
59
|
+
|
62
60
|
# Flag current job
|
63
61
|
@queued_at = Time.now
|
62
|
+
@updated_at = Time.now
|
64
63
|
|
65
64
|
# Send first notification
|
66
|
-
info "Job.initialize notify
|
65
|
+
info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] update_every_kb[#{@update_every_kb}]"
|
67
66
|
client_notify :queued
|
68
67
|
end
|
69
68
|
|
70
69
|
def process
|
71
70
|
# Update job's status
|
72
71
|
@error = nil
|
72
|
+
info "Job.process starting"
|
73
73
|
|
74
74
|
# Prepare job
|
75
75
|
begin
|
76
|
-
|
77
|
-
newstatus :preparing
|
76
|
+
newstatus :prepare
|
78
77
|
prepare
|
79
78
|
|
80
79
|
rescue RestFtpDaemon::JobMissingAttribute => exception
|
@@ -95,22 +94,25 @@ module RestFtpDaemon
|
|
95
94
|
rescue RestFtpDaemon::JobAssertionFailed => exception
|
96
95
|
return oops :started, exception, :assertion_failed
|
97
96
|
|
98
|
-
rescue RestFtpDaemon::
|
99
|
-
|
97
|
+
# rescue RestFtpDaemon::JobTimeout => exception
|
98
|
+
# info "Job.process propagate JobTimeout to Worker"
|
99
|
+
# raise RestFtpDaemon::JobTimeout
|
100
100
|
|
101
|
-
rescue
|
102
|
-
|
101
|
+
# rescue RestFtpDaemon::RestFtpDaemonException => exception
|
102
|
+
# return oops :started, exception, :prepare_failed, true
|
103
|
+
|
104
|
+
# rescue Exception => exception
|
105
|
+
# return oops :started, exception, :prepare_unhandled, true
|
103
106
|
|
104
107
|
else
|
105
108
|
# Prepare done !
|
106
109
|
newstatus :prepared
|
107
|
-
info "Job.process notify
|
110
|
+
info "Job.process notify[started]"
|
108
111
|
client_notify :started
|
109
112
|
end
|
110
113
|
|
111
114
|
# Process job
|
112
115
|
begin
|
113
|
-
info "Job.process transfer"
|
114
116
|
newstatus :starting
|
115
117
|
transfer
|
116
118
|
|
@@ -162,16 +164,20 @@ module RestFtpDaemon
|
|
162
164
|
rescue RestFtpDaemon::JobAssertionFailed => exception
|
163
165
|
return oops :ended, exception, :assertion_failed
|
164
166
|
|
165
|
-
rescue RestFtpDaemon::
|
166
|
-
|
167
|
+
# rescue RestFtpDaemon::JobTimeout => exception
|
168
|
+
# info "Job.process propagate JobTimeout to Worker"
|
169
|
+
# raise RestFtpDaemon::JobTimeout
|
170
|
+
|
171
|
+
# rescue RestFtpDaemon::RestFtpDaemonException => exception
|
172
|
+
# return oops :ended, exception, :transfer_failed, true
|
167
173
|
|
168
|
-
rescue Exception => exception
|
169
|
-
|
174
|
+
# rescue Exception => exception
|
175
|
+
# return oops :ended, exception, :transfer_unhandled, true
|
170
176
|
|
171
177
|
else
|
172
178
|
# All done !
|
173
179
|
newstatus JOB_STATUS_FINISHED
|
174
|
-
info "Job.process notify
|
180
|
+
info "Job.process notify[ended]"
|
175
181
|
client_notify :ended
|
176
182
|
end
|
177
183
|
|
@@ -184,16 +190,22 @@ module RestFtpDaemon
|
|
184
190
|
end
|
185
191
|
end
|
186
192
|
|
193
|
+
def weight
|
194
|
+
@weight = [@priority.to_i, -@queued_at.to_i]
|
195
|
+
end
|
196
|
+
|
187
197
|
def set_queued
|
188
198
|
# Update job status
|
189
199
|
newstatus JOB_STATUS_QUEUED
|
190
200
|
end
|
191
201
|
|
192
202
|
def oops_after_crash exception
|
193
|
-
|
194
|
-
return oops :crashed, exception, :crashed
|
203
|
+
return oops :ended, exception, :crashed
|
195
204
|
end
|
196
205
|
|
206
|
+
def oops_you_stop_now exception
|
207
|
+
return oops :ended, exception, :timeout
|
208
|
+
end
|
197
209
|
|
198
210
|
protected
|
199
211
|
|
@@ -338,8 +350,6 @@ module RestFtpDaemon
|
|
338
350
|
# Handle each source file matched, and start a transfer
|
339
351
|
done = 0
|
340
352
|
source_matches.each do |filename|
|
341
|
-
# Increment counter
|
342
|
-
|
343
353
|
# Do the transfer, only if it's a file
|
344
354
|
ftp_transfer filename, target_name
|
345
355
|
|
@@ -355,21 +365,20 @@ module RestFtpDaemon
|
|
355
365
|
|
356
366
|
private
|
357
367
|
|
358
|
-
def info message,
|
368
|
+
def info message, lines = []
|
359
369
|
return if @logger.nil?
|
360
370
|
|
361
|
-
# Inject context
|
362
|
-
context[:id] = @id
|
363
|
-
context[:origin] = self.class
|
364
|
-
|
365
371
|
# Forward to logger
|
366
|
-
@logger.info_with_id message,
|
372
|
+
@logger.info_with_id message,
|
373
|
+
wid: @wid,
|
374
|
+
jid: @id,
|
375
|
+
lines: lines,
|
376
|
+
origin: self.class.to_s
|
367
377
|
end
|
368
378
|
|
369
379
|
def newstatus name
|
370
|
-
# Update local status
|
371
380
|
@status = name
|
372
|
-
|
381
|
+
worker_is_still_active
|
373
382
|
end
|
374
383
|
|
375
384
|
def flag_default name, default
|
@@ -518,7 +527,7 @@ module RestFtpDaemon
|
|
518
527
|
end
|
519
528
|
|
520
529
|
# Now we were able to chdir inside, just tell it
|
521
|
-
info "#{pref}
|
530
|
+
info "#{pref} > ftp.pwd [#{@ftp.pwd}]"
|
522
531
|
end
|
523
532
|
|
524
533
|
def ftp_presence target_name
|
@@ -565,10 +574,6 @@ module RestFtpDaemon
|
|
565
574
|
end
|
566
575
|
end
|
567
576
|
|
568
|
-
# Read source file size and parameters
|
569
|
-
update_every_kb = (Settings.transfer.update_every_kb rescue nil) || JOB_UPDATE_KB
|
570
|
-
notify_after_sec = Settings.transfer.notify_after_sec rescue nil
|
571
|
-
|
572
577
|
# Compute temp target name
|
573
578
|
target_real = target_name
|
574
579
|
if @tempfile
|
@@ -577,90 +582,103 @@ module RestFtpDaemon
|
|
577
582
|
end
|
578
583
|
|
579
584
|
# Start transfer
|
580
|
-
|
581
|
-
|
582
|
-
|
585
|
+
transfer_started_at = Time.now
|
586
|
+
@transfer_pointer_at = transfer_started_at
|
587
|
+
|
588
|
+
@notified_at = Time.now
|
583
589
|
newstatus JOB_STATUS_UPLOADING
|
584
590
|
|
585
|
-
@ftp.putbinaryfile(source_filename, target_real, chunk_size) do |block|
|
586
|
-
# Update
|
587
|
-
|
588
|
-
set :transfer_sent, @transfer_sent
|
589
|
-
|
590
|
-
# Update bitrate
|
591
|
-
#dt = Time.now - t0
|
592
|
-
bitrate0 = get_bitrate(chunk_size, t0).round(0)
|
593
|
-
set :transfer_bitrate, bitrate0
|
594
|
-
|
595
|
-
# Update job info
|
596
|
-
percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
|
597
|
-
set :progress, percent0
|
598
|
-
|
599
|
-
# Log progress
|
600
|
-
stack = []
|
601
|
-
stack << "#{percent0} %"
|
602
|
-
stack << (Helpers.format_bytes @transfer_sent, "B")
|
603
|
-
stack << (Helpers.format_bytes @transfer_total, "B")
|
604
|
-
stack << (Helpers.format_bytes bitrate0, "bps")
|
605
|
-
info "Job.ftp_transfer" + stack.map{|txt| ("%#{DEFAULT_LOGS_PIPE_LEN.to_i}s" % txt)}.join("\t")
|
606
|
-
|
607
|
-
# Update time pointer
|
608
|
-
t0 = Time.now
|
609
|
-
|
610
|
-
# Notify if requested
|
611
|
-
unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
|
612
|
-
notif_status = {
|
613
|
-
progress: percent0,
|
614
|
-
transfer_sent: @transfer_sent,
|
615
|
-
transfer_total: @transfer_total,
|
616
|
-
transfer_bitrate: bitrate0
|
617
|
-
}
|
618
|
-
client_notify :progress, status: notif_status
|
619
|
-
notified_at = Time.now
|
620
|
-
end
|
591
|
+
@ftp.putbinaryfile(source_filename, target_real, @chunk_size) do |block|
|
592
|
+
# Update the worker activity marker
|
593
|
+
worker_is_still_active
|
621
594
|
|
595
|
+
# Update job status after this block transfer
|
596
|
+
ftp_transfer_block block
|
622
597
|
end
|
623
598
|
|
624
599
|
# Rename temp file to target_temp
|
625
600
|
if @tempfile
|
626
|
-
newstatus
|
601
|
+
newstatus JOB_STATUS_RENAMING
|
627
602
|
info "Job.ftp_transfer renaming: #{target_name}"
|
628
603
|
@ftp.rename target_real, target_name
|
629
604
|
end
|
630
605
|
|
631
606
|
# Compute final bitrate
|
632
|
-
set :transfer_bitrate, get_bitrate(@transfer_total,
|
607
|
+
set :transfer_bitrate, get_bitrate(@transfer_total, transfer_started_at).round(0)
|
633
608
|
|
634
609
|
# Done
|
635
610
|
set :source_current, nil
|
636
611
|
info "Job.ftp_transfer finished"
|
637
612
|
end
|
638
613
|
|
614
|
+
def ftp_transfer_block block
|
615
|
+
# Update counters
|
616
|
+
@transfer_sent += block.bytesize
|
617
|
+
set :transfer_sent, @transfer_sent
|
618
|
+
|
619
|
+
# Update bitrate
|
620
|
+
#dt = Time.now - t0
|
621
|
+
bitrate0 = get_bitrate(@chunk_size, @transfer_pointer_at).round(0)
|
622
|
+
set :transfer_bitrate, bitrate0
|
623
|
+
|
624
|
+
# Update job info
|
625
|
+
percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
|
626
|
+
set :progress, percent0
|
627
|
+
|
628
|
+
# Log progress
|
629
|
+
stack = []
|
630
|
+
stack << "#{percent0} %"
|
631
|
+
stack << (Helpers.format_bytes @transfer_sent, "B")
|
632
|
+
stack << (Helpers.format_bytes @transfer_total, "B")
|
633
|
+
stack << (Helpers.format_bytes bitrate0, "bps")
|
634
|
+
stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
|
635
|
+
info "Job.ftp_transfer #{stack2}"
|
636
|
+
|
637
|
+
# Update time pointer
|
638
|
+
@transfer_pointer_at = Time.now
|
639
|
+
|
640
|
+
# Notify if requested
|
641
|
+
# info "Job.ftp_transfer next notif (#{(@notified_at+@notify_after_sec).to_f}) now #{Time.now.to_f}"
|
642
|
+
if @notify_after_sec.nil? || (Time.now > @notified_at + @notify_after_sec)
|
643
|
+
notif_status = {
|
644
|
+
progress: percent0,
|
645
|
+
transfer_sent: @transfer_sent,
|
646
|
+
transfer_total: @transfer_total,
|
647
|
+
transfer_bitrate: bitrate0
|
648
|
+
}
|
649
|
+
client_notify :progress, status: notif_status
|
650
|
+
@notified_at = Time.now
|
651
|
+
end
|
652
|
+
end
|
653
|
+
|
639
654
|
def client_notify event, payload = {}
|
640
655
|
# Skip if no URL given
|
641
656
|
return unless @notify
|
642
657
|
|
643
658
|
# Ok, create a notification!
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
end
|
659
|
+
payload[:id] = @id
|
660
|
+
payload[:event] = event
|
661
|
+
RestFtpDaemon::Notification.new @notify, payload
|
662
|
+
|
663
|
+
rescue Exception => ex
|
664
|
+
info "Job.client_notify EXCEPTION: #{ex.inspect}"
|
665
|
+
end
|
652
666
|
|
653
667
|
def get_bitrate total, last_timestamp
|
654
668
|
8*total.to_f / (Time.now - last_timestamp)
|
655
669
|
end
|
656
670
|
|
671
|
+
def worker_is_still_active
|
672
|
+
Thread.current.thread_variable_set :updted_at, Time.now
|
673
|
+
end
|
674
|
+
|
657
675
|
def oops event, exception, error = nil, include_backtrace = false
|
658
676
|
# Log this error
|
659
677
|
error = exception.class if error.nil?
|
660
678
|
|
661
679
|
message = "Job.oops event[#{event.to_s}] error[#{error.to_s}] ex[#{exception.class}] #{exception.message}"
|
662
680
|
if include_backtrace
|
663
|
-
info message,
|
681
|
+
info message, exception.backtrace
|
664
682
|
else
|
665
683
|
info message
|
666
684
|
end
|
@@ -669,9 +687,9 @@ module RestFtpDaemon
|
|
669
687
|
@ftp.close unless @ftp.nil? || @ftp.welcome.nil?
|
670
688
|
|
671
689
|
# Update job's internal status
|
672
|
-
newstatus
|
690
|
+
newstatus JOB_STATUS_FAILED
|
673
691
|
@error = error
|
674
|
-
set :error_exception, exception.class
|
692
|
+
set :error_exception, exception.class.to_s
|
675
693
|
set :error_message, exception.message
|
676
694
|
|
677
695
|
# Build status stack
|