rest-ftp-daemon 0.424.3 → 0.430.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +4 -4
- data/config.ru +7 -7
- data/defaults.yml +2 -0
- data/lib/rest-ftp-daemon/api/dashboard.rb +5 -1
- data/lib/rest-ftp-daemon/api/jobs.rb +4 -9
- data/lib/rest-ftp-daemon/api/root.rb +7 -6
- data/lib/rest-ftp-daemon/constants.rb +17 -4
- data/lib/rest-ftp-daemon/initialize.rb +10 -0
- data/lib/rest-ftp-daemon/job.rb +7 -16
- data/lib/rest-ftp-daemon/job_queue.rb +1 -1
- data/lib/rest-ftp-daemon/jobs/transfer.rb +31 -43
- data/lib/rest-ftp-daemon/jobs/video.rb +5 -5
- data/lib/rest-ftp-daemon/remote/ftp.rb +2 -2
- data/lib/rest-ftp-daemon/static/css/main.css +1 -1
- data/lib/rest-ftp-daemon/uri.rb +5 -6
- data/lib/rest-ftp-daemon/views/dashboard_jobs.haml +3 -3
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +9 -9
- data/rest-ftp-daemon.gemspec +2 -2
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e891f3c30c8f698eaec3f92b7bb7ac49da5bee21
|
4
|
+
data.tar.gz: 48aa70fa0ca68c9c521736f7167ad8fe638cf264
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3aa33cbc8a12a1a0850a199e7ca4f79b1f68c7e4920989ca38de06e97b3b98205761f5d5aeab1f59dce5b874bfab5a6cb99ffed08bc668ae512d9843448f533d
|
7
|
+
data.tar.gz: ce099ad49bbca8d97b333a09bd03eff8a98f123086741a8aae0e1eee41652dc07875a9992b925b0314011552fc3f1a6468b273ca53d4df8200cb3115d7fd1a2b
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rest-ftp-daemon (0.
|
4
|
+
rest-ftp-daemon (0.430.0)
|
5
5
|
activesupport (~> 4.2)
|
6
6
|
api-auth
|
7
7
|
aws-sdk-resources (~> 2)
|
8
|
-
bmc-daemon-lib (~> 0.3.
|
8
|
+
bmc-daemon-lib (~> 0.3.10)
|
9
9
|
double-bag-ftps
|
10
10
|
facter
|
11
11
|
get_process_mem
|
@@ -47,7 +47,7 @@ GEM
|
|
47
47
|
descendants_tracker (~> 0.0.4)
|
48
48
|
ice_nine (~> 0.11.0)
|
49
49
|
thread_safe (~> 0.3, >= 0.3.1)
|
50
|
-
bmc-daemon-lib (0.3.
|
50
|
+
bmc-daemon-lib (0.3.10)
|
51
51
|
chamber (~> 2.9)
|
52
52
|
builder (3.2.2)
|
53
53
|
chamber (2.9.1)
|
@@ -163,7 +163,7 @@ GEM
|
|
163
163
|
ruby-progressbar (~> 1.4)
|
164
164
|
ruby-progressbar (1.8.1)
|
165
165
|
slop (3.6.0)
|
166
|
-
streamio-ffmpeg (
|
166
|
+
streamio-ffmpeg (3.0.0)
|
167
167
|
multi_json (~> 1.8)
|
168
168
|
sys-cpu (0.7.2)
|
169
169
|
ffi
|
data/config.ru
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# Load gem files
|
2
|
-
|
2
|
+
Conf.log :init, "rackup: project code"
|
3
|
+
|
3
4
|
load_path_libs = File.expand_path "lib", File.dirname(__FILE__)
|
4
5
|
$LOAD_PATH.unshift(load_path_libs) unless $LOAD_PATH.include?(load_path_libs)
|
5
6
|
require "rest-ftp-daemon"
|
6
7
|
|
7
8
|
# Rack authent
|
9
|
+
Conf.log :init, "rackup: rack setup"
|
8
10
|
unless Conf[:adminpwd].nil?
|
9
11
|
use Rack::Auth::Basic, "Restricted Area" do |username, password|
|
10
12
|
[username, password] == ["admin", Conf[:adminpwd]]
|
@@ -21,14 +23,12 @@ use Rack::Static, root: "#{Conf.app_libs}/static/", urls: [
|
|
21
23
|
MOUNT_SWAGGER_UI,
|
22
24
|
]
|
23
25
|
|
24
|
-
# Rack reloader
|
25
|
-
|
26
|
+
# Rack reloader
|
27
|
+
if Conf.app_env == "development"
|
26
28
|
# use Rack::Reloader, 1
|
27
|
-
# use Rack::MiniProfiler
|
28
29
|
end
|
29
30
|
|
30
|
-
# Initialize workers
|
31
|
-
RestFtpDaemon::WorkerPool.instance.start_em_all
|
32
|
-
|
33
31
|
# Launch the API
|
32
|
+
Conf.log :init, "rackup: start endpoint"
|
34
33
|
run RestFtpDaemon::API::Root
|
34
|
+
|
data/defaults.yml
CHANGED
@@ -55,10 +55,12 @@ newrelic:
|
|
55
55
|
# app_name: "rftpd-bigbusiness-dev" # app_name used for naming app (used as-is if provided)
|
56
56
|
prefix: "rftpd" # app prefix to build app_name
|
57
57
|
# platform: "bigbusiness" # app platform to build app_name
|
58
|
+
enabled: true
|
58
59
|
|
59
60
|
rollbar:
|
60
61
|
debug: false
|
61
62
|
# token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
|
63
|
+
enabled: true
|
62
64
|
|
63
65
|
logs:
|
64
66
|
path: "/tmp/"
|
@@ -27,7 +27,11 @@ module RestFtpDaemon
|
|
27
27
|
|
28
28
|
def build_dashboard filter = ''
|
29
29
|
# Initialize Facter
|
30
|
-
|
30
|
+
begin
|
31
|
+
Facter.loadfacts
|
32
|
+
rescue StandardError => exception
|
33
|
+
log_error "dashboard/build: #{exception.inspect}"
|
34
|
+
end
|
31
35
|
|
32
36
|
# Detect QS filters
|
33
37
|
@filter = filter.to_s
|
@@ -5,26 +5,21 @@ module RestFtpDaemon
|
|
5
5
|
class Jobs < Grape::API
|
6
6
|
include BmcDaemonLib
|
7
7
|
|
8
|
-
|
9
8
|
### EXCEPTIONS HANDLERS
|
10
9
|
rescue_from RestFtpDaemon::JobNotFound do |exception|
|
11
10
|
exception_error :api_job_not_found, 404, exception
|
12
11
|
end
|
13
12
|
rescue_from JSON::ParserError do |exception|
|
14
|
-
exception_error :
|
15
|
-
|
16
|
-
log_error "JSON::ParserError: #{exception.message}"
|
17
|
-
error!({error: :api_parse_error, message: exception.message}, 422)
|
13
|
+
exception_error :api_parse_error, 422, exception
|
18
14
|
end
|
19
15
|
rescue_from RestFtpDaemon::QueueCantCreateJob do |exception|
|
20
16
|
exception_error :api_cant_create_job, 422, exception
|
21
|
-
error!({error: :api_cant_create_job, message: exception.message}, 422)
|
22
17
|
end
|
23
|
-
rescue_from
|
24
|
-
exception_error
|
25
|
-
Rollbar.error exception
|
18
|
+
rescue_from RestFtpDaemon::UnresolvedTokens do |exception|
|
19
|
+
exception_error :api_unresolved_tokens, 422, exception
|
26
20
|
end
|
27
21
|
|
22
|
+
|
28
23
|
### ENDPOINTS
|
29
24
|
desc "Read job with ID", http_codes: [
|
30
25
|
{ code: 200, message: "Here is the job you requested" },
|
@@ -23,18 +23,19 @@ module RestFtpDaemon
|
|
23
23
|
Root.logger
|
24
24
|
end
|
25
25
|
|
26
|
-
def exception_error
|
26
|
+
def exception_error error, http_code, exception
|
27
27
|
# Extract message lines
|
28
28
|
lines = exception.message.lines.collect(&:strip).reject(&:empty?)
|
29
29
|
|
30
30
|
# Log error to file
|
31
|
-
log_error "#{http_code} [#{
|
31
|
+
log_error "#{http_code} [#{error}] #{lines.shift} ", lines
|
32
32
|
|
33
33
|
# Return error
|
34
34
|
error!({
|
35
|
-
error:
|
35
|
+
error: error,
|
36
36
|
http_code: http_code,
|
37
|
-
|
37
|
+
class: exception.class.name,
|
38
|
+
message: exception.message,
|
38
39
|
}, http_code)
|
39
40
|
end
|
40
41
|
|
@@ -47,9 +48,9 @@ module RestFtpDaemon
|
|
47
48
|
|
48
49
|
## EXCEPTION HANDLERS
|
49
50
|
rescue_from :all do |exception|
|
50
|
-
Rollbar.error exception
|
51
|
+
Rollbar.error "#{exception.class.name}: #{exception.message}"
|
51
52
|
#error!({error: :internal_server_error, message: exception.message}, 500)
|
52
|
-
exception_error :
|
53
|
+
exception_error :api_unexpected_error, 500, exception
|
53
54
|
end
|
54
55
|
|
55
56
|
|
@@ -1,7 +1,6 @@
|
|
1
1
|
# Configuration defaults
|
2
2
|
DEFAULT_POOL = "default"
|
3
3
|
DEFAULT_SFTP_TIMEOUT = 600 # 10mn
|
4
|
-
DEFAULT_FTP_CHUNK = 1024 # 1 MB
|
5
4
|
DEFAULT_PAGE_SIZE = 50 # 50 lines
|
6
5
|
DEFAULT_RETRY_AFTER = 10 # 10s
|
7
6
|
TARGET_BLANK = "_blank"
|
@@ -10,14 +9,28 @@ TARGET_BLANK = "_blank"
|
|
10
9
|
JOB_RANDOM_LEN = 8
|
11
10
|
JOB_IDENT_LEN = 4
|
12
11
|
JOB_TEMPFILE_LEN = 8
|
13
|
-
|
12
|
+
JOB_FTP_CHUNKMB = 2048 # 2 MB
|
14
13
|
|
15
14
|
JOB_FFMPEG_THREADS = 2
|
16
15
|
JOB_FFMPEG_ATTRIBUTES = [:video_codec, :video_bitrate, :video_bitrate_tolerance, :frame_rate, :resolution, :aspect, :keyframe_interval, :x264_vprofile, :x264_preset, :audio_codec, :audio_bitrate, :audio_sample_rate, :audio_channels]
|
17
16
|
|
18
17
|
# Internal job infos
|
19
|
-
|
20
|
-
|
18
|
+
INFO_PARAMS = :params
|
19
|
+
|
20
|
+
|
21
|
+
INFO_ERROR_MESSAGE = :error_message
|
22
|
+
INFO_ERROR_EXCEPTION = :error_exception
|
23
|
+
INFO_ERROR_BACKTRACE = :error_backtrace
|
24
|
+
INFO_SOURCE_COUNT = :source_count
|
25
|
+
INFO_SOURCE_PROCESSED = :source_processed
|
26
|
+
INFO_SOURCE_CURRENT = :source_current
|
27
|
+
INFO_SOURCE_FILES = :source_files
|
28
|
+
INFO_TRANSFER_TOTAL = :transfer_total
|
29
|
+
INFO_TRANFER_SENT = :transfer_sent
|
30
|
+
INFO_TRANFER_PROGRESS = :progress
|
31
|
+
INFO_TRANFER_BITRATE = :bitrate
|
32
|
+
INFO_TARGET_FILES = :target_files
|
33
|
+
|
21
34
|
|
22
35
|
# Constants: logger
|
23
36
|
LOG_FORMAT_PROGNAME = "%d\t%s"
|
@@ -1,3 +1,13 @@
|
|
1
1
|
# Init Rollbar and Newrelic
|
2
|
+
Conf.log :init, "init: Newrelic and Rollbar"
|
2
3
|
Conf.prepare_newrelic
|
3
4
|
Conf.prepare_rollbar
|
5
|
+
|
6
|
+
# Initialize Facter
|
7
|
+
Conf.log :init, "init: Facter"
|
8
|
+
Facter.loadfacts
|
9
|
+
|
10
|
+
# Initialize workers
|
11
|
+
Conf.log :init, "init: workers"
|
12
|
+
RestFtpDaemon::WorkerPool.instance.start_em_all
|
13
|
+
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -34,10 +34,6 @@ module RestFtpDaemon
|
|
34
34
|
attr_reader :finished_at
|
35
35
|
|
36
36
|
attr_reader :infos
|
37
|
-
#attr_reader :config
|
38
|
-
|
39
|
-
# delegate :type, :pool, :label, :priority,
|
40
|
-
# to: :params
|
41
37
|
|
42
38
|
# Define readers from imported fields
|
43
39
|
IMPORTED.each do |field|
|
@@ -71,7 +67,7 @@ module RestFtpDaemon
|
|
71
67
|
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
72
68
|
|
73
69
|
# Import query params
|
74
|
-
set_info
|
70
|
+
set_info INFO_PARAMS, params
|
75
71
|
IMPORTED.each do |field|
|
76
72
|
instance_variable_set "@#{field}", params[field]
|
77
73
|
end
|
@@ -82,19 +78,11 @@ module RestFtpDaemon
|
|
82
78
|
# Prepare sources/target
|
83
79
|
raise RestFtpDaemon::AttributeMissing, "source" unless params[:source]
|
84
80
|
@source_loc = Location.new(params[:source])
|
85
|
-
#set_info :location_source, params[:source]
|
86
81
|
log_info "Job.initialize source #{@source_loc.uri}"
|
87
82
|
|
88
83
|
raise RestFtpDaemon::AttributeMissing, "target" unless params[:target]
|
89
84
|
@target_loc = Location.new(params[:target])
|
90
|
-
#set_info :location_target, params[:target]
|
91
85
|
log_info "Job.initialize target #{@target_loc.uri}"
|
92
|
-
|
93
|
-
# Handle exceptions
|
94
|
-
# rescue RestFtpDaemon::UnresolvedTokens => exception
|
95
|
-
# return oops :started, exception
|
96
|
-
# rescue RestFtpDaemon::UnsupportedScheme => exception
|
97
|
-
# return oops :started, exception
|
98
86
|
end
|
99
87
|
|
100
88
|
def reset
|
@@ -144,6 +132,7 @@ module RestFtpDaemon
|
|
144
132
|
do_after
|
145
133
|
|
146
134
|
rescue StandardError => exception
|
135
|
+
Rollbar.error "process: #{exception.class.name}: #{exception.message}"
|
147
136
|
return oops current_signal, exception
|
148
137
|
|
149
138
|
else
|
@@ -182,10 +171,12 @@ module RestFtpDaemon
|
|
182
171
|
end
|
183
172
|
|
184
173
|
def oops_after_crash exception
|
174
|
+
Rollbar.error "oops_after_crash: #{exception.class.name}: #{exception.message}"
|
185
175
|
oops :ended, exception, "crashed"
|
186
176
|
end
|
187
177
|
|
188
178
|
def oops_you_stop_now exception
|
179
|
+
Rollbar.error "oops_you_stop_now: #{exception.class.name}: #{exception.message}"
|
189
180
|
oops :ended, exception, "timeout"
|
190
181
|
end
|
191
182
|
|
@@ -312,13 +303,13 @@ module RestFtpDaemon
|
|
312
303
|
# Update job's internal status
|
313
304
|
set_status JOB_STATUS_FAILED
|
314
305
|
set_error error
|
315
|
-
set_info
|
316
|
-
set_info
|
306
|
+
set_info INFO_ERROR_EXCEPTION, exception.class.to_s
|
307
|
+
set_info INFO_ERROR_MESSAGE, exception.message
|
317
308
|
|
318
309
|
# Build status stack
|
319
310
|
notif_status = nil
|
320
311
|
if include_backtrace
|
321
|
-
set_info
|
312
|
+
set_info INFO_ERROR_BACKTRACE, exception.backtrace
|
322
313
|
notif_status = {
|
323
314
|
backtrace: exception.backtrace,
|
324
315
|
}
|
@@ -102,7 +102,7 @@ module RestFtpDaemon
|
|
102
102
|
next unless job.status == JOB_STATUS_UPLOADING
|
103
103
|
|
104
104
|
# Extract current rate, next if not available
|
105
|
-
rate = job.get_info
|
105
|
+
rate = job.get_info INFO_TRANFER_BITRATE
|
106
106
|
next if rate.nil?
|
107
107
|
|
108
108
|
# Add its current rate
|
@@ -13,7 +13,7 @@ module RestFtpDaemon
|
|
13
13
|
|
14
14
|
# Some init
|
15
15
|
@transfer_sent = 0
|
16
|
-
set_info
|
16
|
+
set_info INFO_SOURCE_PROCESSED, 0
|
17
17
|
|
18
18
|
# Ensure source is FILE
|
19
19
|
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless @source_loc.is? URI::FILE
|
@@ -33,8 +33,9 @@ module RestFtpDaemon
|
|
33
33
|
log_info "JobTransfer.before target_method S3"
|
34
34
|
@remote = Remote::RemoteS3.new @target_loc, log_prefix, @config[:debug_s3]
|
35
35
|
else
|
36
|
-
|
37
|
-
|
36
|
+
message = "unknown scheme [#{@target_loc.scheme}] [#{target_uri.class.name}]"
|
37
|
+
log_info "JobTransfer.before #{message}"
|
38
|
+
raise RestFtpDaemon::TargetNotSupported, message
|
38
39
|
end
|
39
40
|
|
40
41
|
# Plug this Job into @remote to allow it to log
|
@@ -48,8 +49,8 @@ module RestFtpDaemon
|
|
48
49
|
# Scan local source files from disk
|
49
50
|
set_status JOB_STATUS_CHECKING_SRC
|
50
51
|
sources = @source_loc.scan_files
|
51
|
-
set_info
|
52
|
-
set_info
|
52
|
+
set_info INFO_SOURCE_COUNT, sources.size
|
53
|
+
set_info INFO_SOURCE_FILES, sources.collect(&:name)
|
53
54
|
log_info "JobTransfer.work sources #{sources.collect(&:name)}"
|
54
55
|
raise RestFtpDaemon::SourceNotFound if sources.empty?
|
55
56
|
|
@@ -66,7 +67,7 @@ module RestFtpDaemon
|
|
66
67
|
|
67
68
|
# Compute total files size
|
68
69
|
@transfer_total = sources.collect(&:size).sum
|
69
|
-
set_info
|
70
|
+
set_info INFO_TRANSFER_TOTAL, @transfer_total
|
70
71
|
|
71
72
|
# Reset counters
|
72
73
|
@last_data = 0
|
@@ -85,10 +86,10 @@ module RestFtpDaemon
|
|
85
86
|
|
86
87
|
# Add it to transferred target names
|
87
88
|
targets << target.name
|
88
|
-
set_info
|
89
|
+
set_info INFO_TARGET_FILES, targets
|
89
90
|
|
90
91
|
# Update counters
|
91
|
-
set_info
|
92
|
+
set_info INFO_SOURCE_PROCESSED, source_processed += 1
|
92
93
|
end
|
93
94
|
end
|
94
95
|
|
@@ -116,7 +117,7 @@ module RestFtpDaemon
|
|
116
117
|
|
117
118
|
# Use source filename if target path provided none (typically with multiple sources)
|
118
119
|
log_info "JobTransfer.remote_upload [#{source.name}]: [#{source.path}] > [#{target.path}]"
|
119
|
-
set_info
|
120
|
+
set_info INFO_SOURCE_CURRENT, source.name
|
120
121
|
|
121
122
|
# Remove any existing version if present, or check if it's there
|
122
123
|
if @overwrite
|
@@ -128,61 +129,60 @@ module RestFtpDaemon
|
|
128
129
|
|
129
130
|
# Start transfer
|
130
131
|
transfer_started_at = Time.now
|
131
|
-
@
|
132
|
-
@notified_at = transfer_started_at
|
132
|
+
@last_notify_at = transfer_started_at
|
133
133
|
|
134
134
|
# Start the transfer, update job status after each block transfer
|
135
135
|
set_status JOB_STATUS_UPLOADING
|
136
136
|
log_debug "JobTransfer.remote_upload source[#{source.path}] temp[#{@tempfile}]"
|
137
137
|
@remote.upload source, target, @tempfile do |transferred, name|
|
138
|
-
|
139
138
|
# Update transfer statistics
|
140
139
|
update_progress transferred, name
|
141
|
-
|
142
|
-
# Touch my worker status
|
143
|
-
touch_job
|
144
140
|
end
|
145
141
|
|
146
142
|
# Compute final bitrate
|
147
143
|
global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
|
148
|
-
set_info
|
144
|
+
set_info INFO_TRANFER_BITRATE, global_transfer_bitrate.round(0)
|
149
145
|
|
150
146
|
# Done
|
151
|
-
set_info
|
147
|
+
set_info INFO_SOURCE_CURRENT, nil
|
152
148
|
end
|
153
149
|
|
154
150
|
def update_progress transferred, name = ""
|
155
151
|
# Update counters
|
156
152
|
@transfer_sent += transferred
|
157
|
-
set_info
|
153
|
+
set_info INFO_TRANFER_SENT, @transfer_sent
|
158
154
|
|
159
155
|
# Update job info
|
160
156
|
percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
|
161
|
-
set_info
|
157
|
+
set_info INFO_TRANFER_PROGRESS, percent0
|
158
|
+
|
159
|
+
# Update bitrates
|
160
|
+
@current_bitrate = running_bitrate @transfer_sent
|
161
|
+
set_info INFO_TRANFER_BITRATE, @current_bitrate.round(0)
|
162
162
|
|
163
163
|
# What's current time ?
|
164
164
|
now = Time.now
|
165
165
|
|
166
|
-
# Update job status
|
167
|
-
update_progress_jobinfo now, percent0, name
|
168
|
-
|
169
166
|
# Notify if requested
|
170
|
-
|
167
|
+
progress_notify now, percent0, name
|
168
|
+
|
169
|
+
# Touch my worker status
|
170
|
+
touch_job
|
171
171
|
end
|
172
172
|
|
173
173
|
private
|
174
174
|
|
175
|
-
def
|
175
|
+
def progress_notify now, percent0, name
|
176
176
|
# No delay provided ?
|
177
|
-
return if
|
177
|
+
return if @config[:notify_after].nil?
|
178
178
|
|
179
179
|
# Still too early to notify again ?
|
180
|
-
how_long_ago = (now.to_f - @
|
181
|
-
return unless how_long_ago >
|
180
|
+
how_long_ago = (now.to_f - @last_notify_at.to_f)
|
181
|
+
return unless how_long_ago > @config[:notify_after]
|
182
182
|
|
183
|
-
# Update bitrates
|
184
|
-
@current_bitrate = running_bitrate @transfer_sent
|
185
|
-
set_info
|
183
|
+
# # Update bitrates
|
184
|
+
# @current_bitrate = running_bitrate @transfer_sent
|
185
|
+
# set_info INFO_TRANFER_BITRATE, @current_bitrate.round(0)
|
186
186
|
|
187
187
|
# Log progress
|
188
188
|
stack = [
|
@@ -193,18 +193,6 @@ module RestFtpDaemon
|
|
193
193
|
stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
|
194
194
|
log_debug "progress #{stack2} \t#{name}"
|
195
195
|
|
196
|
-
# Remember when we last did it
|
197
|
-
@progress_at = now
|
198
|
-
end
|
199
|
-
|
200
|
-
def update_progress_notify now, percent0, name
|
201
|
-
# No delay provided ?
|
202
|
-
return if @config[:notify_after].nil?
|
203
|
-
|
204
|
-
# Still too early to notify again ?
|
205
|
-
how_long_ago = (now.to_f - @notified_at.to_f)
|
206
|
-
return unless how_long_ago > @config[:notify_after]
|
207
|
-
|
208
196
|
# Prepare and send notification
|
209
197
|
client_notify :progress, status: {
|
210
198
|
progress: percent0,
|
@@ -215,7 +203,7 @@ module RestFtpDaemon
|
|
215
203
|
}
|
216
204
|
|
217
205
|
# Remember when we last did it
|
218
|
-
@
|
206
|
+
@last_notify_at = now
|
219
207
|
end
|
220
208
|
|
221
209
|
def get_bitrate delta_data, delta_time
|
@@ -44,20 +44,20 @@ module RestFtpDaemon
|
|
44
44
|
FileUtils.mkdir_p @target_loc.dir
|
45
45
|
|
46
46
|
# Do the work, for each file
|
47
|
-
set_info
|
47
|
+
set_info INFO_SOURCE_CURRENT, @source_loc.name
|
48
48
|
ffmpeg_command @source_loc, target_final
|
49
49
|
|
50
50
|
# Done
|
51
|
-
set_info
|
51
|
+
set_info INFO_SOURCE_CURRENT, nil
|
52
52
|
end
|
53
53
|
|
54
54
|
def do_after
|
55
55
|
# Done
|
56
|
-
set_info
|
56
|
+
set_info INFO_SOURCE_CURRENT, nil
|
57
57
|
end
|
58
58
|
|
59
59
|
def ffmpeg_command source, target
|
60
|
-
set_info
|
60
|
+
set_info INFO_SOURCE_CURRENT, source.name
|
61
61
|
|
62
62
|
# Read info about source file
|
63
63
|
movie = FFMPEG::Movie.new(source.path)
|
@@ -81,7 +81,7 @@ module RestFtpDaemon
|
|
81
81
|
# Build command
|
82
82
|
movie.transcode(target.path, options) do |ffmpeg_progress|
|
83
83
|
# set_info :work, :ffmpeg_progress, ffmpeg_progress
|
84
|
-
set_info
|
84
|
+
set_info INFO_TRANFER_PROGRESS, (100.0 * ffmpeg_progress).round(1)
|
85
85
|
log_debug "progress #{ffmpeg_progress}"
|
86
86
|
end
|
87
87
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require "net/ftp"
|
2
2
|
require "double_bag_ftps"
|
3
3
|
|
4
|
-
# Handle FTP and
|
4
|
+
# Handle FTP and FTPES transfers for Remote class
|
5
5
|
module RestFtpDaemon
|
6
6
|
module Remote
|
7
7
|
class RemoteFTP < RemoteBase
|
@@ -20,7 +20,7 @@ module RestFtpDaemon
|
|
20
20
|
@ftp.debug_mode = @debug
|
21
21
|
|
22
22
|
# Config
|
23
|
-
@chunk_size =
|
23
|
+
@chunk_size = JOB_FTP_CHUNKMB.to_i * 1024
|
24
24
|
|
25
25
|
# Announce object
|
26
26
|
log_debug "Remote::RemoteFTP.prepare chunk_size:#{@chunk_size}"
|
data/lib/rest-ftp-daemon/uri.rb
CHANGED
@@ -18,10 +18,9 @@ module URI
|
|
18
18
|
DEFAULT_PORT = 22
|
19
19
|
end
|
20
20
|
|
21
|
-
|
22
|
-
@@schemes["
|
23
|
-
@@schemes["
|
24
|
-
@@schemes["
|
25
|
-
@@schemes["
|
26
|
-
@@schemes["FILE"] = FILE
|
21
|
+
@@schemes["FTPS"] = FTPS
|
22
|
+
@@schemes["FTPES"] = FTPES
|
23
|
+
@@schemes["SFTP"] = SFTP
|
24
|
+
@@schemes["S3"] = S3
|
25
|
+
@@schemes["FILE"] = FILE
|
27
26
|
end
|
@@ -29,8 +29,8 @@
|
|
29
29
|
|
30
30
|
%thead
|
31
31
|
%tr
|
32
|
-
%th ID
|
33
|
-
%th T
|
32
|
+
%th{title: "Job identifier"} ID
|
33
|
+
%th{title: "Job type"} T
|
34
34
|
%th pool
|
35
35
|
%th queued
|
36
36
|
%th label
|
@@ -39,7 +39,7 @@
|
|
39
39
|
%th.error status
|
40
40
|
%th{"min-width" => 120} detail
|
41
41
|
%th.text-right size
|
42
|
-
%th.text-right rate
|
42
|
+
%th.text-right{title: "updated every #{JOB_FTP_CHUNKMB} MB block"} rate
|
43
43
|
%th{title: "Worker ID"} W
|
44
44
|
%th{title: "Priority"} P
|
45
45
|
%th{title: "Runs count"} R
|
@@ -1,14 +1,14 @@
|
|
1
1
|
-# coding: utf-8
|
2
2
|
- jobs.each do |job|
|
3
|
-
-
|
4
|
-
-
|
5
|
-
- transfer_total = job.get_info(
|
6
|
-
- transfer_bitrate = job.get_info(
|
7
|
-
- work_sent = job.get_info(
|
8
|
-
|
9
|
-
- source_count = job.get_info(
|
10
|
-
- source_processed = job.get_info(
|
11
|
-
- source_current = job.get_info(
|
3
|
+
- error_message = job.get_info(INFO_ERROR_MESSAGE)
|
4
|
+
- work_progress = job.get_info(INFO_TRANFER_PROGRESS)
|
5
|
+
- transfer_total = job.get_info(INFO_TRANSFER_TOTAL)
|
6
|
+
- transfer_bitrate = job.get_info(INFO_TRANFER_BITRATE)
|
7
|
+
- work_sent = job.get_info(INFO_TRANFER_SENT)
|
8
|
+
|
9
|
+
- source_count = job.get_info(INFO_SOURCE_COUNT) || 0
|
10
|
+
- source_processed = job.get_info(INFO_SOURCE_PROCESSED) || 0
|
11
|
+
- source_current = job.get_info(INFO_SOURCE_CURRENT)
|
12
12
|
|
13
13
|
- job_working = [JOB_STATUS_UPLOADING, JOB_STATUS_TRANSFORMING].include? job.status
|
14
14
|
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
|
4
4
|
# Project version
|
5
|
-
spec.version = "0.
|
5
|
+
spec.version = "0.430.0"
|
6
6
|
|
7
7
|
# Project description
|
8
8
|
spec.name = "rest-ftp-daemon"
|
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.add_development_dependency "http"
|
32
32
|
|
33
33
|
# Runtime dependencies
|
34
|
-
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.
|
34
|
+
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.10"
|
35
35
|
spec.add_runtime_dependency "json", "~> 1.8"
|
36
36
|
spec.add_runtime_dependency "thin", "~> 1.7"
|
37
37
|
spec.add_runtime_dependency "activesupport", "~> 4.2"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.430.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
@@ -100,14 +100,14 @@ dependencies:
|
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0.3.
|
103
|
+
version: 0.3.10
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0.3.
|
110
|
+
version: 0.3.10
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: json
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|