rest-ftp-daemon 0.430.1 → 0.432.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +6 -6
- data/README.md +6 -6
- data/defaults.yml +13 -11
- data/lib/rest-ftp-daemon/api/jobs.rb +1 -1
- data/lib/rest-ftp-daemon/constants.rb +8 -4
- data/lib/rest-ftp-daemon/exceptions.rb +21 -21
- data/lib/rest-ftp-daemon/helpers/views.rb +5 -5
- data/lib/rest-ftp-daemon/job.rb +10 -7
- data/lib/rest-ftp-daemon/jobs/errors.rb +14 -2
- data/lib/rest-ftp-daemon/jobs/transfer.rb +10 -9
- data/lib/rest-ftp-daemon/jobs/video.rb +27 -14
- data/lib/rest-ftp-daemon/location.rb +4 -3
- data/lib/rest-ftp-daemon/remote/base.rb +3 -1
- data/lib/rest-ftp-daemon/remote/ftp.rb +9 -9
- data/lib/rest-ftp-daemon/remote/s3.rb +6 -5
- data/lib/rest-ftp-daemon/remote/sftp.rb +7 -7
- data/lib/rest-ftp-daemon/views/dashboard_jobs.haml +1 -1
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -2
- data/lib/rest-ftp-daemon/workers/reporter.rb +3 -6
- data/lib/rest-ftp-daemon/workers/transfer.rb +43 -15
- data/rest-ftp-daemon.gemspec +3 -3
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89e094d356f8271d07f64222768a60997ce50a88
|
4
|
+
data.tar.gz: f3deb47de89bc54b1b5f807aa8ecfc868d0c1926
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad21099fd7392bc545888394974d95c1e0ce8204e28de1682e3b93087d38c95f20e6594ec59ecc465f562625b9c1def500076aa7a8c3695b1edddea8bd0ff754
|
7
|
+
data.tar.gz: 952ac92bf57af92ec2c99570d5fff30aefcf8b8bf9ba21f3f661b70c8af801757f55ed5c876fc3df590ec8ce708d1ed882bf0ab85cbd6ffbc68ccfc7f397a12a
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rest-ftp-daemon (0.
|
4
|
+
rest-ftp-daemon (0.432.0)
|
5
5
|
activesupport (~> 4.2)
|
6
6
|
api-auth
|
7
7
|
aws-sdk-resources (~> 2)
|
8
|
-
bmc-daemon-lib (~> 0.3.
|
8
|
+
bmc-daemon-lib (~> 0.3.12)
|
9
9
|
double-bag-ftps
|
10
10
|
facter
|
11
11
|
get_process_mem
|
@@ -39,15 +39,15 @@ GEM
|
|
39
39
|
ast (2.3.0)
|
40
40
|
astrolabe (1.3.1)
|
41
41
|
parser (~> 2.2)
|
42
|
-
aws-sdk-core (2.5.
|
42
|
+
aws-sdk-core (2.5.11)
|
43
43
|
jmespath (~> 1.0)
|
44
|
-
aws-sdk-resources (2.5.
|
45
|
-
aws-sdk-core (= 2.5.
|
44
|
+
aws-sdk-resources (2.5.11)
|
45
|
+
aws-sdk-core (= 2.5.11)
|
46
46
|
axiom-types (0.1.1)
|
47
47
|
descendants_tracker (~> 0.0.4)
|
48
48
|
ice_nine (~> 0.11.0)
|
49
49
|
thread_safe (~> 0.3, >= 0.3.1)
|
50
|
-
bmc-daemon-lib (0.3.
|
50
|
+
bmc-daemon-lib (0.3.12)
|
51
51
|
chamber (~> 2.9)
|
52
52
|
builder (3.2.2)
|
53
53
|
chamber (2.9.1)
|
data/README.md
CHANGED
@@ -289,7 +289,7 @@ TODO for this document
|
|
289
289
|
Debian install preparation
|
290
290
|
------------------------------------------------------------------------------------
|
291
291
|
|
292
|
-
This project is available as a rubygem, requires Ruby 2.
|
292
|
+
This project is available as a rubygem, requires Ruby 2.3.0 and RubyGems installed.
|
293
293
|
|
294
294
|
#### Using rbenv and ruby-build
|
295
295
|
|
@@ -302,15 +302,15 @@ You may have to install some extra packages for the compilations to complete.
|
|
302
302
|
# git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
|
303
303
|
# echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
|
304
304
|
# echo 'eval "$(rbenv init -)"' >> ~/.bashrc
|
305
|
-
# rbenv install --list | grep '2.
|
305
|
+
# rbenv install --list | grep '2.3'
|
306
306
|
```
|
307
307
|
|
308
308
|
```
|
309
309
|
# curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.2.3
|
310
310
|
```
|
311
311
|
|
312
|
-
Otherwise, you way have to update ruby-build to include Ruby 2.
|
313
|
-
On Debian, 2.
|
312
|
+
Otherwise, you way have to update ruby-build to include Ruby 2.3 definitions.
|
313
|
+
On Debian, 2.3 is not included in Wheezy and appears in Jessie's version of the package.
|
314
314
|
|
315
315
|
#### Dedicated user
|
316
316
|
|
@@ -326,8 +326,8 @@ Use a dedicated user for the daemon, switch to this user and enable rbenv
|
|
326
326
|
Install the right ruby version and activate it
|
327
327
|
|
328
328
|
```
|
329
|
-
# rbenv install 2.
|
330
|
-
# rbenv local 2.
|
329
|
+
# rbenv install 2.3.0
|
330
|
+
# rbenv local 2.3.0
|
331
331
|
# rbenv rehash
|
332
332
|
```
|
333
333
|
|
data/defaults.yml
CHANGED
@@ -35,8 +35,10 @@ transfer:
|
|
35
35
|
debug_sftp: false
|
36
36
|
|
37
37
|
retry_on: # job error values that will allow a retry
|
38
|
+
- ftp_temp_error
|
38
39
|
- ftp_perm_error
|
39
40
|
- net_temp_error
|
41
|
+
- net_perm_error
|
40
42
|
- conn_reset_by_peer
|
41
43
|
- conn_timed_out
|
42
44
|
- conn_refused
|
@@ -66,16 +68,16 @@ logs:
|
|
66
68
|
path: "/tmp/"
|
67
69
|
level: debug
|
68
70
|
|
69
|
-
thin: "rftpd-thin.log"
|
70
|
-
default: "rftpd-core.log"
|
71
|
+
thin: "rftpd-default-thin.log"
|
72
|
+
default: "rftpd-default-core.log"
|
71
73
|
|
72
|
-
queue: "rftpd-core.log"
|
73
|
-
api: "rftpd-core.log"
|
74
|
-
workers: "rftpd-core.log"
|
75
|
-
transfer: "rftpd-workers.log"
|
76
|
-
conchita: "rftpd-workers.log"
|
77
|
-
reporter: "rftpd-workers.log"
|
78
|
-
notify: "rftpd-workers.log"
|
74
|
+
queue: "rftpd-default-core.log"
|
75
|
+
api: "rftpd-default-core.log"
|
76
|
+
workers: "rftpd-default-core.log"
|
77
|
+
transfer: "rftpd-default-workers.log"
|
78
|
+
conchita: "rftpd-default-workers.log"
|
79
|
+
reporter: "rftpd-default-workers.log"
|
80
|
+
notify: "rftpd-default-workers.log"
|
79
81
|
|
80
|
-
newrelic: "rftpd-newrelic.log"
|
81
|
-
rollbar: "rftpd-rollbar.log"
|
82
|
+
newrelic: "rftpd-default-newrelic.log"
|
83
|
+
rollbar: "rftpd-default-rollbar.log"
|
@@ -15,7 +15,7 @@ module RestFtpDaemon
|
|
15
15
|
rescue_from RestFtpDaemon::QueueCantCreateJob do |exception|
|
16
16
|
exception_error :api_cant_create_job, 422, exception
|
17
17
|
end
|
18
|
-
rescue_from RestFtpDaemon::
|
18
|
+
rescue_from RestFtpDaemon::JobUnresolvedTokens do |exception|
|
19
19
|
exception_error :api_unresolved_tokens, 422, exception
|
20
20
|
end
|
21
21
|
|
@@ -101,15 +101,19 @@ WORKER_STATUS_STARTING = "starting"
|
|
101
101
|
WORKER_STATUS_WAITING = "waiting"
|
102
102
|
WORKER_STATUS_RUNNING = "running"
|
103
103
|
WORKER_STATUS_FINISHED = "finished"
|
104
|
+
WORKER_STATUS_RETRYING = "retrying"
|
104
105
|
WORKER_STATUS_TIMEOUT = "timeout"
|
105
106
|
WORKER_STATUS_CRASHED = "crashed"
|
106
107
|
WORKER_STATUS_CLEANING = "cleaning"
|
107
108
|
WORKER_STATUS_REPORTING = "reporting"
|
108
109
|
WORKER_STYLES = {
|
109
|
-
WORKER_STATUS_WAITING
|
110
|
-
WORKER_STATUS_RUNNING
|
111
|
-
|
112
|
-
|
110
|
+
WORKER_STATUS_WAITING => nil,
|
111
|
+
WORKER_STATUS_RUNNING => :info,
|
112
|
+
WORKER_STATUS_REPORTING => :info,
|
113
|
+
WORKER_STATUS_CLEANING => :info,
|
114
|
+
WORKER_STATUS_RETRYING => :warning,
|
115
|
+
WORKER_STATUS_CRASHED => :danger,
|
116
|
+
WORKER_STATUS_FINISHED => :success,
|
113
117
|
}
|
114
118
|
|
115
119
|
|
@@ -1,31 +1,31 @@
|
|
1
1
|
module RestFtpDaemon
|
2
2
|
|
3
|
-
class
|
3
|
+
class BaseException < StandardError; end
|
4
4
|
|
5
|
-
class
|
5
|
+
class AssertionFailed < BaseException; end
|
6
6
|
|
7
|
-
class
|
8
|
-
class
|
9
|
-
class
|
10
|
-
class
|
11
|
-
class JobTimeout < RestFtpDaemonException; end
|
12
|
-
class JobNotFound < RestFtpDaemonException; end
|
7
|
+
class InvalidWorkerNumber < BaseException; end
|
8
|
+
class QueueCantCreateJob < BaseException; end
|
9
|
+
class JobTimeout < BaseException; end
|
10
|
+
class JobNotFound < BaseException; end
|
13
11
|
|
12
|
+
class JobException < BaseException; end
|
13
|
+
class JobAttributeMissing < BaseException; end
|
14
|
+
class JobUnresolvedTokens < BaseException; end
|
14
15
|
|
15
|
-
class
|
16
|
-
class
|
17
|
-
class UnresolvedTokens < RestFtpDaemonException; end
|
18
|
-
class LocationMalformed < RestFtpDaemonException; end
|
19
|
-
class LocationParseError < RestFtpDaemonException; end
|
20
|
-
class UnsupportedScheme < RestFtpDaemonException; end
|
21
|
-
class MissingFfmpegLibraries < RestFtpDaemonException; end
|
16
|
+
class LocationParseError < BaseException; end
|
17
|
+
class SchemeUnsupported < BaseException; end
|
22
18
|
|
23
|
-
class
|
24
|
-
class SourceNotFound <
|
19
|
+
class SourceUnsupported < BaseException; end
|
20
|
+
class SourceNotFound < BaseException; end
|
21
|
+
|
22
|
+
class TargetUnsupported < BaseException; end
|
23
|
+
class TargetFileExists < BaseException; end
|
24
|
+
class TargetDirectoryError < BaseException; end
|
25
|
+
class TargetPermissionError < BaseException; end
|
26
|
+
|
27
|
+
class VideoMissingBinary < BaseException; end
|
28
|
+
class VideoMovieError < BaseException; end
|
25
29
|
|
26
|
-
class TargetNotSupported < RestFtpDaemonException; end
|
27
|
-
class TargetFileExists < RestFtpDaemonException; end
|
28
|
-
class TargetDirectoryError < RestFtpDaemonException; end
|
29
|
-
class TargetPermissionError < RestFtpDaemonException; end
|
30
30
|
|
31
31
|
end
|
@@ -23,11 +23,11 @@ module RestFtpDaemon
|
|
23
23
|
"#{MOUNT_JOBS}/#{job.id}" if job.respond_to? :id
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
return "label-outline" if
|
28
|
-
return "label-info" if
|
29
|
-
return "label-warning" if
|
30
|
-
return "label-danger" if
|
26
|
+
def job_tentatives_style count
|
27
|
+
return "label-outline" if count <= 0
|
28
|
+
return "label-info" if count == 1
|
29
|
+
return "label-warning" if count == 2
|
30
|
+
return "label-danger" if count > 2
|
31
31
|
end
|
32
32
|
|
33
33
|
def location_style uri
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -26,7 +26,7 @@ module RestFtpDaemon
|
|
26
26
|
attr_reader :id
|
27
27
|
attr_reader :error
|
28
28
|
attr_reader :status
|
29
|
-
attr_reader :
|
29
|
+
attr_reader :tentatives
|
30
30
|
|
31
31
|
attr_reader :queued_at
|
32
32
|
attr_reader :updated_at
|
@@ -55,7 +55,7 @@ module RestFtpDaemon
|
|
55
55
|
@updated_at = nil
|
56
56
|
@error = nil
|
57
57
|
@status = nil
|
58
|
-
@
|
58
|
+
@tentatives = 0
|
59
59
|
@wid = nil
|
60
60
|
|
61
61
|
# Prepare configuration
|
@@ -76,11 +76,11 @@ module RestFtpDaemon
|
|
76
76
|
@pool = DEFAULT_POOL unless @pools.keys.include?(@pool)
|
77
77
|
|
78
78
|
# Prepare sources/target
|
79
|
-
raise RestFtpDaemon::
|
79
|
+
raise RestFtpDaemon::JobAttributeMissing, "source" unless params[:source]
|
80
80
|
@source_loc = Location.new(params[:source])
|
81
81
|
log_info "Job.initialize source #{@source_loc.uri}"
|
82
82
|
|
83
|
-
raise RestFtpDaemon::
|
83
|
+
raise RestFtpDaemon::JobAttributeMissing, "target" unless params[:target]
|
84
84
|
@target_loc = Location.new(params[:target])
|
85
85
|
log_info "Job.initialize target #{@target_loc.uri}"
|
86
86
|
end
|
@@ -89,6 +89,9 @@ module RestFtpDaemon
|
|
89
89
|
# Update job status
|
90
90
|
set_status JOB_STATUS_PREPARING
|
91
91
|
|
92
|
+
# Increment run cours
|
93
|
+
@tentatives +=1
|
94
|
+
|
92
95
|
# Flag current job timestamps
|
93
96
|
@queued_at = Time.now
|
94
97
|
@updated_at = Time.now
|
@@ -101,7 +104,7 @@ module RestFtpDaemon
|
|
101
104
|
set_status JOB_STATUS_QUEUED
|
102
105
|
set_error nil
|
103
106
|
client_notify :queued
|
104
|
-
log_info "Job.reset notify[queued]"
|
107
|
+
log_info "Job.reset notify[queued] tentative[#{@tentatives}]"
|
105
108
|
end
|
106
109
|
|
107
110
|
# Process job
|
@@ -132,7 +135,7 @@ module RestFtpDaemon
|
|
132
135
|
do_after
|
133
136
|
|
134
137
|
rescue StandardError => exception
|
135
|
-
Rollbar.error "process: #{exception.class.name}: #{exception.message}"
|
138
|
+
Rollbar.error "Job.process: #{exception.class.name}: #{exception.message}"
|
136
139
|
return oops current_signal, exception
|
137
140
|
|
138
141
|
else
|
@@ -159,7 +162,7 @@ module RestFtpDaemon
|
|
159
162
|
|
160
163
|
def weight
|
161
164
|
@weight = [
|
162
|
-
- @
|
165
|
+
- @tentatives.to_i,
|
163
166
|
+ @priority.to_i,
|
164
167
|
- @queued_at.to_i,
|
165
168
|
]
|
@@ -12,12 +12,13 @@ module RestFtpDaemon
|
|
12
12
|
invalid_argument: Errno::EINVAL,
|
13
13
|
runtime_error: RuntimeError,
|
14
14
|
|
15
|
-
|
15
|
+
job_timeout: RestFtpDaemon::JobTimeout,
|
16
|
+
source_not_supported: RestFtpDaemon::SourceUnsupported,
|
16
17
|
source_not_found: RestFtpDaemon::SourceNotFound,
|
17
18
|
target_file_exists: RestFtpDaemon::TargetFileExists,
|
18
19
|
target_directory_error: RestFtpDaemon::TargetDirectoryError,
|
19
20
|
target_permission_error: RestFtpDaemon::TargetPermissionError,
|
20
|
-
target_not_supported: RestFtpDaemon::
|
21
|
+
target_not_supported: RestFtpDaemon::TargetUnsupported,
|
21
22
|
assertion_failed: RestFtpDaemon::AssertionFailed,
|
22
23
|
location_parse_error: RestFtpDaemon::LocationParseError,
|
23
24
|
|
@@ -58,9 +59,20 @@ module RestFtpDaemon
|
|
58
59
|
sftp_key_mismatch: Net::SSH::HostKeyMismatch,
|
59
60
|
sftp_auth_failed: Net::SSH::AuthenticationFailed,
|
60
61
|
sftp_openssl_error: OpenSSL::SSL::SSLError,
|
62
|
+
|
63
|
+
video_missing_binary: RestFtpDaemon::VideoMissingBinary,
|
64
|
+
video_movie_error: RestFtpDaemon::VideoMovieError,
|
65
|
+
|
61
66
|
# rescue Encoding::UndefinedConversionError => exception
|
62
67
|
# return oops :ended, exception, "encoding_error", true
|
63
68
|
}
|
64
69
|
|
70
|
+
class InvalidWorkerNumber < BaseException; end
|
71
|
+
class QueueCantCreateJob < BaseException; end
|
72
|
+
class JobException < BaseException; end
|
73
|
+
class JobNotFound < BaseException; end
|
74
|
+
|
75
|
+
|
76
|
+
|
65
77
|
end
|
66
78
|
end
|
@@ -16,26 +16,26 @@ module RestFtpDaemon
|
|
16
16
|
set_info INFO_SOURCE_PROCESSED, 0
|
17
17
|
|
18
18
|
# Ensure source is FILE
|
19
|
-
raise RestFtpDaemon::
|
19
|
+
raise RestFtpDaemon::SourceUnsupported, @source_loc.scheme unless @source_loc.is? URI::FILE
|
20
20
|
|
21
21
|
# Prepare remote object
|
22
22
|
case target_uri
|
23
23
|
when URI::FTP
|
24
|
-
log_info "JobTransfer.
|
24
|
+
log_info "JobTransfer.do_before target_method FTP"
|
25
25
|
@remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftp]
|
26
26
|
when URI::FTPES, URI::FTPS
|
27
|
-
log_info "JobTransfer.
|
27
|
+
log_info "JobTransfer.do_before target_method FTPES/FTPS"
|
28
28
|
@remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
|
29
29
|
when URI::SFTP
|
30
|
-
log_info "JobTransfer.
|
30
|
+
log_info "JobTransfer.do_before target_method SFTP"
|
31
31
|
@remote = Remote::RemoteSFTP.new @target_loc, log_prefix, @config[:debug_sftp]
|
32
32
|
when URI::S3
|
33
|
-
log_info "JobTransfer.
|
33
|
+
log_info "JobTransfer.do_before target_method S3"
|
34
34
|
@remote = Remote::RemoteS3.new @target_loc, log_prefix, @config[:debug_s3]
|
35
35
|
else
|
36
36
|
message = "unknown scheme [#{@target_loc.scheme}] [#{target_uri.class.name}]"
|
37
|
-
log_info "JobTransfer.
|
38
|
-
raise RestFtpDaemon::
|
37
|
+
log_info "JobTransfer.do_before #{message}"
|
38
|
+
raise RestFtpDaemon::TargetUnsupported, message
|
39
39
|
end
|
40
40
|
|
41
41
|
# Plug this Job into @remote to allow it to log
|
@@ -51,7 +51,7 @@ module RestFtpDaemon
|
|
51
51
|
sources = @source_loc.scan_files
|
52
52
|
set_info INFO_SOURCE_COUNT, sources.size
|
53
53
|
set_info INFO_SOURCE_FILES, sources.collect(&:name)
|
54
|
-
log_info "JobTransfer.
|
54
|
+
log_info "JobTransfer.do_work sources #{sources.collect(&:name)}"
|
55
55
|
raise RestFtpDaemon::SourceNotFound if sources.empty?
|
56
56
|
|
57
57
|
# Guess target file name, and fail if present while we matched multiple sources
|
@@ -95,9 +95,10 @@ module RestFtpDaemon
|
|
95
95
|
|
96
96
|
def do_after
|
97
97
|
# Close FTP connexion and free up memory
|
98
|
+
log_info "JobTransfer.do_after close connexion, update status and counters"
|
98
99
|
@remote.close
|
99
100
|
|
100
|
-
# Free
|
101
|
+
# Free @remote object
|
101
102
|
@remote = nil
|
102
103
|
|
103
104
|
# Update job status
|
@@ -16,16 +16,14 @@ module RestFtpDaemon
|
|
16
16
|
log_info "JobVideo.before target_loc.path: #{@target_loc.path}"
|
17
17
|
|
18
18
|
# Ensure FFMPEG lib is available
|
19
|
-
|
20
|
-
|
21
|
-
raise RestFtpDaemon::MissingFfmpegLibraries, ffmpeg_binary_path
|
22
|
-
end
|
19
|
+
check_ffmpeg_binary :ffmpeg_binary
|
20
|
+
check_ffmpeg_binary :ffprobe_binary
|
23
21
|
|
24
22
|
# Ensure source and target are FILE
|
25
|
-
raise RestFtpDaemon::AssertionFailed
|
26
|
-
raise RestFtpDaemon::AssertionFailed
|
27
|
-
raise RestFtpDaemon::
|
28
|
-
raise RestFtpDaemon::
|
23
|
+
raise RestFtpDaemon::AssertionFailed unless @video_options.is_a? Hash
|
24
|
+
raise RestFtpDaemon::AssertionFailed unless @video_custom.is_a? Hash
|
25
|
+
raise RestFtpDaemon::SourceUnsupported, @source_loc.scheme unless @source_loc.is? URI::FILE
|
26
|
+
raise RestFtpDaemon::TargetUnsupported, @target_loc.scheme unless @target_loc.is? URI::FILE
|
29
27
|
end
|
30
28
|
|
31
29
|
def do_work
|
@@ -57,13 +55,17 @@ module RestFtpDaemon
|
|
57
55
|
end
|
58
56
|
|
59
57
|
def ffmpeg_command source, target
|
60
|
-
set_info INFO_SOURCE_CURRENT, source.name
|
61
|
-
|
62
58
|
# Read info about source file
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
59
|
+
set_info INFO_SOURCE_CURRENT, source.name
|
60
|
+
begin
|
61
|
+
movie = FFMPEG::Movie.new(source.path)
|
62
|
+
rescue StandardError => exception
|
63
|
+
raise RestFtpDaemon::VideoMovieError, exception.message
|
64
|
+
else
|
65
|
+
set_info :ffmpeg_size, movie.size
|
66
|
+
set_info :ffmpeg_duration, movie.duration
|
67
|
+
set_info :ffmpeg_resolution, movie.resolution
|
68
|
+
end
|
67
69
|
|
68
70
|
# Build options
|
69
71
|
options = {
|
@@ -101,6 +103,17 @@ module RestFtpDaemon
|
|
101
103
|
return custom_parts
|
102
104
|
end
|
103
105
|
|
106
|
+
def check_ffmpeg_binary method
|
107
|
+
# Get or evaluate the path which can raise a Errno::ENOENT
|
108
|
+
path = FFMPEG.send method
|
109
|
+
|
110
|
+
# Check that it returns something which exists on disk
|
111
|
+
raise StandardError unless path && File.exists?(path)
|
112
|
+
|
113
|
+
rescue StandardError, Errno::ENOENT => exception
|
114
|
+
raise RestFtpDaemon::VideoMissingBinary, "missing ffmpeg binary: #{method}"
|
115
|
+
end
|
116
|
+
|
104
117
|
end
|
105
118
|
end
|
106
119
|
|
@@ -28,7 +28,7 @@ module RestFtpDaemon
|
|
28
28
|
|
29
29
|
def initialize original
|
30
30
|
unless original.is_a? String
|
31
|
-
raise RestFtpDaemon::
|
31
|
+
raise RestFtpDaemon::AssertionFailed, "location/original/string: #{original.inspect}"
|
32
32
|
end
|
33
33
|
|
34
34
|
# Strip spaces before/after, copying original "path" at the same time
|
@@ -42,7 +42,7 @@ module RestFtpDaemon
|
|
42
42
|
# Ensure result does not contain tokens after replacement
|
43
43
|
detected_tokens = detect_tokens(location_uri)
|
44
44
|
unless detected_tokens.empty?
|
45
|
-
raise RestFtpDaemon::
|
45
|
+
raise RestFtpDaemon::JobUnresolvedTokens, detected_tokens.join(' ')
|
46
46
|
end
|
47
47
|
|
48
48
|
# Parse URL and do specific initializations
|
@@ -54,7 +54,8 @@ module RestFtpDaemon
|
|
54
54
|
|
55
55
|
# Check that scheme is supported
|
56
56
|
unless @uri.scheme
|
57
|
-
raise RestFtpDaemon::
|
57
|
+
raise RestFtpDaemon::SchemeUnsupported, url
|
58
|
+
# raise RestFtpDaemon::SchemeUnsupported, @uri
|
58
59
|
end
|
59
60
|
end
|
60
61
|
|
@@ -26,7 +26,7 @@ module RestFtpDaemon
|
|
26
26
|
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
27
27
|
|
28
28
|
# Annnounce object
|
29
|
-
log_info "
|
29
|
+
log_info "RemoteBase.initialize debug[#{debug}] target[#{target.path}] "
|
30
30
|
|
31
31
|
# Prepare real object
|
32
32
|
prepare
|
@@ -54,6 +54,8 @@ module RestFtpDaemon
|
|
54
54
|
end
|
55
55
|
|
56
56
|
def close
|
57
|
+
log_debug "RemoteBase.close"
|
58
|
+
|
57
59
|
# Debug mode ?
|
58
60
|
return unless @debug
|
59
61
|
puts "-------------------- SESSION CLOSING --------------------------"
|
@@ -23,7 +23,7 @@ module RestFtpDaemon
|
|
23
23
|
@chunk_size = JOB_FTP_CHUNKMB.to_i * 1024
|
24
24
|
|
25
25
|
# Announce object
|
26
|
-
log_debug "
|
26
|
+
log_debug "RemoteFTP.prepare chunk_size:#{@chunk_size}"
|
27
27
|
end
|
28
28
|
|
29
29
|
def connect
|
@@ -36,7 +36,7 @@ module RestFtpDaemon
|
|
36
36
|
|
37
37
|
def present? target
|
38
38
|
size = @ftp.size target.path
|
39
|
-
log_debug "
|
39
|
+
log_debug "RemoteFTP.present? [#{target.name}]"
|
40
40
|
|
41
41
|
rescue Net::FTPPermError
|
42
42
|
return false
|
@@ -47,13 +47,13 @@ module RestFtpDaemon
|
|
47
47
|
def remove! target
|
48
48
|
@ftp.delete target.path
|
49
49
|
rescue Net::FTPPermError
|
50
|
-
log_debug "
|
50
|
+
log_debug "RemoteFTP.remove! [#{target.name}] not found"
|
51
51
|
else
|
52
|
-
log_debug "
|
52
|
+
log_debug "RemoteFTP.remove! [#{target.name}] removed"
|
53
53
|
end
|
54
54
|
|
55
55
|
def mkdir directory
|
56
|
-
log_debug "
|
56
|
+
log_debug "RemoteFTP.mkdir [#{directory}]"
|
57
57
|
@ftp.mkdir directory
|
58
58
|
|
59
59
|
rescue StandardError => ex
|
@@ -62,7 +62,7 @@ module RestFtpDaemon
|
|
62
62
|
|
63
63
|
def chdir_or_create directory, mkdir = false
|
64
64
|
# Init, extract my parent name and my own name
|
65
|
-
log_debug "
|
65
|
+
log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
66
66
|
parent, current = extract_parent(directory)
|
67
67
|
|
68
68
|
#dirname, _current = extract_parent(directory)
|
@@ -98,11 +98,11 @@ module RestFtpDaemon
|
|
98
98
|
end
|
99
99
|
|
100
100
|
# Move to the directory
|
101
|
-
log_debug "
|
101
|
+
log_debug "RemoteFTP.upload chdir [#{dest.dir}]"
|
102
102
|
@ftp.chdir "/#{dest.dir}"
|
103
103
|
|
104
104
|
# Do the transfer
|
105
|
-
log_debug "
|
105
|
+
log_debug "RemoteFTP.upload putbinaryfile [#{dest.name}]"
|
106
106
|
@ftp.putbinaryfile source.path, dest.name, @chunk_size do |data|
|
107
107
|
# Update job status after this block transfer
|
108
108
|
yield data.bytesize, dest.name
|
@@ -110,7 +110,7 @@ module RestFtpDaemon
|
|
110
110
|
|
111
111
|
# Move the file back to its original name
|
112
112
|
if use_temp_name
|
113
|
-
log_debug "
|
113
|
+
log_debug "RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
|
114
114
|
@ftp.rename dest.name, target.name
|
115
115
|
end
|
116
116
|
end
|
@@ -13,19 +13,20 @@ module RestFtpDaemon
|
|
13
13
|
|
14
14
|
def prepare
|
15
15
|
@multipart_threshold = MULTIPART_THRESHOLD_MB.to_i * 1024 * 1024
|
16
|
-
log_debug "
|
16
|
+
log_debug "RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
|
17
17
|
end
|
18
18
|
|
19
19
|
def connect
|
20
20
|
super
|
21
21
|
|
22
22
|
# Connect init
|
23
|
-
log_debug "
|
23
|
+
log_debug "RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
|
24
24
|
|
25
25
|
# Connect remote server
|
26
26
|
@client = Aws::S3::Resource.new(
|
27
27
|
region: @target.aws_region,
|
28
28
|
credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
|
29
|
+
# thread_count: 4,
|
29
30
|
http_wire_trace: @debug
|
30
31
|
)
|
31
32
|
#s3 = Aws::S3::Client.new(http_wire_trace: true)
|
@@ -34,7 +35,7 @@ module RestFtpDaemon
|
|
34
35
|
def upload source, target, use_temp_name = false, &callback
|
35
36
|
# Push init
|
36
37
|
raise RestFtpDaemon::AssertionFailed, "upload/client" if @client.nil?
|
37
|
-
log_debug "
|
38
|
+
log_debug "RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
|
38
39
|
|
39
40
|
# Update progress before
|
40
41
|
#yield 0, target.name
|
@@ -63,8 +64,8 @@ module RestFtpDaemon
|
|
63
64
|
#yield target.size, target.name
|
64
65
|
|
65
66
|
# Dump information about this file
|
66
|
-
log_debug "
|
67
|
-
log_debug "
|
67
|
+
log_debug "RemoteS3.upload url[#{object.public_url}]"
|
68
|
+
log_debug "RemoteS3.upload etag[#{object.etag}]"
|
68
69
|
set_info :target_aws_public_url, object.public_url
|
69
70
|
set_info :target_aws_etag, object.etag
|
70
71
|
end
|
@@ -15,7 +15,7 @@ module RestFtpDaemon
|
|
15
15
|
super
|
16
16
|
|
17
17
|
# Connect init
|
18
|
-
log_debug "
|
18
|
+
log_debug "RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
|
19
19
|
|
20
20
|
# Debug level
|
21
21
|
verbosity = @debug ? Logger::DEBUG : false
|
@@ -31,7 +31,7 @@ module RestFtpDaemon
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def present? target
|
34
|
-
log_debug "
|
34
|
+
log_debug "RemoteSFTP.present? [#{target.name}]"
|
35
35
|
stat = @sftp.stat! target.path
|
36
36
|
|
37
37
|
rescue Net::SFTP::StatusException
|
@@ -41,7 +41,7 @@ module RestFtpDaemon
|
|
41
41
|
end
|
42
42
|
|
43
43
|
def remove! target
|
44
|
-
log_debug "
|
44
|
+
log_debug "RemoteSFTP.remove! [#{target.name}]"
|
45
45
|
@sftp.remove target.path
|
46
46
|
|
47
47
|
rescue Net::SFTP::StatusException
|
@@ -51,7 +51,7 @@ module RestFtpDaemon
|
|
51
51
|
end
|
52
52
|
|
53
53
|
def mkdir directory
|
54
|
-
log_debug "
|
54
|
+
log_debug "RemoteSFTP.mkdir [#{directory}]"
|
55
55
|
@sftp.mkdir! directory
|
56
56
|
|
57
57
|
rescue StandardError => ex
|
@@ -60,7 +60,7 @@ module RestFtpDaemon
|
|
60
60
|
|
61
61
|
def chdir_or_create directory, mkdir = false
|
62
62
|
# Init, extract my parent name and my own name
|
63
|
-
log_debug "
|
63
|
+
log_debug "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
64
64
|
parent, _current = extract_parent(directory)
|
65
65
|
|
66
66
|
# Access this directory
|
@@ -99,7 +99,7 @@ module RestFtpDaemon
|
|
99
99
|
end
|
100
100
|
|
101
101
|
# Do the transfer
|
102
|
-
log_debug "
|
102
|
+
log_debug "RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
|
103
103
|
@sftp.upload! source.path, dest.path do |event, _uploader, *args|
|
104
104
|
case event
|
105
105
|
when :open then
|
@@ -125,7 +125,7 @@ module RestFtpDaemon
|
|
125
125
|
# Move the file back to its original name
|
126
126
|
if use_temp_name
|
127
127
|
flags = 0x00000001
|
128
|
-
log_debug "
|
128
|
+
log_debug "RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
|
129
129
|
@sftp.rename! dest.path, target.path, flags
|
130
130
|
end
|
131
131
|
|
@@ -42,7 +42,7 @@
|
|
42
42
|
%th.text-right{title: "updated every #{JOB_FTP_CHUNKMB} MB block"} rate
|
43
43
|
%th{title: "Worker ID"} W
|
44
44
|
%th{title: "Priority"} P
|
45
|
-
%th{title: "
|
45
|
+
%th{title: "Tentatives count"} T
|
46
46
|
|
47
47
|
- @jobs_queued.each do |pool, jobs|
|
48
48
|
- unless jobs.empty?
|
@@ -13,7 +13,6 @@
|
|
13
13
|
- job_working = [JOB_STATUS_UPLOADING, JOB_STATUS_TRANSFORMING].include? job.status
|
14
14
|
|
15
15
|
- trclass = JOB_STYLES[job.status]
|
16
|
-
- runs = job.runs.to_i
|
17
16
|
|
18
17
|
- unless job.error.nil?
|
19
18
|
- trclass = "warning"
|
@@ -84,4 +83,4 @@
|
|
84
83
|
.label.label-default.flag.worker-label= job.priority
|
85
84
|
|
86
85
|
%td
|
87
|
-
.label.flag.worker-label{class:
|
86
|
+
.label.flag.worker-label{class: job_tentatives_style(job.tentatives)}= job.tentatives
|
@@ -10,7 +10,7 @@ module RestFtpDaemon
|
|
10
10
|
config_section :reporter
|
11
11
|
|
12
12
|
# Other configuration options
|
13
|
-
@
|
13
|
+
@feature_newrelic = Conf.feature?(:newrelic)
|
14
14
|
|
15
15
|
# Check that everything is OK
|
16
16
|
return "invalid timer" unless @config[:timer].to_i > 0
|
@@ -34,9 +34,6 @@ module RestFtpDaemon
|
|
34
34
|
private
|
35
35
|
|
36
36
|
def do_metrics
|
37
|
-
# What metrics to report?
|
38
|
-
report_newrelic = Conf.feature?(:newrelic)
|
39
|
-
|
40
37
|
# Get common metrics and dump them to logs
|
41
38
|
log_debug "begin metrics sample"
|
42
39
|
metrics = Metrics.sample
|
@@ -46,10 +43,10 @@ module RestFtpDaemon
|
|
46
43
|
log_error "unable to collect metrics"
|
47
44
|
return
|
48
45
|
end
|
49
|
-
log_info "collected metrics (newrelic: #{@
|
46
|
+
log_info "collected metrics (newrelic: #{@feature_newrelic.inspect})", metrics
|
50
47
|
|
51
48
|
# Transpose metrics to NewRelic metrics
|
52
|
-
report_newrelic(metrics) if @
|
49
|
+
report_newrelic(metrics) if @feature_newrelic
|
53
50
|
end
|
54
51
|
|
55
52
|
def report_newrelic metrics
|
@@ -56,20 +56,17 @@ module RestFtpDaemon
|
|
56
56
|
job.process
|
57
57
|
end
|
58
58
|
|
59
|
-
# Processing done
|
60
|
-
worker_status WORKER_STATUS_FINISHED, job
|
61
|
-
|
62
59
|
# Increment total processed jobs count
|
63
60
|
RestFtpDaemon::Counters.instance.increment :jobs, :processed
|
64
61
|
|
65
62
|
rescue RestFtpDaemon::JobTimeout => ex
|
66
|
-
log_error "JOB
|
67
|
-
worker_status WORKER_STATUS_TIMEOUT
|
63
|
+
log_error "JOB TIMEOUT", ex.backtrace
|
64
|
+
worker_status WORKER_STATUS_TIMEOUT, job
|
68
65
|
|
69
66
|
# Inform the job
|
70
67
|
job.oops_you_stop_now ex unless job.nil?
|
71
68
|
|
72
|
-
rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::
|
69
|
+
rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::JobAttributeMissing, StandardError => ex
|
73
70
|
log_error "JOB EXCEPTION ex[#{ex.class}] #{ex.message}", ex.backtrace
|
74
71
|
worker_status WORKER_STATUS_CRASHED
|
75
72
|
|
@@ -80,30 +77,61 @@ module RestFtpDaemon
|
|
80
77
|
def handle_job_result job
|
81
78
|
# If job status requires a retry, just restack it
|
82
79
|
if !job.error
|
83
|
-
#
|
80
|
+
# Processing successful
|
81
|
+
log_error "job finished with no error"
|
82
|
+
worker_status WORKER_STATUS_FINISHED, job
|
84
83
|
|
85
|
-
elsif
|
86
|
-
log_error "not retrying
|
84
|
+
elsif error_not_eligible(job)
|
85
|
+
log_error "not retrying [#{job.error}] retry_on not eligible"
|
87
86
|
|
88
|
-
elsif
|
89
|
-
log_error "not retrying
|
87
|
+
elsif error_reached_for(job)
|
88
|
+
log_error "not retrying [#{job.error}] retry_for reached [#{@config[:retry_for]}s]"
|
90
89
|
|
91
|
-
elsif
|
92
|
-
log_error "not retrying
|
90
|
+
elsif error_reached_max(job)
|
91
|
+
log_error "not retrying [#{job.error}] retry_max reached #{tentatives(job)}"
|
93
92
|
|
94
93
|
else
|
95
94
|
# Delay cannot be negative, and will be 1s minimum
|
96
95
|
retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
|
97
|
-
log_info "
|
96
|
+
log_info "retry job [#{job.id}] in [#{retry_after}s] tried #{tentatives(job)}"
|
98
97
|
|
99
98
|
# Wait !
|
99
|
+
worker_status WORKER_STATUS_RETRYING, job
|
100
100
|
sleep retry_after
|
101
|
-
|
101
|
+
log_debug "job [#{job.id}] requeued after [#{retry_after}s] delay"
|
102
102
|
|
103
103
|
# Now, requeue this job
|
104
104
|
RestFtpDaemon::JobQueue.instance.requeue job
|
105
105
|
end
|
106
106
|
end
|
107
107
|
|
108
|
+
def error_not_eligible job
|
109
|
+
# No, if no eligible errors
|
110
|
+
return true unless @config[:retry_on].is_a?(Enumerable)
|
111
|
+
|
112
|
+
# Tell if this error is in the list
|
113
|
+
return !@config[:retry_on].include?(job.error.to_s)
|
114
|
+
end
|
115
|
+
|
116
|
+
def error_reached_for job
|
117
|
+
# Not above, if no limit definded
|
118
|
+
return false unless @config[:retry_for]
|
119
|
+
|
120
|
+
# Job age above this limit
|
121
|
+
return job.age >= @config[:retry_for]
|
122
|
+
end
|
123
|
+
|
124
|
+
def error_reached_max job
|
125
|
+
# Not above, if no limit definded
|
126
|
+
return false unless @config[:retry_max]
|
127
|
+
|
128
|
+
# Job age above this limit
|
129
|
+
return job.tentatives >= @config[:retry_max]
|
130
|
+
end
|
131
|
+
|
132
|
+
def tentatives job
|
133
|
+
"[#{job.tentatives}/#{@config[:retry_max]}]"
|
134
|
+
end
|
135
|
+
|
108
136
|
end
|
109
137
|
end
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
|
4
4
|
# Project version
|
5
|
-
spec.version = "0.
|
5
|
+
spec.version = "0.432.0"
|
6
6
|
|
7
7
|
# Project description
|
8
8
|
spec.name = "rest-ftp-daemon"
|
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
|
|
20
20
|
reject{ |f| f =~ /^dashboard.+\.png/ }
|
21
21
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
22
22
|
spec.require_paths = ["lib"]
|
23
|
-
spec.required_ruby_version = ">= 2.
|
23
|
+
spec.required_ruby_version = ">= 2.3"
|
24
24
|
|
25
25
|
# Development dependencies
|
26
26
|
spec.add_development_dependency "bundler", "~> 1.6"
|
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.add_development_dependency "http"
|
32
32
|
|
33
33
|
# Runtime dependencies
|
34
|
-
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.
|
34
|
+
spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.12"
|
35
35
|
spec.add_runtime_dependency "json", "~> 1.8"
|
36
36
|
spec.add_runtime_dependency "thin", "~> 1.7"
|
37
37
|
spec.add_runtime_dependency "activesupport", "~> 4.2"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.432.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -100,14 +100,14 @@ dependencies:
|
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0.3.
|
103
|
+
version: 0.3.12
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0.3.
|
110
|
+
version: 0.3.12
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: json
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -524,7 +524,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
524
524
|
requirements:
|
525
525
|
- - ">="
|
526
526
|
- !ruby/object:Gem::Version
|
527
|
-
version: 2.
|
527
|
+
version: '2.3'
|
528
528
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
529
529
|
requirements:
|
530
530
|
- - ">="
|
@@ -532,7 +532,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
532
532
|
version: '0'
|
533
533
|
requirements: []
|
534
534
|
rubyforge_project:
|
535
|
-
rubygems_version: 2.
|
535
|
+
rubygems_version: 2.6.6
|
536
536
|
signing_key:
|
537
537
|
specification_version: 4
|
538
538
|
summary: RESTful FTP client daemon
|