rest-ftp-daemon 0.430.1 → 0.432.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 305203e95b7418a3a330d556235fb616abdd7073
4
- data.tar.gz: 6cc68e4dd51423492b07efc96293ff9ca04d5f42
3
+ metadata.gz: 89e094d356f8271d07f64222768a60997ce50a88
4
+ data.tar.gz: f3deb47de89bc54b1b5f807aa8ecfc868d0c1926
5
5
  SHA512:
6
- metadata.gz: afa1b54735152ca343dd641a30d819cb3068d4d9010e3b71f9d27c030e87c01bdeab6b00cf655833b9f1e8eda1c419d95a8d914e17ecfb2e1e5372c6f024dd68
7
- data.tar.gz: 0d1149d6719f7c08cbcdca104fcfaea357fd18536d67c12eebc0f3b77f5b7f1b1a9c0fcb07cf31b6bc25572694fc219872b77dc7484a1b521827e3b4723b1a5f
6
+ metadata.gz: ad21099fd7392bc545888394974d95c1e0ce8204e28de1682e3b93087d38c95f20e6594ec59ecc465f562625b9c1def500076aa7a8c3695b1edddea8bd0ff754
7
+ data.tar.gz: 952ac92bf57af92ec2c99570d5fff30aefcf8b8bf9ba21f3f661b70c8af801757f55ed5c876fc3df590ec8ce708d1ed882bf0ab85cbd6ffbc68ccfc7f397a12a
data/Gemfile.lock CHANGED
@@ -1,11 +1,11 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- rest-ftp-daemon (0.430.1)
4
+ rest-ftp-daemon (0.432.0)
5
5
  activesupport (~> 4.2)
6
6
  api-auth
7
7
  aws-sdk-resources (~> 2)
8
- bmc-daemon-lib (~> 0.3.10)
8
+ bmc-daemon-lib (~> 0.3.12)
9
9
  double-bag-ftps
10
10
  facter
11
11
  get_process_mem
@@ -39,15 +39,15 @@ GEM
39
39
  ast (2.3.0)
40
40
  astrolabe (1.3.1)
41
41
  parser (~> 2.2)
42
- aws-sdk-core (2.5.10)
42
+ aws-sdk-core (2.5.11)
43
43
  jmespath (~> 1.0)
44
- aws-sdk-resources (2.5.10)
45
- aws-sdk-core (= 2.5.10)
44
+ aws-sdk-resources (2.5.11)
45
+ aws-sdk-core (= 2.5.11)
46
46
  axiom-types (0.1.1)
47
47
  descendants_tracker (~> 0.0.4)
48
48
  ice_nine (~> 0.11.0)
49
49
  thread_safe (~> 0.3, >= 0.3.1)
50
- bmc-daemon-lib (0.3.10)
50
+ bmc-daemon-lib (0.3.12)
51
51
  chamber (~> 2.9)
52
52
  builder (3.2.2)
53
53
  chamber (2.9.1)
data/README.md CHANGED
@@ -289,7 +289,7 @@ TODO for this document
289
289
  Debian install preparation
290
290
  ------------------------------------------------------------------------------------
291
291
 
292
- This project is available as a rubygem, requires Ruby 2.2.2 and RubyGems installed.
292
+ This project is available as a rubygem, requires Ruby 2.3.0 and RubyGems installed.
293
293
 
294
294
  #### Using rbenv and ruby-build
295
295
 
@@ -302,15 +302,15 @@ You may have to install some extra packages for the compilations to complete.
302
302
  # git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
303
303
  # echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
304
304
  # echo 'eval "$(rbenv init -)"' >> ~/.bashrc
305
- # rbenv install --list | grep '2.2'
305
+ # rbenv install --list | grep '2.3'
306
306
  ```
307
307
 
308
308
  ```
309
309
  # curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.2.3
310
310
  ```
311
311
 
312
- Otherwise, you way have to update ruby-build to include Ruby 2.2 definitions.
313
- On Debian, 2.2 is not included in Wheezy and appears in Jessie's version of the package.
312
+ Otherwise, you way have to update ruby-build to include Ruby 2.3 definitions.
313
+ On Debian, 2.3 is not included in Wheezy and appears in Jessie's version of the package.
314
314
 
315
315
  #### Dedicated user
316
316
 
@@ -326,8 +326,8 @@ Use a dedicated user for the daemon, switch to this user and enable rbenv
326
326
  Install the right ruby version and activate it
327
327
 
328
328
  ```
329
- # rbenv install 2.2.2
330
- # rbenv local 2.2.2
329
+ # rbenv install 2.3.0
330
+ # rbenv local 2.3.0
331
331
  # rbenv rehash
332
332
  ```
333
333
 
data/defaults.yml CHANGED
@@ -35,8 +35,10 @@ transfer:
35
35
  debug_sftp: false
36
36
 
37
37
  retry_on: # job error values that will allow a retry
38
+ - ftp_temp_error
38
39
  - ftp_perm_error
39
40
  - net_temp_error
41
+ - net_perm_error
40
42
  - conn_reset_by_peer
41
43
  - conn_timed_out
42
44
  - conn_refused
@@ -66,16 +68,16 @@ logs:
66
68
  path: "/tmp/"
67
69
  level: debug
68
70
 
69
- thin: "rftpd-thin.log"
70
- default: "rftpd-core.log"
71
+ thin: "rftpd-default-thin.log"
72
+ default: "rftpd-default-core.log"
71
73
 
72
- queue: "rftpd-core.log"
73
- api: "rftpd-core.log"
74
- workers: "rftpd-core.log"
75
- transfer: "rftpd-workers.log"
76
- conchita: "rftpd-workers.log"
77
- reporter: "rftpd-workers.log"
78
- notify: "rftpd-workers.log"
74
+ queue: "rftpd-default-core.log"
75
+ api: "rftpd-default-core.log"
76
+ workers: "rftpd-default-core.log"
77
+ transfer: "rftpd-default-workers.log"
78
+ conchita: "rftpd-default-workers.log"
79
+ reporter: "rftpd-default-workers.log"
80
+ notify: "rftpd-default-workers.log"
79
81
 
80
- newrelic: "rftpd-newrelic.log"
81
- rollbar: "rftpd-rollbar.log"
82
+ newrelic: "rftpd-default-newrelic.log"
83
+ rollbar: "rftpd-default-rollbar.log"
@@ -15,7 +15,7 @@ module RestFtpDaemon
15
15
  rescue_from RestFtpDaemon::QueueCantCreateJob do |exception|
16
16
  exception_error :api_cant_create_job, 422, exception
17
17
  end
18
- rescue_from RestFtpDaemon::UnresolvedTokens do |exception|
18
+ rescue_from RestFtpDaemon::JobUnresolvedTokens do |exception|
19
19
  exception_error :api_unresolved_tokens, 422, exception
20
20
  end
21
21
 
@@ -101,15 +101,19 @@ WORKER_STATUS_STARTING = "starting"
101
101
  WORKER_STATUS_WAITING = "waiting"
102
102
  WORKER_STATUS_RUNNING = "running"
103
103
  WORKER_STATUS_FINISHED = "finished"
104
+ WORKER_STATUS_RETRYING = "retrying"
104
105
  WORKER_STATUS_TIMEOUT = "timeout"
105
106
  WORKER_STATUS_CRASHED = "crashed"
106
107
  WORKER_STATUS_CLEANING = "cleaning"
107
108
  WORKER_STATUS_REPORTING = "reporting"
108
109
  WORKER_STYLES = {
109
- WORKER_STATUS_WAITING => nil,
110
- WORKER_STATUS_RUNNING => :info,
111
- WORKER_STATUS_CRASHED => :danger,
112
- WORKER_STATUS_FINISHED => :success,
110
+ WORKER_STATUS_WAITING => nil,
111
+ WORKER_STATUS_RUNNING => :info,
112
+ WORKER_STATUS_REPORTING => :info,
113
+ WORKER_STATUS_CLEANING => :info,
114
+ WORKER_STATUS_RETRYING => :warning,
115
+ WORKER_STATUS_CRASHED => :danger,
116
+ WORKER_STATUS_FINISHED => :success,
113
117
  }
114
118
 
115
119
 
@@ -1,31 +1,31 @@
1
1
  module RestFtpDaemon
2
2
 
3
- class RestFtpDaemonException < StandardError; end
3
+ class BaseException < StandardError; end
4
4
 
5
- class DummyException < RestFtpDaemonException; end
5
+ class AssertionFailed < BaseException; end
6
6
 
7
- class MissingPool < RestFtpDaemonException; end
8
- class InvalidWorkerNumber < RestFtpDaemonException; end
9
- class QueueCantCreateJob < RestFtpDaemonException; end
10
- class JobException < RestFtpDaemonException; end
11
- class JobTimeout < RestFtpDaemonException; end
12
- class JobNotFound < RestFtpDaemonException; end
7
+ class InvalidWorkerNumber < BaseException; end
8
+ class QueueCantCreateJob < BaseException; end
9
+ class JobTimeout < BaseException; end
10
+ class JobNotFound < BaseException; end
13
11
 
12
+ class JobException < BaseException; end
13
+ class JobAttributeMissing < BaseException; end
14
+ class JobUnresolvedTokens < BaseException; end
14
15
 
15
- class AttributeMissing < RestFtpDaemonException; end
16
- class AssertionFailed < RestFtpDaemonException; end
17
- class UnresolvedTokens < RestFtpDaemonException; end
18
- class LocationMalformed < RestFtpDaemonException; end
19
- class LocationParseError < RestFtpDaemonException; end
20
- class UnsupportedScheme < RestFtpDaemonException; end
21
- class MissingFfmpegLibraries < RestFtpDaemonException; end
16
+ class LocationParseError < BaseException; end
17
+ class SchemeUnsupported < BaseException; end
22
18
 
23
- class SourceNotSupported < RestFtpDaemonException; end
24
- class SourceNotFound < RestFtpDaemonException; end
19
+ class SourceUnsupported < BaseException; end
20
+ class SourceNotFound < BaseException; end
21
+
22
+ class TargetUnsupported < BaseException; end
23
+ class TargetFileExists < BaseException; end
24
+ class TargetDirectoryError < BaseException; end
25
+ class TargetPermissionError < BaseException; end
26
+
27
+ class VideoMissingBinary < BaseException; end
28
+ class VideoMovieError < BaseException; end
25
29
 
26
- class TargetNotSupported < RestFtpDaemonException; end
27
- class TargetFileExists < RestFtpDaemonException; end
28
- class TargetDirectoryError < RestFtpDaemonException; end
29
- class TargetPermissionError < RestFtpDaemonException; end
30
30
 
31
31
  end
@@ -23,11 +23,11 @@ module RestFtpDaemon
23
23
  "#{MOUNT_JOBS}/#{job.id}" if job.respond_to? :id
24
24
  end
25
25
 
26
- def job_runs_style runs
27
- return "label-outline" if runs <= 0
28
- return "label-info" if runs == 1
29
- return "label-warning" if runs == 2
30
- return "label-danger" if runs > 2
26
+ def job_tentatives_style count
27
+ return "label-outline" if count <= 0
28
+ return "label-info" if count == 1
29
+ return "label-warning" if count == 2
30
+ return "label-danger" if count > 2
31
31
  end
32
32
 
33
33
  def location_style uri
@@ -26,7 +26,7 @@ module RestFtpDaemon
26
26
  attr_reader :id
27
27
  attr_reader :error
28
28
  attr_reader :status
29
- attr_reader :runs
29
+ attr_reader :tentatives
30
30
 
31
31
  attr_reader :queued_at
32
32
  attr_reader :updated_at
@@ -55,7 +55,7 @@ module RestFtpDaemon
55
55
  @updated_at = nil
56
56
  @error = nil
57
57
  @status = nil
58
- @runs = 0
58
+ @tentatives = 0
59
59
  @wid = nil
60
60
 
61
61
  # Prepare configuration
@@ -76,11 +76,11 @@ module RestFtpDaemon
76
76
  @pool = DEFAULT_POOL unless @pools.keys.include?(@pool)
77
77
 
78
78
  # Prepare sources/target
79
- raise RestFtpDaemon::AttributeMissing, "source" unless params[:source]
79
+ raise RestFtpDaemon::JobAttributeMissing, "source" unless params[:source]
80
80
  @source_loc = Location.new(params[:source])
81
81
  log_info "Job.initialize source #{@source_loc.uri}"
82
82
 
83
- raise RestFtpDaemon::AttributeMissing, "target" unless params[:target]
83
+ raise RestFtpDaemon::JobAttributeMissing, "target" unless params[:target]
84
84
  @target_loc = Location.new(params[:target])
85
85
  log_info "Job.initialize target #{@target_loc.uri}"
86
86
  end
@@ -89,6 +89,9 @@ module RestFtpDaemon
89
89
  # Update job status
90
90
  set_status JOB_STATUS_PREPARING
91
91
 
92
+ # Increment run cours
93
+ @tentatives +=1
94
+
92
95
  # Flag current job timestamps
93
96
  @queued_at = Time.now
94
97
  @updated_at = Time.now
@@ -101,7 +104,7 @@ module RestFtpDaemon
101
104
  set_status JOB_STATUS_QUEUED
102
105
  set_error nil
103
106
  client_notify :queued
104
- log_info "Job.reset notify[queued]"
107
+ log_info "Job.reset notify[queued] tentative[#{@tentatives}]"
105
108
  end
106
109
 
107
110
  # Process job
@@ -132,7 +135,7 @@ module RestFtpDaemon
132
135
  do_after
133
136
 
134
137
  rescue StandardError => exception
135
- Rollbar.error "process: #{exception.class.name}: #{exception.message}"
138
+ Rollbar.error "Job.process: #{exception.class.name}: #{exception.message}"
136
139
  return oops current_signal, exception
137
140
 
138
141
  else
@@ -159,7 +162,7 @@ module RestFtpDaemon
159
162
 
160
163
  def weight
161
164
  @weight = [
162
- - @runs.to_i,
165
+ - @tentatives.to_i,
163
166
  + @priority.to_i,
164
167
  - @queued_at.to_i,
165
168
  ]
@@ -12,12 +12,13 @@ module RestFtpDaemon
12
12
  invalid_argument: Errno::EINVAL,
13
13
  runtime_error: RuntimeError,
14
14
 
15
- source_not_supported: RestFtpDaemon::SourceNotSupported,
15
+ job_timeout: RestFtpDaemon::JobTimeout,
16
+ source_not_supported: RestFtpDaemon::SourceUnsupported,
16
17
  source_not_found: RestFtpDaemon::SourceNotFound,
17
18
  target_file_exists: RestFtpDaemon::TargetFileExists,
18
19
  target_directory_error: RestFtpDaemon::TargetDirectoryError,
19
20
  target_permission_error: RestFtpDaemon::TargetPermissionError,
20
- target_not_supported: RestFtpDaemon::TargetNotSupported,
21
+ target_not_supported: RestFtpDaemon::TargetUnsupported,
21
22
  assertion_failed: RestFtpDaemon::AssertionFailed,
22
23
  location_parse_error: RestFtpDaemon::LocationParseError,
23
24
 
@@ -58,9 +59,20 @@ module RestFtpDaemon
58
59
  sftp_key_mismatch: Net::SSH::HostKeyMismatch,
59
60
  sftp_auth_failed: Net::SSH::AuthenticationFailed,
60
61
  sftp_openssl_error: OpenSSL::SSL::SSLError,
62
+
63
+ video_missing_binary: RestFtpDaemon::VideoMissingBinary,
64
+ video_movie_error: RestFtpDaemon::VideoMovieError,
65
+
61
66
  # rescue Encoding::UndefinedConversionError => exception
62
67
  # return oops :ended, exception, "encoding_error", true
63
68
  }
64
69
 
70
+ class InvalidWorkerNumber < BaseException; end
71
+ class QueueCantCreateJob < BaseException; end
72
+ class JobException < BaseException; end
73
+ class JobNotFound < BaseException; end
74
+
75
+
76
+
65
77
  end
66
78
  end
@@ -16,26 +16,26 @@ module RestFtpDaemon
16
16
  set_info INFO_SOURCE_PROCESSED, 0
17
17
 
18
18
  # Ensure source is FILE
19
- raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless @source_loc.is? URI::FILE
19
+ raise RestFtpDaemon::SourceUnsupported, @source_loc.scheme unless @source_loc.is? URI::FILE
20
20
 
21
21
  # Prepare remote object
22
22
  case target_uri
23
23
  when URI::FTP
24
- log_info "JobTransfer.before target_method FTP"
24
+ log_info "JobTransfer.do_before target_method FTP"
25
25
  @remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftp]
26
26
  when URI::FTPES, URI::FTPS
27
- log_info "JobTransfer.before target_method FTPES/FTPS"
27
+ log_info "JobTransfer.do_before target_method FTPES/FTPS"
28
28
  @remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
29
29
  when URI::SFTP
30
- log_info "JobTransfer.before target_method SFTP"
30
+ log_info "JobTransfer.do_before target_method SFTP"
31
31
  @remote = Remote::RemoteSFTP.new @target_loc, log_prefix, @config[:debug_sftp]
32
32
  when URI::S3
33
- log_info "JobTransfer.before target_method S3"
33
+ log_info "JobTransfer.do_before target_method S3"
34
34
  @remote = Remote::RemoteS3.new @target_loc, log_prefix, @config[:debug_s3]
35
35
  else
36
36
  message = "unknown scheme [#{@target_loc.scheme}] [#{target_uri.class.name}]"
37
- log_info "JobTransfer.before #{message}"
38
- raise RestFtpDaemon::TargetNotSupported, message
37
+ log_info "JobTransfer.do_before #{message}"
38
+ raise RestFtpDaemon::TargetUnsupported, message
39
39
  end
40
40
 
41
41
  # Plug this Job into @remote to allow it to log
@@ -51,7 +51,7 @@ module RestFtpDaemon
51
51
  sources = @source_loc.scan_files
52
52
  set_info INFO_SOURCE_COUNT, sources.size
53
53
  set_info INFO_SOURCE_FILES, sources.collect(&:name)
54
- log_info "JobTransfer.work sources #{sources.collect(&:name)}"
54
+ log_info "JobTransfer.do_work sources #{sources.collect(&:name)}"
55
55
  raise RestFtpDaemon::SourceNotFound if sources.empty?
56
56
 
57
57
  # Guess target file name, and fail if present while we matched multiple sources
@@ -95,9 +95,10 @@ module RestFtpDaemon
95
95
 
96
96
  def do_after
97
97
  # Close FTP connexion and free up memory
98
+ log_info "JobTransfer.do_after close connexion, update status and counters"
98
99
  @remote.close
99
100
 
100
- # Free-up remote object
101
+ # Free @remote object
101
102
  @remote = nil
102
103
 
103
104
  # Update job status
@@ -16,16 +16,14 @@ module RestFtpDaemon
16
16
  log_info "JobVideo.before target_loc.path: #{@target_loc.path}"
17
17
 
18
18
  # Ensure FFMPEG lib is available
19
- ffmpeg_binary_path = FFMPEG.ffmpeg_binary
20
- unless ffmpeg_binary_path && File.exists?(ffmpeg_binary_path)
21
- raise RestFtpDaemon::MissingFfmpegLibraries, ffmpeg_binary_path
22
- end
19
+ check_ffmpeg_binary :ffmpeg_binary
20
+ check_ffmpeg_binary :ffprobe_binary
23
21
 
24
22
  # Ensure source and target are FILE
25
- raise RestFtpDaemon::AssertionFailed unless @video_options.is_a? Hash
26
- raise RestFtpDaemon::AssertionFailed unless @video_custom.is_a? Hash
27
- raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless @source_loc.is? URI::FILE
28
- raise RestFtpDaemon::TargetNotSupported, @target_loc.scheme unless @target_loc.is? URI::FILE
23
+ raise RestFtpDaemon::AssertionFailed unless @video_options.is_a? Hash
24
+ raise RestFtpDaemon::AssertionFailed unless @video_custom.is_a? Hash
25
+ raise RestFtpDaemon::SourceUnsupported, @source_loc.scheme unless @source_loc.is? URI::FILE
26
+ raise RestFtpDaemon::TargetUnsupported, @target_loc.scheme unless @target_loc.is? URI::FILE
29
27
  end
30
28
 
31
29
  def do_work
@@ -57,13 +55,17 @@ module RestFtpDaemon
57
55
  end
58
56
 
59
57
  def ffmpeg_command source, target
60
- set_info INFO_SOURCE_CURRENT, source.name
61
-
62
58
  # Read info about source file
63
- movie = FFMPEG::Movie.new(source.path)
64
- set_info :ffmpeg_size, movie.size
65
- set_info :ffmpeg_duration, movie.duration
66
- set_info :ffmpeg_resolution, movie.resolution
59
+ set_info INFO_SOURCE_CURRENT, source.name
60
+ begin
61
+ movie = FFMPEG::Movie.new(source.path)
62
+ rescue StandardError => exception
63
+ raise RestFtpDaemon::VideoMovieError, exception.message
64
+ else
65
+ set_info :ffmpeg_size, movie.size
66
+ set_info :ffmpeg_duration, movie.duration
67
+ set_info :ffmpeg_resolution, movie.resolution
68
+ end
67
69
 
68
70
  # Build options
69
71
  options = {
@@ -101,6 +103,17 @@ module RestFtpDaemon
101
103
  return custom_parts
102
104
  end
103
105
 
106
+ def check_ffmpeg_binary method
107
+ # Get or evaluate the path which can raise a Errno::ENOENT
108
+ path = FFMPEG.send method
109
+
110
+ # Check that it returns something which exists on disk
111
+ raise StandardError unless path && File.exists?(path)
112
+
113
+ rescue StandardError, Errno::ENOENT => exception
114
+ raise RestFtpDaemon::VideoMissingBinary, "missing ffmpeg binary: #{method}"
115
+ end
116
+
104
117
  end
105
118
  end
106
119
 
@@ -28,7 +28,7 @@ module RestFtpDaemon
28
28
 
29
29
  def initialize original
30
30
  unless original.is_a? String
31
- raise RestFtpDaemon::LocationMalformed, original.inspect
31
+ raise RestFtpDaemon::AssertionFailed, "location/original/string: #{original.inspect}"
32
32
  end
33
33
 
34
34
  # Strip spaces before/after, copying original "path" at the same time
@@ -42,7 +42,7 @@ module RestFtpDaemon
42
42
  # Ensure result does not contain tokens after replacement
43
43
  detected_tokens = detect_tokens(location_uri)
44
44
  unless detected_tokens.empty?
45
- raise RestFtpDaemon::UnresolvedTokens, detected_tokens.join(' ')
45
+ raise RestFtpDaemon::JobUnresolvedTokens, detected_tokens.join(' ')
46
46
  end
47
47
 
48
48
  # Parse URL and do specific initializations
@@ -54,7 +54,8 @@ module RestFtpDaemon
54
54
 
55
55
  # Check that scheme is supported
56
56
  unless @uri.scheme
57
- raise RestFtpDaemon::UnsupportedScheme, url
57
+ raise RestFtpDaemon::SchemeUnsupported, url
58
+ # raise RestFtpDaemon::SchemeUnsupported, @uri
58
59
  end
59
60
  end
60
61
 
@@ -26,7 +26,7 @@ module RestFtpDaemon
26
26
  @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
27
27
 
28
28
  # Annnounce object
29
- log_info "Remote.initialize debug[#{debug}] target[#{target.path}] "
29
+ log_info "RemoteBase.initialize debug[#{debug}] target[#{target.path}] "
30
30
 
31
31
  # Prepare real object
32
32
  prepare
@@ -54,6 +54,8 @@ module RestFtpDaemon
54
54
  end
55
55
 
56
56
  def close
57
+ log_debug "RemoteBase.close"
58
+
57
59
  # Debug mode ?
58
60
  return unless @debug
59
61
  puts "-------------------- SESSION CLOSING --------------------------"
@@ -23,7 +23,7 @@ module RestFtpDaemon
23
23
  @chunk_size = JOB_FTP_CHUNKMB.to_i * 1024
24
24
 
25
25
  # Announce object
26
- log_debug "Remote::RemoteFTP.prepare chunk_size:#{@chunk_size}"
26
+ log_debug "RemoteFTP.prepare chunk_size:#{@chunk_size}"
27
27
  end
28
28
 
29
29
  def connect
@@ -36,7 +36,7 @@ module RestFtpDaemon
36
36
 
37
37
  def present? target
38
38
  size = @ftp.size target.path
39
- log_debug "Remote::RemoteFTP.present? [#{target.name}]"
39
+ log_debug "RemoteFTP.present? [#{target.name}]"
40
40
 
41
41
  rescue Net::FTPPermError
42
42
  return false
@@ -47,13 +47,13 @@ module RestFtpDaemon
47
47
  def remove! target
48
48
  @ftp.delete target.path
49
49
  rescue Net::FTPPermError
50
- log_debug "Remote::RemoteFTP.remove! [#{target.name}] not found"
50
+ log_debug "RemoteFTP.remove! [#{target.name}] not found"
51
51
  else
52
- log_debug "Remote::RemoteFTP.remove! [#{target.name}] removed"
52
+ log_debug "RemoteFTP.remove! [#{target.name}] removed"
53
53
  end
54
54
 
55
55
  def mkdir directory
56
- log_debug "Remote::RemoteFTP.mkdir [#{directory}]"
56
+ log_debug "RemoteFTP.mkdir [#{directory}]"
57
57
  @ftp.mkdir directory
58
58
 
59
59
  rescue StandardError => ex
@@ -62,7 +62,7 @@ module RestFtpDaemon
62
62
 
63
63
  def chdir_or_create directory, mkdir = false
64
64
  # Init, extract my parent name and my own name
65
- log_debug "Remote::RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
65
+ log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
66
66
  parent, current = extract_parent(directory)
67
67
 
68
68
  #dirname, _current = extract_parent(directory)
@@ -98,11 +98,11 @@ module RestFtpDaemon
98
98
  end
99
99
 
100
100
  # Move to the directory
101
- log_debug "Remote::RemoteFTP.upload chdir [#{dest.dir}]"
101
+ log_debug "RemoteFTP.upload chdir [#{dest.dir}]"
102
102
  @ftp.chdir "/#{dest.dir}"
103
103
 
104
104
  # Do the transfer
105
- log_debug "Remote::RemoteFTP.upload putbinaryfile [#{dest.name}]"
105
+ log_debug "RemoteFTP.upload putbinaryfile [#{dest.name}]"
106
106
  @ftp.putbinaryfile source.path, dest.name, @chunk_size do |data|
107
107
  # Update job status after this block transfer
108
108
  yield data.bytesize, dest.name
@@ -110,7 +110,7 @@ module RestFtpDaemon
110
110
 
111
111
  # Move the file back to its original name
112
112
  if use_temp_name
113
- log_debug "Remote::RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
113
+ log_debug "RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
114
114
  @ftp.rename dest.name, target.name
115
115
  end
116
116
  end
@@ -13,19 +13,20 @@ module RestFtpDaemon
13
13
 
14
14
  def prepare
15
15
  @multipart_threshold = MULTIPART_THRESHOLD_MB.to_i * 1024 * 1024
16
- log_debug "Remote::RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
16
+ log_debug "RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
17
17
  end
18
18
 
19
19
  def connect
20
20
  super
21
21
 
22
22
  # Connect init
23
- log_debug "Remote::RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
23
+ log_debug "RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
24
24
 
25
25
  # Connect remote server
26
26
  @client = Aws::S3::Resource.new(
27
27
  region: @target.aws_region,
28
28
  credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
29
+ # thread_count: 4,
29
30
  http_wire_trace: @debug
30
31
  )
31
32
  #s3 = Aws::S3::Client.new(http_wire_trace: true)
@@ -34,7 +35,7 @@ module RestFtpDaemon
34
35
  def upload source, target, use_temp_name = false, &callback
35
36
  # Push init
36
37
  raise RestFtpDaemon::AssertionFailed, "upload/client" if @client.nil?
37
- log_debug "Remote::RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
38
+ log_debug "RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
38
39
 
39
40
  # Update progress before
40
41
  #yield 0, target.name
@@ -63,8 +64,8 @@ module RestFtpDaemon
63
64
  #yield target.size, target.name
64
65
 
65
66
  # Dump information about this file
66
- log_debug "Remote::RemoteS3.upload url[#{object.public_url}]"
67
- log_debug "Remote::RemoteS3.upload etag[#{object.etag}]"
67
+ log_debug "RemoteS3.upload url[#{object.public_url}]"
68
+ log_debug "RemoteS3.upload etag[#{object.etag}]"
68
69
  set_info :target_aws_public_url, object.public_url
69
70
  set_info :target_aws_etag, object.etag
70
71
  end
@@ -15,7 +15,7 @@ module RestFtpDaemon
15
15
  super
16
16
 
17
17
  # Connect init
18
- log_debug "Remote::RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
18
+ log_debug "RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
19
19
 
20
20
  # Debug level
21
21
  verbosity = @debug ? Logger::DEBUG : false
@@ -31,7 +31,7 @@ module RestFtpDaemon
31
31
  end
32
32
 
33
33
  def present? target
34
- log_debug "Remote::RemoteSFTP.present? [#{target.name}]"
34
+ log_debug "RemoteSFTP.present? [#{target.name}]"
35
35
  stat = @sftp.stat! target.path
36
36
 
37
37
  rescue Net::SFTP::StatusException
@@ -41,7 +41,7 @@ module RestFtpDaemon
41
41
  end
42
42
 
43
43
  def remove! target
44
- log_debug "Remote::RemoteSFTP.remove! [#{target.name}]"
44
+ log_debug "RemoteSFTP.remove! [#{target.name}]"
45
45
  @sftp.remove target.path
46
46
 
47
47
  rescue Net::SFTP::StatusException
@@ -51,7 +51,7 @@ module RestFtpDaemon
51
51
  end
52
52
 
53
53
  def mkdir directory
54
- log_debug "Remote::RemoteSFTP.mkdir [#{directory}]"
54
+ log_debug "RemoteSFTP.mkdir [#{directory}]"
55
55
  @sftp.mkdir! directory
56
56
 
57
57
  rescue StandardError => ex
@@ -60,7 +60,7 @@ module RestFtpDaemon
60
60
 
61
61
  def chdir_or_create directory, mkdir = false
62
62
  # Init, extract my parent name and my own name
63
- log_debug "Remote::RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
63
+ log_debug "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
64
64
  parent, _current = extract_parent(directory)
65
65
 
66
66
  # Access this directory
@@ -99,7 +99,7 @@ module RestFtpDaemon
99
99
  end
100
100
 
101
101
  # Do the transfer
102
- log_debug "Remote::RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
102
+ log_debug "RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
103
103
  @sftp.upload! source.path, dest.path do |event, _uploader, *args|
104
104
  case event
105
105
  when :open then
@@ -125,7 +125,7 @@ module RestFtpDaemon
125
125
  # Move the file back to its original name
126
126
  if use_temp_name
127
127
  flags = 0x00000001
128
- log_debug "Remote::RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
128
+ log_debug "RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
129
129
  @sftp.rename! dest.path, target.path, flags
130
130
  end
131
131
 
@@ -42,7 +42,7 @@
42
42
  %th.text-right{title: "updated every #{JOB_FTP_CHUNKMB} MB block"} rate
43
43
  %th{title: "Worker ID"} W
44
44
  %th{title: "Priority"} P
45
- %th{title: "Runs count"} R
45
+ %th{title: "Tentatives count"} T
46
46
 
47
47
  - @jobs_queued.each do |pool, jobs|
48
48
  - unless jobs.empty?
@@ -13,7 +13,6 @@
13
13
  - job_working = [JOB_STATUS_UPLOADING, JOB_STATUS_TRANSFORMING].include? job.status
14
14
 
15
15
  - trclass = JOB_STYLES[job.status]
16
- - runs = job.runs.to_i
17
16
 
18
17
  - unless job.error.nil?
19
18
  - trclass = "warning"
@@ -84,4 +83,4 @@
84
83
  .label.label-default.flag.worker-label= job.priority
85
84
 
86
85
  %td
87
- .label.flag.worker-label{class: job_runs_style(runs)}= runs
86
+ .label.flag.worker-label{class: job_tentatives_style(job.tentatives)}= job.tentatives
@@ -10,7 +10,7 @@ module RestFtpDaemon
10
10
  config_section :reporter
11
11
 
12
12
  # Other configuration options
13
- @report_newrelic = Conf.feature?(:newrelic)
13
+ @feature_newrelic = Conf.feature?(:newrelic)
14
14
 
15
15
  # Check that everything is OK
16
16
  return "invalid timer" unless @config[:timer].to_i > 0
@@ -34,9 +34,6 @@ module RestFtpDaemon
34
34
  private
35
35
 
36
36
  def do_metrics
37
- # What metrics to report?
38
- report_newrelic = Conf.feature?(:newrelic)
39
-
40
37
  # Get common metrics and dump them to logs
41
38
  log_debug "begin metrics sample"
42
39
  metrics = Metrics.sample
@@ -46,10 +43,10 @@ module RestFtpDaemon
46
43
  log_error "unable to collect metrics"
47
44
  return
48
45
  end
49
- log_info "collected metrics (newrelic: #{@report_newrelic})", metrics
46
+ log_info "collected metrics (newrelic: #{@feature_newrelic.inspect})", metrics
50
47
 
51
48
  # Transpose metrics to NewRelic metrics
52
- report_newrelic(metrics) if @report_newrelic
49
+ report_newrelic(metrics) if @feature_newrelic
53
50
  end
54
51
 
55
52
  def report_newrelic metrics
@@ -56,20 +56,17 @@ module RestFtpDaemon
56
56
  job.process
57
57
  end
58
58
 
59
- # Processing done
60
- worker_status WORKER_STATUS_FINISHED, job
61
-
62
59
  # Increment total processed jobs count
63
60
  RestFtpDaemon::Counters.instance.increment :jobs, :processed
64
61
 
65
62
  rescue RestFtpDaemon::JobTimeout => ex
66
- log_error "JOB TIMED OUT", ex.backtrace
67
- worker_status WORKER_STATUS_TIMEOUT
63
+ log_error "JOB TIMEOUT", ex.backtrace
64
+ worker_status WORKER_STATUS_TIMEOUT, job
68
65
 
69
66
  # Inform the job
70
67
  job.oops_you_stop_now ex unless job.nil?
71
68
 
72
- rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::AttributeMissing, StandardError => ex
69
+ rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::JobAttributeMissing, StandardError => ex
73
70
  log_error "JOB EXCEPTION ex[#{ex.class}] #{ex.message}", ex.backtrace
74
71
  worker_status WORKER_STATUS_CRASHED
75
72
 
@@ -80,30 +77,61 @@ module RestFtpDaemon
80
77
  def handle_job_result job
81
78
  # If job status requires a retry, just restack it
82
79
  if !job.error
83
- #log_info "job succeeded"
80
+ # Processing successful
81
+ log_error "job finished with no error"
82
+ worker_status WORKER_STATUS_FINISHED, job
84
83
 
85
- elsif !(@config[:retry_on].is_a?(Enumerable) && @config[:retry_on].include?(job.error))
86
- log_error "not retrying: error not eligible"
84
+ elsif error_not_eligible(job)
85
+ log_error "not retrying [#{job.error}] retry_on not eligible"
87
86
 
88
- elsif @config[:retry_for] && (job.age >= @config[:retry_for])
89
- log_error "not retrying: max_age reached (#{@config[:retry_for]} s)"
87
+ elsif error_reached_for(job)
88
+ log_error "not retrying [#{job.error}] retry_for reached [#{@config[:retry_for]}s]"
90
89
 
91
- elsif @config[:retry_max] && (job.runs >= @config[:retry_max])
92
- log_error "not retrying: max_runs reached (#{@config[:retry_max]} tries)"
90
+ elsif error_reached_max(job)
91
+ log_error "not retrying [#{job.error}] retry_max reached #{tentatives(job)}"
93
92
 
94
93
  else
95
94
  # Delay cannot be negative, and will be 1s minimum
96
95
  retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
97
- log_info "retrying job: waiting for #{retry_after} seconds"
96
+ log_info "retry job [#{job.id}] in [#{retry_after}s] tried #{tentatives(job)}"
98
97
 
99
98
  # Wait !
99
+ worker_status WORKER_STATUS_RETRYING, job
100
100
  sleep retry_after
101
- log_info "retrying job: requeued after delay"
101
+ log_debug "job [#{job.id}] requeued after [#{retry_after}s] delay"
102
102
 
103
103
  # Now, requeue this job
104
104
  RestFtpDaemon::JobQueue.instance.requeue job
105
105
  end
106
106
  end
107
107
 
108
+ def error_not_eligible job
109
+ # No, if no eligible errors
110
+ return true unless @config[:retry_on].is_a?(Enumerable)
111
+
112
+ # Tell if this error is in the list
113
+ return !@config[:retry_on].include?(job.error.to_s)
114
+ end
115
+
116
+ def error_reached_for job
117
+ # Not above, if no limit definded
118
+ return false unless @config[:retry_for]
119
+
120
+ # Job age above this limit
121
+ return job.age >= @config[:retry_for]
122
+ end
123
+
124
+ def error_reached_max job
125
+ # Not above, if no limit definded
126
+ return false unless @config[:retry_max]
127
+
128
+ # Job age above this limit
129
+ return job.tentatives >= @config[:retry_max]
130
+ end
131
+
132
+ def tentatives job
133
+ "[#{job.tentatives}/#{@config[:retry_max]}]"
134
+ end
135
+
108
136
  end
109
137
  end
@@ -2,7 +2,7 @@
2
2
  Gem::Specification.new do |spec|
3
3
 
4
4
  # Project version
5
- spec.version = "0.430.1"
5
+ spec.version = "0.432.0"
6
6
 
7
7
  # Project description
8
8
  spec.name = "rest-ftp-daemon"
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
20
20
  reject{ |f| f =~ /^dashboard.+\.png/ }
21
21
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
22
22
  spec.require_paths = ["lib"]
23
- spec.required_ruby_version = ">= 2.2.2"
23
+ spec.required_ruby_version = ">= 2.3"
24
24
 
25
25
  # Development dependencies
26
26
  spec.add_development_dependency "bundler", "~> 1.6"
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
31
31
  spec.add_development_dependency "http"
32
32
 
33
33
  # Runtime dependencies
34
- spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.10"
34
+ spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.12"
35
35
  spec.add_runtime_dependency "json", "~> 1.8"
36
36
  spec.add_runtime_dependency "thin", "~> 1.7"
37
37
  spec.add_runtime_dependency "activesupport", "~> 4.2"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rest-ftp-daemon
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.430.1
4
+ version: 0.432.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bruno MEDICI
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-09-08 00:00:00.000000000 Z
11
+ date: 2016-09-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -100,14 +100,14 @@ dependencies:
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: 0.3.10
103
+ version: 0.3.12
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: 0.3.10
110
+ version: 0.3.12
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: json
113
113
  requirement: !ruby/object:Gem::Requirement
@@ -524,7 +524,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
524
524
  requirements:
525
525
  - - ">="
526
526
  - !ruby/object:Gem::Version
527
- version: 2.2.2
527
+ version: '2.3'
528
528
  required_rubygems_version: !ruby/object:Gem::Requirement
529
529
  requirements:
530
530
  - - ">="
@@ -532,7 +532,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
532
532
  version: '0'
533
533
  requirements: []
534
534
  rubyforge_project:
535
- rubygems_version: 2.4.5
535
+ rubygems_version: 2.6.6
536
536
  signing_key:
537
537
  specification_version: 4
538
538
  summary: RESTful FTP client daemon