rest-ftp-daemon 0.433.0 → 0.434.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9e668f8f43effe91b3332111a2963abb1011343c
4
- data.tar.gz: 79bc3e98795a91cdef80609096b15fc9606f4a6a
3
+ metadata.gz: 591f494a7bf660ee944096abcb623bc0d771157a
4
+ data.tar.gz: 34f150dc4b22573c0cbc948788adee9618f82599
5
5
  SHA512:
6
- metadata.gz: 0a06749631bb61dd15e9ef0556e8909fd6ca75ab32d944e70a854533a8caa9e50854fd13942b2fa11dda2dfea2f098dbf15c9fdd2e123c00971911f35f7ce396
7
- data.tar.gz: c6ae3a6914a80a1c932947f63be90d8d9a722134411839cdfa6d0694e8c3555ac86dac8b9afdbc923561489c0993186c61811d49b375a981f8fa4dbae0b8513a
6
+ metadata.gz: 087935f7b3179f7033f233e3344d98e26e6ad00e8e192722991409cc2d86f320857d7a87272d70812c6a34144e4f9b668b36b2a16f624fb2c00011478d7f914f
7
+ data.tar.gz: 58807f103079e56eb17b679b6d5d29552aea8d30d417c7b9b2dc0a46237b23bf1b4c9d3191a5a906a872ab36798ad698f6d1d2259f5031d15363eecc3ac3cd90
data/Gemfile.lock CHANGED
@@ -1,11 +1,11 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- rest-ftp-daemon (0.433.0)
4
+ rest-ftp-daemon (0.434.0)
5
5
  activesupport (~> 4.2)
6
6
  api-auth
7
7
  aws-sdk-resources (~> 2)
8
- bmc-daemon-lib (~> 0.3.12)
8
+ bmc-daemon-lib (~> 0.3.18)
9
9
  double-bag-ftps
10
10
  facter
11
11
  get_process_mem
@@ -37,17 +37,15 @@ GEM
37
37
  addressable (2.4.0)
38
38
  api-auth (2.0.1)
39
39
  ast (2.3.0)
40
- astrolabe (1.3.1)
41
- parser (~> 2.2)
42
- aws-sdk-core (2.5.11)
40
+ aws-sdk-core (2.6.1)
43
41
  jmespath (~> 1.0)
44
- aws-sdk-resources (2.5.11)
45
- aws-sdk-core (= 2.5.11)
42
+ aws-sdk-resources (2.6.1)
43
+ aws-sdk-core (= 2.6.1)
46
44
  axiom-types (0.1.1)
47
45
  descendants_tracker (~> 0.0.4)
48
46
  ice_nine (~> 0.11.0)
49
47
  thread_safe (~> 0.3, >= 0.3.1)
50
- bmc-daemon-lib (0.3.12)
48
+ bmc-daemon-lib (0.3.18)
51
49
  chamber (~> 2.9)
52
50
  builder (3.2.2)
53
51
  chamber (2.9.1)
@@ -95,7 +93,7 @@ GEM
95
93
  representable
96
94
  haml (4.0.7)
97
95
  tilt
98
- hashie (3.4.4)
96
+ hashie (3.4.6)
99
97
  http (2.0.3)
100
98
  addressable (~> 2.3)
101
99
  http-cookie (~> 1.0)
@@ -121,7 +119,7 @@ GEM
121
119
  net-ssh (3.2.0)
122
120
  netrc (0.11.0)
123
121
  newrelic_rpm (3.16.2.321)
124
- parser (2.3.1.2)
122
+ parser (2.3.1.3)
125
123
  ast (~> 2.2)
126
124
  powerpack (0.1.1)
127
125
  pry (0.10.4)
@@ -140,7 +138,7 @@ GEM
140
138
  http-cookie (>= 1.0.2, < 2.0)
141
139
  mime-types (>= 1.16, < 3.0)
142
140
  netrc (~> 0.7)
143
- rollbar (2.12.0)
141
+ rollbar (2.13.0)
144
142
  multi_json
145
143
  rspec (3.5.0)
146
144
  rspec-core (~> 3.5.0)
@@ -155,12 +153,13 @@ GEM
155
153
  diff-lcs (>= 1.2.0, < 2.0)
156
154
  rspec-support (~> 3.5.0)
157
155
  rspec-support (3.5.0)
158
- rubocop (0.32.1)
159
- astrolabe (~> 1.3)
160
- parser (>= 2.2.2.5, < 3.0)
156
+ rubocop (0.43.0)
157
+ parser (>= 2.3.1.1, < 3.0)
161
158
  powerpack (~> 0.1)
162
159
  rainbow (>= 1.99.1, < 3.0)
163
- ruby-progressbar (~> 1.4)
160
+ ruby-progressbar (~> 1.7)
161
+ unicode-display_width (~> 1.0, >= 1.0.1)
162
+ ruby-prof (0.16.2)
164
163
  ruby-progressbar (1.8.1)
165
164
  slop (3.6.0)
166
165
  streamio-ffmpeg (3.0.0)
@@ -180,6 +179,7 @@ GEM
180
179
  unf (0.1.4)
181
180
  unf_ext
182
181
  unf_ext (0.0.7.2)
182
+ unicode-display_width (1.1.1)
183
183
  virtus (1.0.5)
184
184
  axiom-types (~> 0.1)
185
185
  coercible (~> 1.0)
@@ -196,7 +196,8 @@ DEPENDENCIES
196
196
  rake
197
197
  rest-ftp-daemon!
198
198
  rspec
199
- rubocop (~> 0.32.0)
199
+ rubocop
200
+ ruby-prof
200
201
 
201
202
  BUNDLED WITH
202
203
  1.12.5
data/bin/rest-ftp-daemon CHANGED
@@ -91,7 +91,7 @@ end
91
91
 
92
92
 
93
93
  # Thin: basic options
94
- thin_logfile = Conf.logfile_path(:thin)
94
+ thin_logfile = Conf.logfile(:thin)
95
95
 
96
96
  argv = []
97
97
  argv << ["-e", Conf.app_env]
data/config.ru CHANGED
@@ -1,12 +1,12 @@
1
1
  # Load gem files
2
- Conf.log :init, "rackup: project code"
2
+ Conf.log :rackup, "load project code"
3
3
 
4
4
  load_path_libs = File.expand_path "lib", File.dirname(__FILE__)
5
5
  $LOAD_PATH.unshift(load_path_libs) unless $LOAD_PATH.include?(load_path_libs)
6
6
  require "rest-ftp-daemon"
7
7
 
8
8
  # Rack authent
9
- Conf.log :init, "rackup: rack setup"
9
+ Conf.log :rackup, "authentication, assets, env-specific"
10
10
  unless Conf[:adminpwd].nil?
11
11
  use Rack::Auth::Basic, "Restricted Area" do |username, password|
12
12
  [username, password] == ["admin", Conf[:adminpwd]]
@@ -23,12 +23,20 @@ use Rack::Static, root: "#{Conf.app_libs}/static/", urls: [
23
23
  MOUNT_SWAGGER_UI,
24
24
  ]
25
25
 
26
- # Rack reloader
27
- if Conf.app_env == "development"
28
- # use Rack::Reloader, 1
26
+ # Env-specific configuration
27
+ case Conf.app_env
28
+ when "development"
29
+ # Rack reloader
30
+ # use Rack::Reloader, 1
31
+
32
+ # Newrelic dev mode
33
+ require 'new_relic/rack/developer_mode'
34
+ use NewRelic::Rack::DeveloperMode
35
+
36
+ when "production"
29
37
  end
30
38
 
31
39
  # Launch the API
32
- Conf.log :init, "rackup: start endpoint"
40
+ Conf.log :rackup, "start API::Root"
33
41
  run RestFtpDaemon::API::Root
34
42
 
@@ -48,7 +48,7 @@ module RestFtpDaemon
48
48
 
49
49
  ## EXCEPTION HANDLERS
50
50
  rescue_from :all do |exception|
51
- Rollbar.error "#{exception.class.name}: #{exception.message}"
51
+ Rollbar.error exception, "api [#{error}]: #{exception.class.name}: #{exception.message}"
52
52
  #error!({error: :internal_server_error, message: exception.message}, 500)
53
53
  exception_error :api_unexpected_error, 500, exception
54
54
  end
@@ -30,16 +30,19 @@ module RestFtpDaemon
30
30
  # Status and error
31
31
  expose :status, format_with: :utf8_filter
32
32
  expose :error, format_with: :utf8_filter
33
+ expose :tentatives
33
34
 
34
35
  # Time stamps
35
- expose :queued_at
36
- expose :updated_at
36
+ expose :created_at
37
+ expose :created_since #, safe: true
38
+
37
39
  expose :started_at
40
+ expose :started_since
41
+
38
42
  expose :finished_at
43
+ expose :finished_in
39
44
 
40
- # Computed fields
41
- expose :age #, safe: true
42
- expose :exectime
45
+ expose :updated_at
43
46
 
44
47
  # Infos
45
48
  expose :infos, unless: :hide_infos
@@ -1,17 +1,15 @@
1
1
  # Init Rollbar and Newrelic
2
- Conf.log :init, "init: Newrelic and Rollbar"
3
2
  Conf.prepare_newrelic
4
3
  Conf.prepare_rollbar
5
4
 
6
5
  # Initialize Facter
7
- Conf.log :init, "init: Facter"
6
+ Conf.log :initialize, "prepare Facter"
8
7
  begin
9
8
  Facter.loadfacts
10
9
  rescue StandardError => exception
11
- Conf.log :init, "init: Facter failed to initialize: #{exception.message}"
10
+ Conf.log :initialize, "facter failed to initialize: #{exception.message}"
12
11
  end
13
12
 
14
13
  # Initialize workers
15
- Conf.log :init, "init: workers"
14
+ Conf.log :initialize, "prepare workers"
16
15
  RestFtpDaemon::WorkerPool.instance.start_em_all
17
-
@@ -28,11 +28,15 @@ module RestFtpDaemon
28
28
  attr_reader :status
29
29
  attr_reader :tentatives
30
30
 
31
- attr_reader :queued_at
31
+ attr_reader :created_at
32
32
  attr_reader :updated_at
33
33
  attr_reader :started_at
34
34
  attr_reader :finished_at
35
35
 
36
+ attr_reader :created_since
37
+ attr_reader :started_since
38
+ attr_reader :finished_in
39
+
36
40
  attr_reader :infos
37
41
 
38
42
  # Define readers from imported fields
@@ -50,14 +54,15 @@ module RestFtpDaemon
50
54
 
51
55
  # Init context
52
56
  @id = job_id.to_s
53
- @started_at = nil
54
- @finished_at = nil
55
57
  @updated_at = nil
56
58
  @error = nil
57
59
  @status = nil
58
- @tentatives = 0
60
+ @tentatives = 0
59
61
  @wid = nil
60
62
 
63
+ # Update timestamps
64
+ @created_at = Time.now
65
+
61
66
  # Prepare configuration
62
67
  @config = Conf[:transfer] || {}
63
68
  @endpoints = Conf[:endpoints] || {}
@@ -91,10 +96,9 @@ module RestFtpDaemon
91
96
 
92
97
  # Increment run cours
93
98
  @tentatives +=1
94
-
95
- # Flag current job timestamps
96
- @queued_at = Time.now
97
99
  @updated_at = Time.now
100
+ @started_at = nil
101
+ @finished_at = nil
98
102
 
99
103
  # Job has been prepared, reset infos
100
104
  set_status JOB_STATUS_PREPARED
@@ -108,40 +112,43 @@ module RestFtpDaemon
108
112
  end
109
113
 
110
114
  # Process job
111
- def process
115
+ def start
112
116
  # Check prerequisites
113
117
  raise RestFtpDaemon::AssertionFailed, "run/source_loc" unless @source_loc
114
118
  raise RestFtpDaemon::AssertionFailed, "run/target_loc" unless @target_loc
115
119
 
120
+ # Remember when we started
121
+ @started_at = Time.now
122
+
116
123
  # Notify we start working
117
- log_info "Job.process notify [started]"
124
+ log_info "Job.start notify [started]"
118
125
  current_signal = :started
119
126
  set_status JOB_STATUS_WORKING
120
127
  client_notify :started
121
128
 
122
129
  # Before work
123
- log_debug "Job.process do_before"
130
+ log_debug "Job.start do_before"
124
131
  current_signal = :started
125
132
  do_before
126
133
 
127
134
  # Do the hard work
128
- log_debug "Job.process do_work"
135
+ log_debug "Job.start do_work"
129
136
  current_signal = :ended
130
137
  do_work
131
138
 
132
139
  # Finalize all this
133
- log_debug "Job.process do_after"
140
+ log_debug "Job.start do_after"
134
141
  current_signal = :ended
135
142
  do_after
136
143
 
137
144
  rescue StandardError => exception
138
- Rollbar.error "Job.process: #{exception.class.name}: #{exception.message}"
145
+ Rollbar.error exception, "job [#{error}]: #{exception.class.name}: #{exception.message}"
139
146
  return oops current_signal, exception
140
147
 
141
148
  else
142
149
  # All done !
143
150
  set_status JOB_STATUS_FINISHED
144
- log_info "JobVideo.process notify [ended]"
151
+ log_info "Job.start notify [ended]"
145
152
  client_notify :ended
146
153
  end
147
154
 
@@ -164,28 +171,26 @@ module RestFtpDaemon
164
171
  @weight = [
165
172
  - @tentatives.to_i,
166
173
  + @priority.to_i,
167
- - @queued_at.to_i,
174
+ - @created_at.to_i,
168
175
  ]
169
176
  end
170
177
 
171
- def exectime
172
- return nil if @started_at.nil? || @finished_at.nil?
173
- (@finished_at - @started_at).round(2)
178
+ def started_since
179
+ since @started_at
174
180
  end
175
181
 
176
- def oops_after_crash exception
177
- Rollbar.error "oops_after_crash: #{exception.class.name}: #{exception.message}"
178
- oops :ended, exception, "crashed"
182
+ def created_since
183
+ since @created_at
179
184
  end
180
185
 
181
- def oops_you_stop_now exception
182
- Rollbar.error "oops_you_stop_now: #{exception.class.name}: #{exception.message}"
183
- oops :ended, exception, "timeout"
186
+ def finished_in
187
+ return nil if @started_at.nil? || @finished_at.nil?
188
+ (@finished_at - @started_at).round(2)
184
189
  end
185
190
 
186
- def age
187
- return nil if @queued_at.nil?
188
- (Time.now - @queued_at).round(2)
191
+ def oops_end what, exception
192
+ Rollbar.error exception, "oops_end [#{what}]: #{exception.class.name}: #{exception.message}"
193
+ oops :ended, exception, what
189
194
  end
190
195
 
191
196
  def targethost
@@ -225,6 +230,12 @@ module RestFtpDaemon
225
230
  Thread.current.thread_variable_set :updated_at, now
226
231
  end
227
232
 
233
+ # Timestamps calculation
234
+ def since timestamp
235
+ return nil if timestamp.nil?
236
+ return (Time.now - timestamp).round(2)
237
+ end
238
+
228
239
  # Force strings to UTF8
229
240
  def debug_value_utf8 value
230
241
  case value
@@ -298,7 +309,7 @@ module RestFtpDaemon
298
309
  end
299
310
 
300
311
  # Log to Rollbar
301
- Rollbar.error(exception)
312
+ Rollbar.warning e, "oops [#{error}]: #{exception.class.name}: #{exception.message}"
302
313
 
303
314
  # Close ftp connexion if open
304
315
  @remote.close unless @remote.nil? || !@remote.connected?
@@ -32,7 +32,7 @@
32
32
  %th{title: "Job identifier"} ID
33
33
  %th{title: "Job type"} T
34
34
  %th pool
35
- %th queued
35
+ %th created
36
36
  %th label
37
37
  %th source
38
38
  %th target
@@ -27,7 +27,9 @@
27
27
 
28
28
  %td= job.pool
29
29
 
30
- %td= datetime_short(job.queued_at)
30
+ %td
31
+ %nobr
32
+ = datetime_short(job.created_at)
31
33
 
32
34
  %td= job.label
33
35
 
@@ -69,7 +71,7 @@
69
71
  %td.nobr.text-right
70
72
  = format_bytes(transfer_total, "B")
71
73
 
72
- %td.nobr.text-right{title: "time: #{job.exectime} s"}
74
+ %td.nobr.text-right{title: "started since: #{job.started_since} s"}
73
75
  - if transfer_bitrate
74
76
  %span.push-bitrate
75
77
  = format_bytes(transfer_bitrate, "bps")
@@ -94,7 +94,10 @@ module RestFtpDaemon
94
94
  log_info "spawning #{klass.name} wid[#{wid}] pool[#{pool}]"
95
95
  @workers[wid] = Thread.new do
96
96
  begin
97
+ # Create a worker inside
97
98
  worker = klass.new wid, pool
99
+
100
+ # FIXME: sleep for a small amount of time to allow correct sequence of logging
98
101
  sleep 0.1
99
102
  rescue StandardError => ex
100
103
  log_error "#{klass.name} EXCEPTION: #{ex.message}"
@@ -60,7 +60,8 @@ module RestFtpDaemon
60
60
  end
61
61
 
62
62
  # Don't dump metrics unless we're debugging
63
- msg_newrelic = "reported metrics to NewRelic [#{ENV['NEW_RELIC_APP_NAME']}]"
63
+ newrelic_app_name = Conf.at(:newrelic, :app_name)
64
+ msg_newrelic = "reported metrics to NewRelic [#{newrelic_app_name}]"
64
65
  if @config[:debug]
65
66
  log_debug msg_newrelic, metrics_newrelic
66
67
  else
@@ -53,25 +53,25 @@ module RestFtpDaemon
53
53
 
54
54
  # Processs this job protected by a timeout
55
55
  Timeout.timeout(@config[:timeout], RestFtpDaemon::JobTimeout) do
56
- job.process
56
+ job.start
57
57
  end
58
58
 
59
59
  # Increment total processed jobs count
60
60
  RestFtpDaemon::Counters.instance.increment :jobs, :processed
61
61
 
62
62
  rescue RestFtpDaemon::JobTimeout => ex
63
- log_error "JOB TIMEOUT", ex.backtrace
63
+ log_error "JOB TIMEOUT started_at[#{job.started_at}] started_since[#{job.started_since}] #{ex.message}", ex.backtrace
64
64
  worker_status WORKER_STATUS_TIMEOUT, job
65
65
 
66
66
  # Inform the job
67
- job.oops_you_stop_now ex unless job.nil?
67
+ job.oops_end(:timeout, ex) unless job.nil?
68
68
 
69
69
  rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::JobAttributeMissing, StandardError => ex
70
- log_error "JOB EXCEPTION ex[#{ex.class}] #{ex.message}", ex.backtrace
70
+ log_error "JOB CRASHED ex[#{ex.class}] #{ex.message}", ex.backtrace
71
71
  worker_status WORKER_STATUS_CRASHED
72
72
 
73
73
  # Inform the job
74
- job.oops_after_crash ex unless job.nil?
74
+ job.oops_end(:crashed, ex) unless job.nil?
75
75
  end
76
76
 
77
77
  def handle_job_result job
@@ -118,7 +118,7 @@ module RestFtpDaemon
118
118
  return false unless @config[:retry_for]
119
119
 
120
120
  # Job age above this limit
121
- return job.age >= @config[:retry_for]
121
+ return job.created_since >= @config[:retry_for]
122
122
  end
123
123
 
124
124
  def error_reached_max job
@@ -2,7 +2,7 @@
2
2
  Gem::Specification.new do |spec|
3
3
 
4
4
  # Project version
5
- spec.version = "0.433.0"
5
+ spec.version = "0.434.0"
6
6
 
7
7
  # Project description
8
8
  spec.name = "rest-ftp-daemon"
@@ -26,12 +26,13 @@ Gem::Specification.new do |spec|
26
26
  spec.add_development_dependency "bundler", "~> 1.6"
27
27
  spec.add_development_dependency "rake"
28
28
  spec.add_development_dependency "rspec"
29
- spec.add_development_dependency "rubocop", "~> 0.32.0"
29
+ spec.add_development_dependency "rubocop"
30
30
  spec.add_development_dependency "pry"
31
31
  spec.add_development_dependency "http"
32
+ spec.add_development_dependency "ruby-prof"
32
33
 
33
34
  # Runtime dependencies
34
- spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.12"
35
+ spec.add_runtime_dependency "bmc-daemon-lib", "~> 0.3.18"
35
36
  spec.add_runtime_dependency "json", "~> 1.8"
36
37
  spec.add_runtime_dependency "thin", "~> 1.7"
37
38
  spec.add_runtime_dependency "activesupport", "~> 4.2"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rest-ftp-daemon
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.433.0
4
+ version: 0.434.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bruno MEDICI
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-09-12 00:00:00.000000000 Z
11
+ date: 2016-09-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -56,16 +56,16 @@ dependencies:
56
56
  name: rubocop
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "~>"
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
- version: 0.32.0
61
+ version: '0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - "~>"
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
- version: 0.32.0
68
+ version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: pry
71
71
  requirement: !ruby/object:Gem::Requirement
@@ -94,20 +94,34 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: ruby-prof
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: bmc-daemon-lib
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
115
  - - "~>"
102
116
  - !ruby/object:Gem::Version
103
- version: 0.3.12
117
+ version: 0.3.18
104
118
  type: :runtime
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
122
  - - "~>"
109
123
  - !ruby/object:Gem::Version
110
- version: 0.3.12
124
+ version: 0.3.18
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: json
113
127
  requirement: !ruby/object:Gem::Requirement