funktor 0.2.14 → 0.2.19

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 78d588a9932c62a7d4cb3d4e86adacf7f0eb99ac422447e54c5ce336fe77f6f2
4
- data.tar.gz: 1a4ef97fdbab5d9d38600e4a247141ffba81fe82ba3fda42de0d024dbcfc180d
3
+ metadata.gz: 215509407881bb7d177a5b1e917211f9e5adf3a1784109ffbd4fb63fcb9ec433
4
+ data.tar.gz: b4db3877393474e0f510e18b9460e2a1a96400379dc60ed23be546f6884e3c59
5
5
  SHA512:
6
- metadata.gz: dda1f909c912b01bd469b4b13398546abc9482ab3d60e6ec275f2002ba5161fa580e123d68732fd19bd148aaf97cfbd42112dc72c416fb745b9bfc14fab555dc
7
- data.tar.gz: acccfa11ddd50cde6b372bdeecfc34ce9fe0d0d0294cd5e99d905d6456373979ef9f03c67ed38ce3e54b2acdd6ebf8b6ecaa2d678d40c37ecfe6f153ca36d757
6
+ metadata.gz: ae3a977afab0505ead85b4d22c971cf51ed778a2a4964999990a5c2d74c2adc98d3a92dad19c3a4ea1e9dcf21e0e149d7da44e73ea6aea26c504687f1fa39b95
7
+ data.tar.gz: 86372b3097334bf619a3b23ad639c5e73b669b129899f9697b705010de37528ba0145dca7d15cee0d02183e8477a3e87ca38ceedb15027cb9a9e6c1f13b8b573
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- funktor (0.2.14)
4
+ funktor (0.2.19)
5
5
  activesupport
6
6
  aws-sdk-sqs (~> 1.37)
7
7
  thor
@@ -18,13 +18,13 @@ GEM
18
18
  addressable (2.7.0)
19
19
  public_suffix (>= 2.0.2, < 5.0)
20
20
  aws-eventstream (1.1.1)
21
- aws-partitions (1.472.0)
21
+ aws-partitions (1.473.0)
22
22
  aws-sdk-core (3.115.0)
23
23
  aws-eventstream (~> 1, >= 1.0.2)
24
24
  aws-partitions (~> 1, >= 1.239.0)
25
25
  aws-sigv4 (~> 1.1)
26
26
  jmespath (~> 1.0)
27
- aws-sdk-sqs (1.39.0)
27
+ aws-sdk-sqs (1.40.0)
28
28
  aws-sdk-core (~> 3, >= 3.112.0)
29
29
  aws-sigv4 (~> 1.1)
30
30
  aws-sigv4 (1.2.3)
data/lib/funktor.rb CHANGED
@@ -3,16 +3,21 @@ require 'funktor/aws/sqs/event'
3
3
  require 'funktor/aws/sqs/record'
4
4
  require 'funktor/counter'
5
5
  require 'funktor/job'
6
+ require 'funktor/logger'
6
7
  require 'funktor/worker'
7
8
  require 'funktor/middleware_chain'
8
9
  require 'funktor/incoming_job_handler'
9
- require 'funktor/active_job_handler'
10
10
 
11
11
  require 'json'
12
12
 
13
13
  module Funktor
14
14
  class Error < StandardError; end
15
15
 
16
+ DEFAULT_OPTIONS = {
17
+ error_handlers: [],
18
+ log_level: Logger::DEBUG # Set a high log level during early, active development
19
+ }
20
+
16
21
  def self.configure_job_pusher
17
22
  yield self
18
23
  end
@@ -55,9 +60,41 @@ module Funktor
55
60
  def self.dump_json(object)
56
61
  JSON.generate(object)
57
62
  end
63
+
64
+ def self.options
65
+ @options ||= DEFAULT_OPTIONS.dup
66
+ end
67
+
68
+ def self.options=(opts)
69
+ @options = opts
70
+ end
71
+
72
+ # Register a proc to handle any error which occurs within the Funktor active job handler.
73
+ #
74
+ # Funktor.error_handlers << proc {|error, context| ErrorsAsAService.notify(error, context) }
75
+ #
76
+ # The default error handler logs errors to STDOUT
77
+ def self.error_handlers
78
+ options[:error_handlers]
79
+ end
80
+
81
+ def self.logger
82
+ @logger ||= Funktor::Logger.new($stdout, level: options[:log_level])
83
+ end
84
+
85
+ def self.logger=(logger)
86
+ if logger.nil?
87
+ self.logger.level = Logger::FATAL
88
+ return self.logger
89
+ end
90
+
91
+ @logger = logger
92
+ end
58
93
  end
59
94
 
60
95
  # TODO - Should we require this by default or let people opt in?
61
- # Is it a code smell that we need to include it at the bottom, after
96
+ # TODO - Is it a code smell that we need to include these at the bottom, after
62
97
  # the main Funktor module is defined?
63
98
  require 'funktor/middleware/metrics'
99
+ require 'funktor/error_handler'
100
+ require 'funktor/active_job_handler'
@@ -2,6 +2,7 @@ require 'aws-sdk-sqs'
2
2
 
3
3
  module Funktor
4
4
  class ActiveJobHandler
5
+ include Funktor::ErrorHandler
5
6
 
6
7
  def initialize
7
8
  @failed_counter = Funktor::Counter.new('failed')
@@ -28,8 +29,7 @@ module Funktor
28
29
  @processed_counter.incr(job)
29
30
  # rescue Funktor::Job::InvalidJsonError # TODO Make this work
30
31
  rescue Exception => e
31
- puts "Error during processing: #{$!}"
32
- puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
32
+ handle_error(e, job)
33
33
  @failed_counter.incr(job)
34
34
  attempt_retry_or_bail(job)
35
35
  end
@@ -7,10 +7,126 @@ Resources:
7
7
  {
8
8
  "widgets": [
9
9
 
10
+ <% current_y = 0 %>
11
+
12
+ { <% "Funktor Banner" %>
13
+ "height": 3,
14
+ "width": 24,
15
+ "y": <%= current_y %>,
16
+ "x": 0,
17
+ "type": "text",
18
+ "properties": {
19
+ "markdown": "\n# Funktor Auto-generated Dashbaord\n\n This dashboard is auto-generated by Funktor and will be updated as Funktor progresses. If you want to customize this dashboard you should use 'Actions => Save dashbaord as' in the toolbar above to create a new dashboard of your own.\n\n The upper section shows you high level queue and worker stats, the lower sections show more details about all the AWS resources at play."
20
+ }
21
+ },
22
+ <% current_y += 3 %>
23
+
24
+ { <% "Job Duration By Worker" %>
25
+ "height": 6,
26
+ "width": 12,
27
+ "y": <%= current_y %>,
28
+ "x": 12,
29
+ "type": "metric",
30
+ "properties": {
31
+ "metrics": [
32
+ <%- app_worker_names.each do |worker_name| -%>
33
+ [ "<%= app_name %>", "Duration", "WorkerClassName", "<%= worker_name %>" ],
34
+ [ "...", { "stat": "p99" } ]<%= worker_name == app_worker_names.last ? "" : "," %>
35
+ <%- end -%>
36
+ ],
37
+ "view": "timeSeries",
38
+ "stacked": false,
39
+ "region": "us-east-1",
40
+ "stat": "Average",
41
+ "period": 60,
42
+ "title": "Job Duration by Worker"
43
+ }
44
+ },
45
+
46
+ { <% "Processed/Failed Jobs By Worker" %>
47
+ "height": 6,
48
+ "width": 12,
49
+ "y": <%= current_y %>,
50
+ "x": 0,
51
+ "type": "metric",
52
+ "properties": {
53
+ "metrics": [
54
+ <%- app_worker_names.each do |worker_name| -%>
55
+ [ "<%= app_name %>", "processed", "WorkerClassName", "<%= worker_name %>" ],
56
+ [ ".", "failed", ".", "." ]<%= worker_name == app_worker_names.last ? "" : "," %>
57
+ <%- end -%>
58
+ ],
59
+ "view": "timeSeries",
60
+ "stacked": false,
61
+ "region": "us-east-1",
62
+ "title": "Process/Failed Jobs By Worker",
63
+ "period": 60,
64
+ "stat": "Sum"
65
+ }
66
+ },
67
+ <% current_y += 6 %>
68
+ { <% "Job Duration By Queue" %>
69
+ "height": 6,
70
+ "width": 12,
71
+ "y": <%= current_y %>,
72
+ "x": 12,
73
+ "type": "metric",
74
+ "properties": {
75
+ "metrics": [
76
+ <%- queue_names.each do |queue_name| -%>
77
+ [ "<%= app_name %>", "Duration", "Queue", "<%= queue_name %>" ],
78
+ [ "...", { "stat": "p99" } ]<%= queue_name == queue_names.last ? "" : "," %>
79
+ <%- end -%>
80
+ ],
81
+ "view": "timeSeries",
82
+ "stacked": false,
83
+ "region": "us-east-1",
84
+ "stat": "Average",
85
+ "period": 60,
86
+ "title": "Job Duration by Queue"
87
+ }
88
+ },
89
+ { <% "Processed/Failed Jobs By Queue" %>
90
+ "height": 6,
91
+ "width": 12,
92
+ "y": <%= current_y %>,
93
+ "x": 0,
94
+ "type": "metric",
95
+ "properties": {
96
+ "metrics": [
97
+ <%- queue_names.each do |queue_name| -%>
98
+ [ "<%= app_name %>", "processed", "Queue", "<%= queue_name %>" ],
99
+ [ ".", "failed", ".", "." ]<%= queue_name == queue_names.last ? "" : "," %>
100
+ <%- end -%>
101
+ ],
102
+ "view": "timeSeries",
103
+ "stacked": false,
104
+ "region": "us-east-1",
105
+ "title": "Process/Failed Jobs By Queue",
106
+ "period": 60,
107
+ "stat": "Sum"
108
+ }
109
+ },
110
+
111
+
112
+ { <% "Funktor Behind the Scenes Banner" %>
113
+ "height": 3,
114
+ "width": 24,
115
+ "y": <%= current_y %>,
116
+ "x": 0,
117
+ "type": "text",
118
+ "properties": {
119
+ "markdown": "\n# Behind the scenes\n\n The stats below give some insight into the inner workings of the Funktor apparatus."
120
+ }
121
+ },
122
+ <% current_y += 3 %>
123
+
124
+
125
+ <% current_y += 6 %>
10
126
  { <% "Incoming Jobs" %>
11
127
  "height": 3,
12
128
  "width": 6,
13
- "y": 0,
129
+ "y": <%= current_y %>,
14
130
  "x": 0,
15
131
  "type": "text",
16
132
  "properties": {
@@ -20,7 +136,7 @@ Resources:
20
136
  { <% "Incoming Job Queue Messages per minute" %>
21
137
  "height": 3,
22
138
  "width": 3,
23
- "y": 0,
139
+ "y": <%= current_y %>,
24
140
  "x": 6,
25
141
  "type": "metric",
26
142
  "properties": {
@@ -37,7 +153,7 @@ Resources:
37
153
  { <% "Incoming Job Handler Duration" %>
38
154
  "height": 3,
39
155
  "width": 15,
40
- "y": 0,
156
+ "y": <%= current_y %>,
41
157
  "x": 9,
42
158
  "type": "metric",
43
159
  "properties": {
@@ -55,10 +171,12 @@ Resources:
55
171
  }
56
172
  },
57
173
 
174
+
175
+ <% current_y += 3 %>
58
176
  { <% "Incoming Job Queue (Async & scheduled jobs land here first)" %>
59
177
  "height": 6,
60
178
  "width": 9,
61
- "y": 3,
179
+ "y": <%= current_y %>,
62
180
  "x": 0,
63
181
  "type": "metric",
64
182
  "properties": {
@@ -84,7 +202,7 @@ Resources:
84
202
  { <% "Incoming Job Handler Duration in Milliseconds" %>
85
203
  "height": 6,
86
204
  "width": 9,
87
- "y": 3,
205
+ "y": <%= current_y %>,
88
206
  "x": 9,
89
207
  "type": "metric",
90
208
  "properties": {
@@ -104,7 +222,7 @@ Resources:
104
222
  { <% "Incoming Job Handler Error count and success rate (%)" %>
105
223
  "height": 3,
106
224
  "width": 6,
107
- "y": 3,
225
+ "y": <%= current_y %>,
108
226
  "x": 18,
109
227
  "type": "metric",
110
228
  "properties": {
@@ -131,7 +249,7 @@ Resources:
131
249
  { <% "Incoming Job Handler Concurrent Executions" %>
132
250
  "height": 3,
133
251
  "width": 6,
134
- "y": 6,
252
+ "y": <%= current_y + 3 %>,
135
253
  "x": 18,
136
254
  "type": "metric",
137
255
  "properties": {
@@ -149,7 +267,7 @@ Resources:
149
267
 
150
268
 
151
269
 
152
- <% current_y = 9 %>
270
+ <% current_y += 6 %>
153
271
  <%- queue_names.each do |queue_name| -%>
154
272
  { <% "Active Jobs" %>
155
273
  "height": 3,
@@ -300,51 +418,7 @@ Resources:
300
418
  <% current_y += 9 %>
301
419
  <%- end -%>
302
420
 
303
- { <% "Job Duration By Worker" %>
304
- "height": 6,
305
- "width": 24,
306
- "y": <%= current_y %>,
307
- "x": 0,
308
- "type": "metric",
309
- "properties": {
310
- "metrics": [
311
- <%- app_worker_names.each do |worker_name| -%>
312
- [ "<%= app_name %>", "Duration", "WorkerClassName", "<%= worker_name %>" ],
313
- [ "...", { "stat": "p99" } ]<%= worker_name == app_worker_names.last ? "" : "," %>
314
- <%- end -%>
315
- ],
316
- "view": "timeSeries",
317
- "stacked": false,
318
- "region": "us-east-1",
319
- "stat": "Average",
320
- "period": 60,
321
- "title": "Job Duration by Worker"
322
- }
323
- },
324
-
325
- <% current_y += 6 %>
326
- { <% "Processed/Failed Jobs By Worker" %>
327
- "height": 6,
328
- "width": 24,
329
- "y": <%= current_y %>,
330
- "x": 0,
331
- "type": "metric",
332
- "properties": {
333
- "metrics": [
334
- <%- app_worker_names.each do |worker_name| -%>
335
- [ "<%= app_name %>", "processed", "WorkerClassName", "<%= worker_name %>" ],
336
- [ ".", "failed", ".", "." ]<%= worker_name == app_worker_names.last ? "" : "," %>
337
- <%- end -%>
338
- ],
339
- "view": "timeSeries",
340
- "stacked": false,
341
- "region": "us-east-1",
342
- "title": "Process/Failed Jobs By Worker",
343
- "period": 60,
344
- "stat": "Sum"
345
- }
346
- },
347
- <% current_y += 6 %>
421
+
348
422
 
349
423
  { <% "Delayed Jobs" %>
350
424
  "height": 3,
@@ -21,7 +21,7 @@ module Funktor
21
21
  "CloudWatchMetrics": [
22
22
  {
23
23
  "Namespace": ENV['FUNKTOR_APP_NAME'],
24
- "Dimensions": [["WorkerClassName"]],
24
+ "Dimensions": [["WorkerClassName"], ["Queue"]],
25
25
  "Metrics": [ # CPU, Memory, Duration, etc...
26
26
  {
27
27
  "Name": dimension,
@@ -32,6 +32,7 @@ module Funktor
32
32
  ]
33
33
  },
34
34
  "WorkerClassName": job.worker_class_name,
35
+ "Queue": job.queue,
35
36
  "#{dimension}": 1
36
37
  #"count": value,
37
38
  #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
@@ -0,0 +1,25 @@
1
+ module Funktor
2
+ module ErrorHandler
3
+ class Logger
4
+ def call(error, context)
5
+ Funktor.logger.warn(Funktor.dump_json(context)) if context
6
+ Funktor.logger.warn("#{error.class.name}: #{error.message}")
7
+ Funktor.logger.warn(error.backtrace.join("\n")) unless error.backtrace.nil?
8
+ end
9
+
10
+ Funktor.error_handlers << Funktor::ErrorHandler::Logger.new
11
+ end
12
+
13
+ def handle_error(error, context = {})
14
+ Funktor.error_handlers.each do |handler|
15
+ begin
16
+ handler.call(error, context)
17
+ rescue => new_error
18
+ Funktor.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
19
+ Funktor.logger.error new_error
20
+ Funktor.logger.error new_error.backtrace.join("\n") unless new_error.backtrace.nil?
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,5 @@
1
+ require 'logger'
2
+ module Funktor
3
+ class Logger < ::Logger
4
+ end
5
+ end
@@ -20,7 +20,7 @@ module Funktor
20
20
  "CloudWatchMetrics": [
21
21
  {
22
22
  "Namespace": ENV['FUNKTOR_APP_NAME'],
23
- "Dimensions": [["WorkerClassName"]],
23
+ "Dimensions": [["WorkerClassName"], ["Queue"]],
24
24
  "Metrics": [ # CPU, Memory, Duration, etc...
25
25
  {
26
26
  "Name": "Duration",
@@ -31,6 +31,7 @@ module Funktor
31
31
  ]
32
32
  },
33
33
  "WorkerClassName": job.worker_class_name,
34
+ "Queue": job.queue,
34
35
  "Duration": time_diff_in_seconds * 1_000
35
36
  #"count": value,
36
37
  #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.2.14"
2
+ VERSION = "0.2.19"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.14
4
+ version: 0.2.19
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-07-01 00:00:00.000000000 Z
11
+ date: 2021-07-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs
@@ -137,9 +137,11 @@ files:
137
137
  - lib/funktor/cli/templates/package.json
138
138
  - lib/funktor/cli/templates/serverless.yml
139
139
  - lib/funktor/counter.rb
140
+ - lib/funktor/error_handler.rb
140
141
  - lib/funktor/fake_job_queue.rb
141
142
  - lib/funktor/incoming_job_handler.rb
142
143
  - lib/funktor/job.rb
144
+ - lib/funktor/logger.rb
143
145
  - lib/funktor/middleware/metrics.rb
144
146
  - lib/funktor/middleware_chain.rb
145
147
  - lib/funktor/testing.rb