funktor 0.2.13 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -3
- data/lib/funktor.rb +39 -2
- data/lib/funktor/active_job_handler.rb +2 -2
- data/lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml +130 -30
- data/lib/funktor/counter.rb +2 -1
- data/lib/funktor/error_handler.rb +25 -0
- data/lib/funktor/logger.rb +5 -0
- data/lib/funktor/middleware/metrics.rb +2 -1
- data/lib/funktor/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6e146325cc89da410fb6af4397911f20d66d681eaf14c7bd2919d41c97ab6e62
|
|
4
|
+
data.tar.gz: 4da3a83da0c561ed6ca5e51c84d0fa608063f5fa3d70dedde9a57aed279b91a2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4860f967c03a2cf365c211f5d511591a8778c8570492499603bb6e888fb85e7c3ee7def2906fdc09762c9f166b3e1a503571df5fe4175a1cfeb2b153b37a5ba4
|
|
7
|
+
data.tar.gz: 1f32d8f5d968a08c7c0bd36eba892309b8511414bfc0d9ff2cc082dbd38496f8bdfb259ac7046e3d6f06707f994b7a947648eb1a15996c1f481afe112dad9034
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
funktor (0.2.
|
|
4
|
+
funktor (0.2.18)
|
|
5
5
|
activesupport
|
|
6
6
|
aws-sdk-sqs (~> 1.37)
|
|
7
7
|
thor
|
|
@@ -18,13 +18,13 @@ GEM
|
|
|
18
18
|
addressable (2.7.0)
|
|
19
19
|
public_suffix (>= 2.0.2, < 5.0)
|
|
20
20
|
aws-eventstream (1.1.1)
|
|
21
|
-
aws-partitions (1.
|
|
21
|
+
aws-partitions (1.473.0)
|
|
22
22
|
aws-sdk-core (3.115.0)
|
|
23
23
|
aws-eventstream (~> 1, >= 1.0.2)
|
|
24
24
|
aws-partitions (~> 1, >= 1.239.0)
|
|
25
25
|
aws-sigv4 (~> 1.1)
|
|
26
26
|
jmespath (~> 1.0)
|
|
27
|
-
aws-sdk-sqs (1.
|
|
27
|
+
aws-sdk-sqs (1.40.0)
|
|
28
28
|
aws-sdk-core (~> 3, >= 3.112.0)
|
|
29
29
|
aws-sigv4 (~> 1.1)
|
|
30
30
|
aws-sigv4 (1.2.3)
|
data/lib/funktor.rb
CHANGED
|
@@ -3,16 +3,21 @@ require 'funktor/aws/sqs/event'
|
|
|
3
3
|
require 'funktor/aws/sqs/record'
|
|
4
4
|
require 'funktor/counter'
|
|
5
5
|
require 'funktor/job'
|
|
6
|
+
require 'funktor/logger'
|
|
6
7
|
require 'funktor/worker'
|
|
7
8
|
require 'funktor/middleware_chain'
|
|
8
9
|
require 'funktor/incoming_job_handler'
|
|
9
|
-
require 'funktor/active_job_handler'
|
|
10
10
|
|
|
11
11
|
require 'json'
|
|
12
12
|
|
|
13
13
|
module Funktor
|
|
14
14
|
class Error < StandardError; end
|
|
15
15
|
|
|
16
|
+
DEFAULT_OPTIONS = {
|
|
17
|
+
error_handlers: [],
|
|
18
|
+
log_level: Logger::DEBUG # Set a high log level during early, active development
|
|
19
|
+
}
|
|
20
|
+
|
|
16
21
|
def self.configure_job_pusher
|
|
17
22
|
yield self
|
|
18
23
|
end
|
|
@@ -55,9 +60,41 @@ module Funktor
|
|
|
55
60
|
def self.dump_json(object)
|
|
56
61
|
JSON.generate(object)
|
|
57
62
|
end
|
|
63
|
+
|
|
64
|
+
def self.options
|
|
65
|
+
@options ||= DEFAULT_OPTIONS.dup
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def self.options=(opts)
|
|
69
|
+
@options = opts
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Register a proc to handle any error which occurs within the Funktor active job handler.
|
|
73
|
+
#
|
|
74
|
+
# Funktor.error_handlers << proc {|error, context| ErrorsAsAService.notify(error, context) }
|
|
75
|
+
#
|
|
76
|
+
# The default error handler logs errors to STDOUT
|
|
77
|
+
def self.error_handlers
|
|
78
|
+
options[:error_handlers]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def self.logger
|
|
82
|
+
@logger ||= Funktor::Logger.new($stdout, level: options[:log_level])
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def self.logger=(logger)
|
|
86
|
+
if logger.nil?
|
|
87
|
+
self.logger.level = Logger::FATAL
|
|
88
|
+
return self.logger
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
@logger = logger
|
|
92
|
+
end
|
|
58
93
|
end
|
|
59
94
|
|
|
60
95
|
# TODO - Should we require this by default or let people opt in?
|
|
61
|
-
# Is it a code smell that we need to include
|
|
96
|
+
# TODO - Is it a code smell that we need to include these at the bottom, after
|
|
62
97
|
# the main Funktor module is defined?
|
|
63
98
|
require 'funktor/middleware/metrics'
|
|
99
|
+
require 'funktor/error_handler'
|
|
100
|
+
require 'funktor/active_job_handler'
|
|
@@ -2,6 +2,7 @@ require 'aws-sdk-sqs'
|
|
|
2
2
|
|
|
3
3
|
module Funktor
|
|
4
4
|
class ActiveJobHandler
|
|
5
|
+
include Funktor::ErrorHandler
|
|
5
6
|
|
|
6
7
|
def initialize
|
|
7
8
|
@failed_counter = Funktor::Counter.new('failed')
|
|
@@ -28,8 +29,7 @@ module Funktor
|
|
|
28
29
|
@processed_counter.incr(job)
|
|
29
30
|
# rescue Funktor::Job::InvalidJsonError # TODO Make this work
|
|
30
31
|
rescue Exception => e
|
|
31
|
-
|
|
32
|
-
puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
|
|
32
|
+
handle_error(e, job)
|
|
33
33
|
@failed_counter.incr(job)
|
|
34
34
|
attempt_retry_or_bail(job)
|
|
35
35
|
end
|
|
@@ -7,10 +7,126 @@ Resources:
|
|
|
7
7
|
{
|
|
8
8
|
"widgets": [
|
|
9
9
|
|
|
10
|
+
<% current_y = 0 %>
|
|
11
|
+
|
|
12
|
+
{ <% "Funktor Banner" %>
|
|
13
|
+
"height": 3,
|
|
14
|
+
"width": 24,
|
|
15
|
+
"y": <%= current_y %>,
|
|
16
|
+
"x": 0,
|
|
17
|
+
"type": "text",
|
|
18
|
+
"properties": {
|
|
19
|
+
"markdown": "\n# Funktor Auto-generated Dashbaord\n\n This dashboard is auto-generated by Funktor and will be updated as Funktor progresses. If you want to customize this dashboard you should use 'Actions => Save dashbaord as' in the toolbar above to create a new dashboard of your own.\n\n The upper section shows you high level queue and worker stats, the lower sections show more details about all the AWS resources at play."
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
<% current_y += 3 %>
|
|
23
|
+
|
|
24
|
+
{ <% "Job Duration By Worker" %>
|
|
25
|
+
"height": 6,
|
|
26
|
+
"width": 12,
|
|
27
|
+
"y": <%= current_y %>,
|
|
28
|
+
"x": 12,
|
|
29
|
+
"type": "metric",
|
|
30
|
+
"properties": {
|
|
31
|
+
"metrics": [
|
|
32
|
+
<%- app_worker_names.each do |worker_name| -%>
|
|
33
|
+
[ "<%= app_name %>", "Duration", "WorkerClassName", "<%= worker_name %>" ],
|
|
34
|
+
[ "...", { "stat": "p99" } ]<%= worker_name == app_worker_names.last ? "" : "," %>
|
|
35
|
+
<%- end -%>
|
|
36
|
+
],
|
|
37
|
+
"view": "timeSeries",
|
|
38
|
+
"stacked": false,
|
|
39
|
+
"region": "us-east-1",
|
|
40
|
+
"stat": "Average",
|
|
41
|
+
"period": 60,
|
|
42
|
+
"title": "Job Duration by Worker"
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
|
|
46
|
+
{ <% "Processed/Failed Jobs By Worker" %>
|
|
47
|
+
"height": 6,
|
|
48
|
+
"width": 12,
|
|
49
|
+
"y": <%= current_y %>,
|
|
50
|
+
"x": 0,
|
|
51
|
+
"type": "metric",
|
|
52
|
+
"properties": {
|
|
53
|
+
"metrics": [
|
|
54
|
+
<%- app_worker_names.each do |worker_name| -%>
|
|
55
|
+
[ "<%= app_name %>", "processed", "WorkerClassName", "<%= worker_name %>" ],
|
|
56
|
+
[ ".", "failed", ".", "." ]<%= worker_name == app_worker_names.last ? "" : "," %>
|
|
57
|
+
<%- end -%>
|
|
58
|
+
],
|
|
59
|
+
"view": "timeSeries",
|
|
60
|
+
"stacked": false,
|
|
61
|
+
"region": "us-east-1",
|
|
62
|
+
"title": "Process/Failed Jobs By Worker",
|
|
63
|
+
"period": 60,
|
|
64
|
+
"stat": "Sum"
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
<% current_y += 6 %>
|
|
68
|
+
{ <% "Job Duration By Queue" %>
|
|
69
|
+
"height": 6,
|
|
70
|
+
"width": 12,
|
|
71
|
+
"y": <%= current_y %>,
|
|
72
|
+
"x": 12,
|
|
73
|
+
"type": "metric",
|
|
74
|
+
"properties": {
|
|
75
|
+
"metrics": [
|
|
76
|
+
<%- queue_names.each do |queue_name| -%>
|
|
77
|
+
[ "<%= app_name %>", "Duration", "Queue", "<%= queue_name %>" ],
|
|
78
|
+
[ "...", { "stat": "p99" } ]<%= queue_name == queue_names.last ? "" : "," %>
|
|
79
|
+
<%- end -%>
|
|
80
|
+
],
|
|
81
|
+
"view": "timeSeries",
|
|
82
|
+
"stacked": false,
|
|
83
|
+
"region": "us-east-1",
|
|
84
|
+
"stat": "Average",
|
|
85
|
+
"period": 60,
|
|
86
|
+
"title": "Job Duration by Queue"
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
{ <% "Processed/Failed Jobs By Queue" %>
|
|
90
|
+
"height": 6,
|
|
91
|
+
"width": 12,
|
|
92
|
+
"y": <%= current_y %>,
|
|
93
|
+
"x": 0,
|
|
94
|
+
"type": "metric",
|
|
95
|
+
"properties": {
|
|
96
|
+
"metrics": [
|
|
97
|
+
<%- queue_names.each do |queue_name| -%>
|
|
98
|
+
[ "<%= app_name %>", "processed", "Queue", "<%= queue_name %>" ],
|
|
99
|
+
[ ".", "failed", ".", "." ]<%= queue_name == queue_names.last ? "" : "," %>
|
|
100
|
+
<%- end -%>
|
|
101
|
+
],
|
|
102
|
+
"view": "timeSeries",
|
|
103
|
+
"stacked": false,
|
|
104
|
+
"region": "us-east-1",
|
|
105
|
+
"title": "Process/Failed Jobs By Queue",
|
|
106
|
+
"period": 60,
|
|
107
|
+
"stat": "Sum"
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
{ <% "Funktor Behind the Scenes Banner" %>
|
|
113
|
+
"height": 3,
|
|
114
|
+
"width": 24,
|
|
115
|
+
"y": <%= current_y %>,
|
|
116
|
+
"x": 0,
|
|
117
|
+
"type": "text",
|
|
118
|
+
"properties": {
|
|
119
|
+
"markdown": "\n# Behind the scenes\n\n The stats below give some insight into the inner workings of the Funktor apparatus."
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
<% current_y += 3 %>
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
<% current_y += 6 %>
|
|
10
126
|
{ <% "Incoming Jobs" %>
|
|
11
127
|
"height": 3,
|
|
12
128
|
"width": 6,
|
|
13
|
-
"y":
|
|
129
|
+
"y": <%= current_y %>,
|
|
14
130
|
"x": 0,
|
|
15
131
|
"type": "text",
|
|
16
132
|
"properties": {
|
|
@@ -20,7 +136,7 @@ Resources:
|
|
|
20
136
|
{ <% "Incoming Job Queue Messages per minute" %>
|
|
21
137
|
"height": 3,
|
|
22
138
|
"width": 3,
|
|
23
|
-
"y":
|
|
139
|
+
"y": <%= current_y %>,
|
|
24
140
|
"x": 6,
|
|
25
141
|
"type": "metric",
|
|
26
142
|
"properties": {
|
|
@@ -37,7 +153,7 @@ Resources:
|
|
|
37
153
|
{ <% "Incoming Job Handler Duration" %>
|
|
38
154
|
"height": 3,
|
|
39
155
|
"width": 15,
|
|
40
|
-
"y":
|
|
156
|
+
"y": <%= current_y %>,
|
|
41
157
|
"x": 9,
|
|
42
158
|
"type": "metric",
|
|
43
159
|
"properties": {
|
|
@@ -55,10 +171,12 @@ Resources:
|
|
|
55
171
|
}
|
|
56
172
|
},
|
|
57
173
|
|
|
174
|
+
|
|
175
|
+
<% current_y += 3 %>
|
|
58
176
|
{ <% "Incoming Job Queue (Async & scheduled jobs land here first)" %>
|
|
59
177
|
"height": 6,
|
|
60
178
|
"width": 9,
|
|
61
|
-
"y":
|
|
179
|
+
"y": <%= current_y %>,
|
|
62
180
|
"x": 0,
|
|
63
181
|
"type": "metric",
|
|
64
182
|
"properties": {
|
|
@@ -84,7 +202,7 @@ Resources:
|
|
|
84
202
|
{ <% "Incoming Job Handler Duration in Milliseconds" %>
|
|
85
203
|
"height": 6,
|
|
86
204
|
"width": 9,
|
|
87
|
-
"y":
|
|
205
|
+
"y": <%= current_y %>,
|
|
88
206
|
"x": 9,
|
|
89
207
|
"type": "metric",
|
|
90
208
|
"properties": {
|
|
@@ -104,7 +222,7 @@ Resources:
|
|
|
104
222
|
{ <% "Incoming Job Handler Error count and success rate (%)" %>
|
|
105
223
|
"height": 3,
|
|
106
224
|
"width": 6,
|
|
107
|
-
"y":
|
|
225
|
+
"y": <%= current_y %>,
|
|
108
226
|
"x": 18,
|
|
109
227
|
"type": "metric",
|
|
110
228
|
"properties": {
|
|
@@ -131,7 +249,7 @@ Resources:
|
|
|
131
249
|
{ <% "Incoming Job Handler Concurrent Executions" %>
|
|
132
250
|
"height": 3,
|
|
133
251
|
"width": 6,
|
|
134
|
-
"y":
|
|
252
|
+
"y": <%= current_y + 3 %>,
|
|
135
253
|
"x": 18,
|
|
136
254
|
"type": "metric",
|
|
137
255
|
"properties": {
|
|
@@ -149,7 +267,7 @@ Resources:
|
|
|
149
267
|
|
|
150
268
|
|
|
151
269
|
|
|
152
|
-
<% current_y
|
|
270
|
+
<% current_y += 6 %>
|
|
153
271
|
<%- queue_names.each do |queue_name| -%>
|
|
154
272
|
{ <% "Active Jobs" %>
|
|
155
273
|
"height": 3,
|
|
@@ -300,28 +418,7 @@ Resources:
|
|
|
300
418
|
<% current_y += 9 %>
|
|
301
419
|
<%- end -%>
|
|
302
420
|
|
|
303
|
-
|
|
304
|
-
"height": 6,
|
|
305
|
-
"width": 24,
|
|
306
|
-
"y": <%= current_y %>,
|
|
307
|
-
"x": 0,
|
|
308
|
-
"type": "metric",
|
|
309
|
-
"properties": {
|
|
310
|
-
"metrics": [
|
|
311
|
-
<%- app_worker_names.each do |worker_name| -%>
|
|
312
|
-
[ "<%= app_name %>", "Duration", "WorkerClassName", "<%= worker_name %>" ],
|
|
313
|
-
[ "...", { "stat": "p99" } ]<%= worker_name == app_worker_names.last ? "" : "," %>
|
|
314
|
-
<%- end -%>
|
|
315
|
-
],
|
|
316
|
-
"view": "timeSeries",
|
|
317
|
-
"stacked": false,
|
|
318
|
-
"region": "us-east-1",
|
|
319
|
-
"stat": "Average",
|
|
320
|
-
"period": 60,
|
|
321
|
-
"title": "Job Duration by Worker"
|
|
322
|
-
}
|
|
323
|
-
},
|
|
324
|
-
<% current_y += 6 %>
|
|
421
|
+
|
|
325
422
|
|
|
326
423
|
{ <% "Delayed Jobs" %>
|
|
327
424
|
"height": 3,
|
|
@@ -551,5 +648,8 @@ Resources:
|
|
|
551
648
|
}
|
|
552
649
|
}
|
|
553
650
|
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
|
|
554
654
|
]
|
|
555
655
|
}
|
data/lib/funktor/counter.rb
CHANGED
|
@@ -21,7 +21,7 @@ module Funktor
|
|
|
21
21
|
"CloudWatchMetrics": [
|
|
22
22
|
{
|
|
23
23
|
"Namespace": ENV['FUNKTOR_APP_NAME'],
|
|
24
|
-
"Dimensions": [["WorkerClassName"]],
|
|
24
|
+
"Dimensions": [["WorkerClassName"], ["Queue"]],
|
|
25
25
|
"Metrics": [ # CPU, Memory, Duration, etc...
|
|
26
26
|
{
|
|
27
27
|
"Name": dimension,
|
|
@@ -32,6 +32,7 @@ module Funktor
|
|
|
32
32
|
]
|
|
33
33
|
},
|
|
34
34
|
"WorkerClassName": job.worker_class_name,
|
|
35
|
+
"Queue": job.queue,
|
|
35
36
|
"#{dimension}": 1
|
|
36
37
|
#"count": value,
|
|
37
38
|
#"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
module Funktor
|
|
2
|
+
module ErrorHandler
|
|
3
|
+
class Logger
|
|
4
|
+
def call(error, context)
|
|
5
|
+
Funktor.logger.warn(Funktor.dump_json(context)) unless context.empty?
|
|
6
|
+
Funktor.logger.warn("#{error.class.name}: #{error.message}")
|
|
7
|
+
Funktor.logger.warn(error.backtrace.join("\n")) unless error.backtrace.nil?
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
Funktor.error_handlers << Funktor::ErrorHandler::Logger.new
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def handle_error(error, context = {})
|
|
14
|
+
Funktor.error_handlers.each do |handler|
|
|
15
|
+
begin
|
|
16
|
+
handler.call(error, context)
|
|
17
|
+
rescue => new_error
|
|
18
|
+
Funktor.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
|
19
|
+
Funktor.logger.error new_error
|
|
20
|
+
Funktor.logger.error new_error.backtrace.join("\n") unless new_error.backtrace.nil?
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -20,7 +20,7 @@ module Funktor
|
|
|
20
20
|
"CloudWatchMetrics": [
|
|
21
21
|
{
|
|
22
22
|
"Namespace": ENV['FUNKTOR_APP_NAME'],
|
|
23
|
-
"Dimensions": [["WorkerClassName"]],
|
|
23
|
+
"Dimensions": [["WorkerClassName"], ["Queue"]],
|
|
24
24
|
"Metrics": [ # CPU, Memory, Duration, etc...
|
|
25
25
|
{
|
|
26
26
|
"Name": "Duration",
|
|
@@ -31,6 +31,7 @@ module Funktor
|
|
|
31
31
|
]
|
|
32
32
|
},
|
|
33
33
|
"WorkerClassName": job.worker_class_name,
|
|
34
|
+
"Queue": job.queue,
|
|
34
35
|
"Duration": time_diff_in_seconds * 1_000
|
|
35
36
|
#"count": value,
|
|
36
37
|
#"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
|
data/lib/funktor/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: funktor
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.18
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jeremy Green
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-07-05 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: aws-sdk-sqs
|
|
@@ -137,9 +137,11 @@ files:
|
|
|
137
137
|
- lib/funktor/cli/templates/package.json
|
|
138
138
|
- lib/funktor/cli/templates/serverless.yml
|
|
139
139
|
- lib/funktor/counter.rb
|
|
140
|
+
- lib/funktor/error_handler.rb
|
|
140
141
|
- lib/funktor/fake_job_queue.rb
|
|
141
142
|
- lib/funktor/incoming_job_handler.rb
|
|
142
143
|
- lib/funktor/job.rb
|
|
144
|
+
- lib/funktor/logger.rb
|
|
143
145
|
- lib/funktor/middleware/metrics.rb
|
|
144
146
|
- lib/funktor/middleware_chain.rb
|
|
145
147
|
- lib/funktor/testing.rb
|