funktor 0.7.6 → 0.7.9

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,5 +1,4 @@
1
1
  require 'json'
2
- require 'aws-sdk-dynamodb'
3
2
 
4
3
  module Funktor
5
4
  class ActivityTracker
@@ -74,7 +73,7 @@ module Funktor
74
73
  end
75
74
 
76
75
  def dynamodb_client
77
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
76
+ Funktor.dynamodb_client
78
77
  end
79
78
 
80
79
  def metric_namespace
@@ -144,7 +144,7 @@ module Funktor
144
144
  def app_worker_names
145
145
  app_worker_files.map do |file|
146
146
  File.basename(file, ".rb").camelize
147
- end
147
+ end.sort
148
148
  end
149
149
 
150
150
  def app_worker_files
@@ -21,6 +21,282 @@ Resources:
21
21
  },
22
22
  <% current_y += 3 %>
23
23
 
24
+
25
+ <%- app_worker_names.each do |worker_name| -%>
26
+ { <% "Worker" %>
27
+ "height": 1,
28
+ "width": 24,
29
+ "y": <%= current_y %>,
30
+ "x": 0,
31
+ "type": "text",
32
+ "properties": {
33
+ "markdown": "\n# <%= worker_name %>\n"
34
+ }
35
+ },
36
+ <% current_y += 1 %>
37
+
38
+ { <% "Worker Jobs per minute" %>
39
+ "height": 2,
40
+ "width": 3,
41
+ "y": <%= current_y %>,
42
+ "x": 0,
43
+ "type": "metric",
44
+ "properties": {
45
+ "metrics": [
46
+ [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ]
47
+ ],
48
+ "view": "singleValue",
49
+ "region": "us-east-1",
50
+ "stat": "Sum",
51
+ "period": 60,
52
+ "sparkline": true,
53
+ "title": "<%= worker_name %> Processed/Min"
54
+ }
55
+ },
56
+ { <% "Worker Jobs per minute" %>
57
+ "height": 2,
58
+ "width": 3,
59
+ "y": <%= current_y %>,
60
+ "x": 3,
61
+ "type": "metric",
62
+ "properties": {
63
+ "metrics": [
64
+ [ "${self:custom.funktor.DashboardNamespace}", "failed", "WorkerClassName", "<%= worker_name %>", { "color": "#d62728" } ]
65
+ ],
66
+ "view": "singleValue",
67
+ "region": "us-east-1",
68
+ "stat": "Sum",
69
+ "period": 60,
70
+ "sparkline": true,
71
+ "title": "<%= worker_name %> Failed/Min"
72
+ }
73
+ },
74
+ { <% "Worker Duration Percentiles" %>
75
+ "height": 2,
76
+ "width": 3,
77
+ "y": <%= current_y %>,
78
+ "x": 6,
79
+ "type": "metric",
80
+ "properties": {
81
+ "metrics": [
82
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "p60", "color": "#1f77b4" } ]
83
+ ],
84
+ "view": "singleValue",
85
+ "region": "us-east-1",
86
+ "stat": "p60",
87
+ "period": 60,
88
+ "sparkline": true,
89
+ "title": "<%= worker_name %> p60"
90
+ }
91
+ },
92
+ { <% "Worker Duration Percentiles" %>
93
+ "height": 2,
94
+ "width": 3,
95
+ "y": <%= current_y %>,
96
+ "x": 9,
97
+ "type": "metric",
98
+ "properties": {
99
+ "metrics": [
100
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "p70", "color": "#ff7f0e" } ]
101
+ ],
102
+ "view": "singleValue",
103
+ "region": "us-east-1",
104
+ "stat": "p70",
105
+ "period": 60,
106
+ "sparkline": true,
107
+ "title": "<%= worker_name %> p70"
108
+ }
109
+ },
110
+ { <% "Worker Duration Percentiles" %>
111
+ "height": 2,
112
+ "width": 3,
113
+ "y": <%= current_y %>,
114
+ "x": 12,
115
+ "type": "metric",
116
+ "properties": {
117
+ "metrics": [
118
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "p80", "color": "#2ca02c" } ]
119
+ ],
120
+ "view": "singleValue",
121
+ "region": "us-east-1",
122
+ "stat": "p80",
123
+ "period": 60,
124
+ "sparkline": true,
125
+ "title": "<%= worker_name %> p80"
126
+ }
127
+ },
128
+ { <% "Worker Duration Percentiles" %>
129
+ "height": 2,
130
+ "width": 3,
131
+ "y": <%= current_y %>,
132
+ "x": 15,
133
+ "type": "metric",
134
+ "properties": {
135
+ "metrics": [
136
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "p90", "color": "#d62728" } ]
137
+ ],
138
+ "view": "singleValue",
139
+ "region": "us-east-1",
140
+ "stat": "p90",
141
+ "period": 60,
142
+ "sparkline": true,
143
+ "title": "<%= worker_name %> p90"
144
+ }
145
+ },
146
+ { <% "Worker Min/Max" %>
147
+ "height": 2,
148
+ "width": 3,
149
+ "y": <%= current_y %>,
150
+ "x": 18,
151
+ "type": "metric",
152
+ "properties": {
153
+ "metrics": [
154
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "Min" } ]
155
+ ],
156
+ "view": "singleValue",
157
+ "region": "us-east-1",
158
+ "stat": "Minimum",
159
+ "period": 60,
160
+ "sparkline": true,
161
+ "title": "<%= worker_name %> Min"
162
+ }
163
+ },
164
+ { <% "Worker Min/Max" %>
165
+ "height": 2,
166
+ "width": 3,
167
+ "y": <%= current_y %>,
168
+ "x": 21,
169
+ "type": "metric",
170
+ "properties": {
171
+ "metrics": [
172
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "Max", "color": "#d62728" } ]
173
+ ],
174
+ "view": "singleValue",
175
+ "region": "us-east-1",
176
+ "stat": "Maximum",
177
+ "period": 60,
178
+ "sparkline": true,
179
+ "title": "<%= worker_name %> Max"
180
+ }
181
+ },
182
+ <% current_y += 3 %>
183
+
184
+
185
+
186
+
187
+ { <% "Processed Jobs By Worker" %>
188
+ "height": 3,
189
+ "width": 3,
190
+ "y": <%= current_y %>,
191
+ "x": 0,
192
+ "type": "metric",
193
+ "properties": {
194
+ "metrics": [
195
+ [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ]
196
+ ],
197
+ "view": "timeSeries",
198
+ "stacked": false,
199
+ "region": "us-east-1",
200
+ "title": "<%= worker_name %> Processed Jobs",
201
+ "period": 60,
202
+ "stat": "Sum",
203
+ "legend": {
204
+ "position": "hidden"
205
+ }
206
+ }
207
+ },
208
+ { <% "Failed Jobs By Worker" %>
209
+ "height": 3,
210
+ "width": 3,
211
+ "y": <%= current_y %>,
212
+ "x": 3,
213
+ "type": "metric",
214
+ "properties": {
215
+ "metrics": [
216
+ [ "${self:custom.funktor.DashboardNamespace}", "failed", "WorkerClassName", "<%= worker_name %>", { "color": "#d62728" }]
217
+ ],
218
+ "view": "timeSeries",
219
+ "stacked": false,
220
+ "region": "us-east-1",
221
+ "title": "<%= worker_name %> Failed Jobs",
222
+ "period": 60,
223
+ "stat": "Sum",
224
+ "legend": {
225
+ "position": "hidden"
226
+ }
227
+ }
228
+ },
229
+ { <% "Job Duration By Worker" %>
230
+ "height": 3,
231
+ "width": 12,
232
+ "y": <%= current_y %>,
233
+ "x": 6,
234
+ "type": "metric",
235
+ "properties": {
236
+ "metrics": [
237
+ [ "funktor-testapp-dev", "Duration", "WorkerClassName", "AuditWorker", { "stat": "p90", "color": "#d62728" } ],
238
+ [ "...", { "stat": "p80", "color": "#2ca02c" } ],
239
+ [ "...", { "stat": "p70", "color": "#ff7f0e" } ],
240
+ [ "...", { "color": "#1f77b4" } ]
241
+ ],
242
+ "view": "timeSeries",
243
+ "stacked": false,
244
+ "region": "us-east-1",
245
+ "stat": "p60",
246
+ "period": 60,
247
+ "title": "<%= worker_name %> Duration Percentiles",
248
+ "legend": {
249
+ "position": "right"
250
+ }
251
+ }
252
+ },
253
+ { <% "Minimum By Worker" %>
254
+ "height": 3,
255
+ "width": 3,
256
+ "y": <%= current_y %>,
257
+ "x": 18,
258
+ "type": "metric",
259
+ "properties": {
260
+ "metrics": [
261
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ]
262
+ ],
263
+ "view": "timeSeries",
264
+ "stacked": false,
265
+ "region": "us-east-1",
266
+ "stat": "Minimum",
267
+ "period": 60,
268
+ "title": "<%= worker_name %> Min",
269
+ "legend": {
270
+ "position": "hidden"
271
+ }
272
+ }
273
+ },
274
+ { <% "Maximum By Worker" %>
275
+ "height": 3,
276
+ "width": 3,
277
+ "y": <%= current_y %>,
278
+ "x": 21,
279
+ "type": "metric",
280
+ "properties": {
281
+ "metrics": [
282
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "color": "#d62728" } ]
283
+ ],
284
+ "view": "timeSeries",
285
+ "stacked": false,
286
+ "region": "us-east-1",
287
+ "stat": "Maximum",
288
+ "period": 60,
289
+ "title": "<%= worker_name %> Max",
290
+ "legend": {
291
+ "position": "hidden"
292
+ }
293
+ }
294
+ },
295
+
296
+ <% current_y += 6 %>
297
+ <%- end -%>
298
+
299
+
24
300
  { <% "Job Duration By Worker" %>
25
301
  "height": 6,
26
302
  "width": 12,
@@ -65,6 +341,7 @@ Resources:
65
341
  }
66
342
  },
67
343
  <% current_y += 6 %>
344
+
68
345
  { <% "Job Duration By Queue" %>
69
346
  "height": 6,
70
347
  "width": 12,
@@ -147,6 +424,7 @@ Resources:
147
424
  "region": "us-east-1",
148
425
  "stat": "Sum",
149
426
  "period": 60,
427
+ "sparkline": true,
150
428
  "title": "Messages Per Minute"
151
429
  }
152
430
  },
@@ -167,6 +445,7 @@ Resources:
167
445
  "region": "us-east-1",
168
446
  "stat": "p10",
169
447
  "period": 60,
448
+ "sparkline": true,
170
449
  "title": "Handler Duration"
171
450
  }
172
451
  },
@@ -294,6 +573,7 @@ Resources:
294
573
  "region": "us-east-1",
295
574
  "stat": "Sum",
296
575
  "period": 60,
576
+ "sparkline": true,
297
577
  "title": "Messages Per Minute"
298
578
  }
299
579
  },
@@ -316,6 +596,7 @@ Resources:
316
596
  "region": "us-east-1",
317
597
  "stat": "p10",
318
598
  "period": 60,
599
+ "sparkline": true,
319
600
  "title": "Handler Duration"
320
601
  }
321
602
  },
@@ -444,6 +725,7 @@ Resources:
444
725
  "region": "us-east-1",
445
726
  "stat": "Average",
446
727
  "period": 60,
728
+ "sparkline": true,
447
729
  "title": "Messages To Be Scheduled"
448
730
  }
449
731
  },
@@ -464,6 +746,7 @@ Resources:
464
746
  "region": "us-east-1",
465
747
  "stat": "p10",
466
748
  "period": 60,
749
+ "sparkline": true,
467
750
  "title": "Handler Duration"
468
751
  }
469
752
  },
@@ -18,11 +18,11 @@ module Funktor
18
18
  end
19
19
 
20
20
  def sqs_client
21
- @sqs_client ||= ::Aws::SQS::Client.new
21
+ Funktor.sqs_client
22
22
  end
23
23
 
24
24
  def dynamodb_client
25
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
25
+ Funktor.dynamodb_client
26
26
  end
27
27
 
28
28
  def dispatch(job)
@@ -9,11 +9,11 @@ module Funktor
9
9
  end
10
10
 
11
11
  def dynamodb_client
12
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
12
+ Funktor.dynamodb_client
13
13
  end
14
14
 
15
15
  def sqs_client
16
- @sqs_client ||= ::Aws::SQS::Client.new
16
+ Funktor.sqs_client
17
17
  end
18
18
 
19
19
  def delayed_job_table
@@ -61,7 +61,6 @@ module Funktor
61
61
  end
62
62
 
63
63
  def activate_job(job_shard, job_id, current_category, queue_immediately = false)
64
-
65
64
  # TODO: WorkQueueVisibilityMiddleware to alter what happens here? Maybe we delete by default and then the middleware puts it back in the table?
66
65
  # First we conditionally update the item in Dynamo to be sure that another scheduler hasn't gotten
67
66
  # to it, and if that works then send to SQS. This is basically how Sidekiq scheduler works.
@@ -19,7 +19,7 @@ module Funktor
19
19
  private
20
20
 
21
21
  def client
22
- @client ||= ::Aws::SQS::Client.new
22
+ Funktor.sqs_client
23
23
  end
24
24
 
25
25
  def queue_url(payload)
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.7.6"
2
+ VERSION = "0.7.9"
3
3
  end
@@ -128,7 +128,7 @@ module Funktor
128
128
  end
129
129
 
130
130
  def dynamodb_client
131
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
131
+ Funktor.dynamodb_client
132
132
  end
133
133
 
134
134
  # start the server if ruby file executed directly
@@ -20,11 +20,11 @@ module Funktor
20
20
  end
21
21
 
22
22
  def dynamodb_client
23
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
23
+ Funktor.dynamodb_client
24
24
  end
25
25
 
26
26
  def sqs_client
27
- @sqs_client ||= ::Aws::SQS::Client.new
27
+ Funktor.sqs_client
28
28
  end
29
29
 
30
30
  def dispatch(job)
data/lib/funktor.rb CHANGED
@@ -12,6 +12,8 @@ require_relative 'funktor/job_activator'
12
12
  require_relative 'funktor/activity_tracker'
13
13
 
14
14
  require 'json'
15
+ require 'aws-sdk-dynamodb'
16
+ require 'aws-sdk-sqs'
15
17
 
16
18
  module Funktor
17
19
  class Error < StandardError; end
@@ -135,6 +137,14 @@ module Funktor
135
137
 
136
138
  @raw_logger = raw_logger
137
139
  end
140
+
141
+ def self.dynamodb_client
142
+ @dynamodb_client ||= ::Aws::DynamoDB::Client.new
143
+ end
144
+
145
+ def self.sqs_client
146
+ @sqs_client ||= ::Aws::SQS::Client.new
147
+ end
138
148
  end
139
149
 
140
150
  # TODO - Is it a code smell that we need to include these at the bottom, after
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.6
4
+ version: 0.7.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-05-12 00:00:00.000000000 Z
11
+ date: 2022-05-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs