funktor 0.2.2 → 0.2.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 38d383dcdafdb71a93c3ab881ec7772e8694fe0a8ca913a11a27ab634b068349
4
- data.tar.gz: 92e17fab7ca35700c1721dbd9f674d14a1dd8a033486ed1af9d65236cad305cc
3
+ metadata.gz: 700b659a76af0a0e7e6a21c91a91553f4d5ff5899140ef18313bd181d97de4e1
4
+ data.tar.gz: f9d311402f6a0686e11c0d30131135fc5372bf58cc8e6c32c0f5a90f649861d7
5
5
  SHA512:
6
- metadata.gz: 1643f95d5f02f83272c2304e6c22cda5c91bb39f4f109cd4d363d9b667294679dbfa4634962162193dd9c2ea90da57f21048a41d2b5710a80f19c91631136f8d
7
- data.tar.gz: 559c5693bf8a958b22e0b6b189f72ca43d758701498ccb85c52c3cf3a9e7c2b148d26d17efdea72089328e393c1ad671b3d03b6f8dcee1ac39498fc6cf6f7535
6
+ metadata.gz: 47e9682f6d0f96e9ce5d393574642b8361049c91f07dc2b9154c4492c0b10257bc651e22ea5512a1f904cd2a895c9435d9722c559b756086618f1ea58983298a
7
+ data.tar.gz: a498a2854d47e3c2b0bfd9f1bb2d2feb31b5bd3b8ac1f34c79a5efd127cb81d77862147cd6d27528589710676b748bd744bdd3ace35ebfdb118d4983a3414ac0
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- funktor (0.2.2)
4
+ funktor (0.2.3)
5
5
  activesupport
6
6
  aws-sdk-sqs (~> 1.37)
7
7
  thor
@@ -18,8 +18,8 @@ GEM
18
18
  addressable (2.7.0)
19
19
  public_suffix (>= 2.0.2, < 5.0)
20
20
  aws-eventstream (1.1.1)
21
- aws-partitions (1.465.0)
22
- aws-sdk-core (3.114.1)
21
+ aws-partitions (1.468.0)
22
+ aws-sdk-core (3.114.3)
23
23
  aws-eventstream (~> 1, >= 1.0.2)
24
24
  aws-partitions (~> 1, >= 1.239.0)
25
25
  aws-sigv4 (~> 1.1)
@@ -29,7 +29,7 @@ GEM
29
29
  aws-sigv4 (~> 1.1)
30
30
  aws-sigv4 (1.2.3)
31
31
  aws-eventstream (~> 1, >= 1.0.2)
32
- concurrent-ruby (1.1.8)
32
+ concurrent-ruby (1.1.9)
33
33
  crack (0.4.5)
34
34
  rexml
35
35
  diff-lcs (1.4.4)
@@ -15,7 +15,7 @@ module Funktor
15
15
 
16
16
  class_option :file, :aliases => "-f",
17
17
  :type => :string, :desc => "The funktor init file.",
18
- :default => "funktor.yml"
18
+ :default => "funktor_init.yml"
19
19
 
20
20
  desc <<~DESC
21
21
  Description:
@@ -127,6 +127,29 @@ module Funktor
127
127
  @work_queue_config
128
128
  end
129
129
 
130
+ def queue_config(queue_name)
131
+ funktor_config["queues"].each do |queue_details|
132
+ if queue_details.keys.first == queue_name
133
+ return queue_details.values.first
134
+ end
135
+ end
136
+ return nil
137
+ end
138
+
139
+ def incoming_config_value(config_name)
140
+ funktor_config.dig("incomingJobHandler", config_name) ||
141
+ funktor_config.dig("handlerDefaults", config_name) ||
142
+ "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
143
+ end
144
+
145
+ def queue_config_value(queue_name, config_name)
146
+ puts funktor_config
147
+ queue_config(queue_name)&.dig(config_name) ||
148
+ funktor_config.dig("handlerDefaults", config_name) ||
149
+ "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
150
+
151
+ end
152
+
130
153
  end
131
154
  end
132
155
  end
@@ -3,7 +3,5 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
3
3
 
4
4
  ruby '~> 2.7'
5
5
 
6
- source "https://gem.fury.io/jagthedrummer/" do
7
- gem 'funktor'
8
- end
6
+ gem 'funktor'
9
7
 
@@ -3,19 +3,23 @@ incomingJobHandler:
3
3
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
4
4
  # expect to spend for one message. The incoming job handler usually will be pretty fast,
5
5
  # but we default to a high number here to allow for the times when things go weird.
6
- timeout: 300
6
+ timeout: <%= incoming_config_value 'timeout' %>
7
7
  # reservedConcurrency represents the maximum number of concurrent executions.
8
8
  # For the incoming job handler you probably don't want to limit it because you
9
9
  # want to get things onto work queues as quickly as possible.
10
- reservedConcurrency: null
10
+ reservedConcurrency: <%= incoming_config_value 'reservedConcurrency' %>
11
11
  # provisionedConcurrency represents the number of lambda functions that will always
12
12
  # be available. For the incoming jobs handler you probably don't need to set this
13
13
  # unless your jobs are very bursty AND very time sensitive.
14
- provisionedConcurrency: null
14
+ provisionedConcurrency: <%= incoming_config_value 'provisionedConcurrency' %>
15
15
  # Use memory_size to adjust the reousrces (both memory and CPU) available.
16
16
  # For the incoming jobs handler you probably don't need this to be too large,
17
17
  # but if you're seeing large delays at this stage it might help to bump it up.
18
- memorySize: 256
18
+ memorySize: <%= incoming_config_value 'memorySize' %>
19
+ batchSize: <%= incoming_config_value 'batchSize' %>
20
+ maximumBatchingWindow: <%= incoming_config_value 'maximumBatchingWindow' %>
21
+ visibilityTimeout: <%= incoming_config_value 'visibilityTimeout' %>
22
+ logRetentionInDays: <%= incoming_config_value 'logRetentionInDays' %>
19
23
 
20
24
  <%- queue_names.each do |queue_name| -%>
21
25
  <%= queue_name.camelize %>QueueHandler:
@@ -23,23 +27,26 @@ incomingJobHandler:
23
27
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
24
28
  # expect to spend for one message. The active job handler may be slow if your jobs are
25
29
  # doing a lot of work, so we default to the maximum here.
26
- timeout: 300 # TODO pull this (and other values) from funktor_init.yml
30
+ timeout: <%= queue_config_value queue_name, 'timeout' %>
27
31
  # reservedConcurrency represents the maximum number of concurrent executions.
28
32
  # For the active job handler you may want to limit it if you have resource limitations
29
33
  # like database connections that you need to avoid exhausting.
30
- reservedConcurrency: null
34
+ reservedConcurrency: <%= queue_config_value queue_name, 'reservedConcurrency' %>
31
35
  # provisionedConcurrency represents the number of lambda functions that will always
32
36
  # be available. For the active job handler you probably don't need to set this
33
37
  # unless your jobs are very bursty AND very time sensitive.
34
- provisionedConcurrency: null
38
+ provisionedConcurrency: <%= queue_config_value queue_name, 'provisionedConcurrency' %>
35
39
  # Use memory_size to adjust the reousrces (both memory and CPU) available.
36
40
  # For the active jobs handler you'll want this to be at least as large as the memory
37
41
  # required to actually do your jobs. You can choose an even higher number to increase
38
42
  # the available CPU to make the jobs run faster.
39
- memorySize: 256
40
- <%- end -%>
41
-
43
+ memorySize: <%= queue_config_value queue_name, 'memorySize' %>
44
+ batchSize: <%= queue_config_value queue_name, 'batchSize' %>
45
+ maximumBatchingWindow: <%= queue_config_value queue_name, 'maximumBatchingWindow' %>
46
+ visibilityTimeout: <%= queue_config_value queue_name, 'visibilityTimeout' %>
47
+ logRetentionInDays: <%= queue_config_value queue_name, 'logRetentionInDays' %>
42
48
 
49
+ <%- end -%>
43
50
 
44
51
 
45
52
  # You shouldn't need to mess with these under most circumstances. But you could if you want to change
@@ -51,6 +58,7 @@ dashboardName: ${self:service}-${self:custom.stage}-dashboard
51
58
  <%- queue_names.each do |queue_name| -%>
52
59
  <%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
53
60
  <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
61
+ <%= queue_name.camelize %>QueueHandlerName: ${self:service}-${self:custom.stage}-<%= queue_name.camelize %>QueueHandler
54
62
  <%- end -%>
55
63
 
56
64
 
@@ -6,123 +6,82 @@ Resources:
6
6
  DashboardBody: >
7
7
  {
8
8
  "widgets": [
9
- {
10
- "height": 6,
11
- "width": 9,
12
- "y": 3,
9
+
10
+ { <% "Incoming Jobs" %>
11
+ "height": 3,
12
+ "width": 6,
13
+ "y": 0,
13
14
  "x": 0,
14
- "type": "metric",
15
+ "type": "text",
15
16
  "properties": {
16
- "metrics": [
17
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
18
- [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
19
- [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
20
- [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
21
- [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
22
- [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
23
- [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
24
- ],
25
- "view": "timeSeries",
26
- "stacked": false,
27
- "region": "us-east-1",
28
- "title": "Incoming Job Queue (Async & scheduled jobs land here first)",
29
- "period": 60,
30
- "stat": "Sum",
31
- "setPeriodToTimeRange": true,
32
- "liveData": true
17
+ "markdown": "\n# Incoming Jobs\n"
33
18
  }
34
19
  },
35
- {
36
- "height": 6,
37
- "width": 9,
38
- "y": 12,
39
- "x": 0,
20
+ { <% "Incoming Job Queue Messages per minute" %>
21
+ "height": 3,
22
+ "width": 3,
23
+ "y": 0,
24
+ "x": 6,
40
25
  "type": "metric",
41
26
  "properties": {
42
27
  "metrics": [
43
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.DefaultQueueName}", { "label": "Received" } ],
44
- [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
45
- [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-DefaultQueueHandler", "Resource", "${self:service}-${self:provider.stage}-DefaultQueueHandler", { "label": "Handler Invocations" } ],
46
- [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.DefaultQueueName}", { "label": "Pending?" } ],
47
- [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
48
- [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
49
- [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
28
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
50
29
  ],
51
- "view": "timeSeries",
52
- "stacked": false,
30
+ "view": "singleValue",
53
31
  "region": "us-east-1",
54
- "title": "Active Job Queue (Async jobs go here immediately, scheduled jobs land here in the minute before they're scheduled)",
55
- "period": 60,
56
32
  "stat": "Sum",
57
- "liveData": true
33
+ "period": 60,
34
+ "title": "Messages Per Minute"
58
35
  }
59
36
  },
60
- {
37
+ { <% "Incoming Job Handler Duration" %>
61
38
  "height": 3,
62
- "width": 9,
63
- "y": 30,
64
- "x": 0,
39
+ "width": 15,
40
+ "y": 0,
41
+ "x": 9,
65
42
  "type": "metric",
66
43
  "properties": {
67
44
  "metrics": [
68
- [ { "expression": "m2/PERIOD(m2)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
69
- [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
70
- [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
71
- [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
45
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
46
+ [ "...", { "label": "p50", "stat": "p50" } ],
47
+ [ "...", { "label": "p99", "stat": "p99" } ],
48
+ [ "...", { "label": "Average", "stat": "Average" } ]
72
49
  ],
73
- "view": "timeSeries",
74
- "stacked": false,
50
+ "view": "singleValue",
75
51
  "region": "us-east-1",
76
- "title": "Scheduled Job Table Read Capacity Units",
52
+ "stat": "p10",
77
53
  "period": 60,
78
- "stat": "Sum",
79
- "liveData": true
54
+ "title": "Handler Duration"
80
55
  }
81
56
  },
82
- {
83
- "height": 3,
84
- "width": 18,
85
- "y": 33,
57
+
58
+ { <% "Incoming Job Queue (Async & scheduled jobs land here first)" %>
59
+ "height": 6,
60
+ "width": 9,
61
+ "y": 3,
86
62
  "x": 0,
87
63
  "type": "metric",
88
64
  "properties": {
89
65
  "metrics": [
90
- [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "PutItem", { "yAxis": "left" } ],
91
- [ "...", "Query" ],
92
- [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
93
- [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
94
- [ ".", "ThrottledRequests", ".", ".", ".", ".", { "yAxis": "right", "visible": false } ]
66
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
67
+ [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
68
+ [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
69
+ [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
70
+ [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
71
+ [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
72
+ [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
95
73
  ],
96
74
  "view": "timeSeries",
97
75
  "stacked": false,
98
76
  "region": "us-east-1",
99
- "stat": "Average",
77
+ "title": "Incoming Job Queue (Async & scheduled jobs land here first)",
100
78
  "period": 60,
101
- "title": "Scheduled Job Table Latency",
102
- "liveData": true
103
- }
104
- },
105
- {
106
- "height": 3,
107
- "width": 6,
108
- "y": 33,
109
- "x": 18,
110
- "type": "metric",
111
- "properties": {
112
- "metrics": [
113
- [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "DeleteItem" ],
114
- [ "...", "PutItem" ]
115
- ],
116
- "view": "timeSeries",
117
- "stacked": false,
118
- "region": "us-east-1",
119
79
  "stat": "Sum",
120
- "period": 60,
121
- "title": "Scheduled Job Table Throttled Operations",
80
+ "setPeriodToTimeRange": true,
122
81
  "liveData": true
123
82
  }
124
83
  },
125
- {
84
+ { <% "Incoming Job Handler Duration in Milliseconds" %>
126
85
  "height": 6,
127
86
  "width": 9,
128
87
  "y": 3,
@@ -142,7 +101,7 @@ Resources:
142
101
  "liveData": true
143
102
  }
144
103
  },
145
- {
104
+ { <% "Incoming Job Handler Error count and success rate (%)" %>
146
105
  "height": 3,
147
106
  "width": 6,
148
107
  "y": 3,
@@ -167,7 +126,9 @@ Resources:
167
126
  "liveData": true
168
127
  }
169
128
  },
170
- {
129
+
130
+
131
+ { <% "Incoming Job Handler Concurrent Executions" %>
171
132
  "height": 3,
172
133
  "width": 6,
173
134
  "y": 6,
@@ -185,104 +146,127 @@ Resources:
185
146
  "liveData": true
186
147
  }
187
148
  },
188
- {
189
- "height": 6,
190
- "width": 9,
191
- "y": 12,
192
- "x": 9,
149
+
150
+
151
+
152
+ <% current_y = 9 %>
153
+ <%- queue_names.each do |queue_name| -%>
154
+ { <% "Active Jobs" %>
155
+ "height": 3,
156
+ "width": 6,
157
+ "y": <%= current_y %>,
158
+ "x": 0,
159
+ "type": "text",
160
+ "properties": {
161
+ "markdown": "\n# <%= queue_name.camelize %> Queue\n"
162
+ }
163
+ },
164
+ { <% "Active Jobs messages per minut" %>
165
+ "height": 3,
166
+ "width": 3,
167
+ "y": <%= current_y %>,
168
+ "x": 6,
193
169
  "type": "metric",
194
170
  "properties": {
195
- "period": 60,
196
171
  "metrics": [
197
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", { "stat": "Minimum" } ],
198
- [ "...", { "stat": "Average" } ],
199
- [ "...", { "stat": "Maximum" } ]
172
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName",
173
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueName}", { "label": "Messages Per Minute" } ]
200
174
  ],
175
+ "view": "singleValue",
201
176
  "region": "us-east-1",
202
- "title": "Active Job Handler Duration in Milliseconds",
203
- "view": "timeSeries",
204
- "stacked": false,
205
- "liveData": true
177
+ "stat": "Sum",
178
+ "period": 60,
179
+ "title": "Messages Per Minute"
206
180
  }
207
181
  },
208
- {
182
+ { <% "Active Job Handler Duration" %>
209
183
  "height": 3,
210
- "width": 6,
211
- "y": 12,
212
- "x": 18,
184
+ "width": 15,
185
+ "y": <%= current_y %>,
186
+ "x": 9,
213
187
  "type": "metric",
214
188
  "properties": {
215
- "period": 60,
216
189
  "metrics": [
217
- [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
218
- [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
219
- [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
190
+ [ "AWS/Lambda", "Duration", "FunctionName",
191
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", "Resource",
192
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "label": "p10" } ],
193
+ [ "...", { "label": "p50", "stat": "p50" } ],
194
+ [ "...", { "label": "p99", "stat": "p99" } ],
195
+ [ "...", { "label": "Average", "stat": "Average" } ]
220
196
  ],
197
+ "view": "singleValue",
221
198
  "region": "us-east-1",
222
- "title": "Active Job HandlerError count and success rate (%)",
223
- "yAxis": {
224
- "right": {
225
- "max": 100
226
- }
227
- },
228
- "view": "timeSeries",
229
- "stacked": false,
230
- "liveData": true
199
+ "stat": "p10",
200
+ "period": 60,
201
+ "title": "Handler Duration"
231
202
  }
232
203
  },
233
- {
234
- "height": 3,
235
- "width": 6,
236
- "y": 15,
237
- "x": 18,
204
+ { <% "Active Job Queue" %>
205
+ "height": 6,
206
+ "width": 9,
207
+ "y": <%= current_y + 3 %>,
208
+ "x": 0,
238
209
  "type": "metric",
239
210
  "properties": {
240
- "period": 60,
241
211
  "metrics": [
242
- [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", { "stat": "Maximum" } ]
212
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName",
213
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueName}",
214
+ { "label": "Received" } ],
215
+ [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
216
+ [ "AWS/Lambda", "Invocations", "FunctionName",
217
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", "Resource",
218
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "label": "Handler Invocations" } ],
219
+ [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName",
220
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueName}", { "label": "Pending?" } ],
221
+ [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
222
+ [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
223
+ [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
243
224
  ],
244
- "region": "us-east-1",
245
- "title": "Active Job Handler Concurrent executions",
246
225
  "view": "timeSeries",
247
226
  "stacked": false,
227
+ "region": "us-east-1",
228
+ "title": "<%= queue_name.camelize %> Queue (Async jobs go here immediately, scheduled jobs land here in the minute before they're scheduled)",
229
+ "period": 60,
230
+ "stat": "Sum",
248
231
  "liveData": true
249
232
  }
250
233
  },
251
- {
252
- "height": 3,
234
+ { <% "Active Job Handler Duration in Milliseconds" %>
235
+ "height": 6,
253
236
  "width": 9,
254
- "y": 27,
237
+ "y": <%= current_y + 3 %>,
255
238
  "x": 9,
256
239
  "type": "metric",
257
240
  "properties": {
258
241
  "period": 60,
259
242
  "metrics": [
260
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Minimum" } ],
243
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "stat": "Minimum" } ],
261
244
  [ "...", { "stat": "Average" } ],
262
245
  [ "...", { "stat": "Maximum" } ]
263
246
  ],
264
247
  "region": "us-east-1",
265
- "title": "Delayed Job Scheduler Duration",
248
+ "title": "<%= queue_name.camelize %> Queue Handler Duration in Milliseconds",
266
249
  "view": "timeSeries",
267
250
  "stacked": false,
268
251
  "liveData": true
269
252
  }
270
253
  },
271
- {
254
+ { <% "Active Job HandlerError count and success rate (%)" %>
272
255
  "height": 3,
273
256
  "width": 6,
274
- "y": 27,
257
+ "y": <%= current_y + 3 %>,
275
258
  "x": 18,
276
259
  "type": "metric",
277
260
  "properties": {
278
261
  "period": 60,
279
262
  "metrics": [
280
- [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
263
+ [ "AWS/Lambda", "Errors", "FunctionName",
264
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
281
265
  [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
282
266
  [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
283
267
  ],
284
268
  "region": "us-east-1",
285
- "title": "Delayed Job SchedulerError count and success rate (%)",
269
+ "title": "<%= queue_name.camelize %> Queue Handler Error count and success rate (%)",
286
270
  "yAxis": {
287
271
  "right": {
288
272
  "max": 100
@@ -293,188 +277,214 @@ Resources:
293
277
  "liveData": true
294
278
  }
295
279
  },
296
- {
280
+ { <% "Active Job Handler Concurrent executions" %>
297
281
  "height": 3,
298
282
  "width": 6,
299
- "y": 30,
283
+ "y": <%= current_y + 6 %>,
300
284
  "x": 18,
301
285
  "type": "metric",
302
286
  "properties": {
303
287
  "period": 60,
304
288
  "metrics": [
305
- [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Maximum" } ]
289
+ [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName",
290
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "stat": "Maximum" } ]
306
291
  ],
307
292
  "region": "us-east-1",
308
- "title": "Delayd Job Schedule Concurrent executions",
293
+ "title": "<%= queue_name.camelize %> Queue Handler Concurrent executions",
309
294
  "view": "timeSeries",
310
295
  "stacked": false,
311
296
  "liveData": true
312
297
  }
313
298
  },
314
- {
315
- "height": 3,
316
- "width": 9,
317
- "y": 27,
299
+
300
+ <% current_y += 9 %>
301
+ <%- end -%>
302
+
303
+ { <% "Job Duration By Worker" %>
304
+ "height": 6,
305
+ "width": 24,
306
+ "y": <%= current_y %>,
318
307
  "x": 0,
319
308
  "type": "metric",
320
309
  "properties": {
321
310
  "metrics": [
322
- [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
311
+ [ "rails-lambda-experiment", "Duration", "WorkerClassName", "HelloWorker" ],
312
+ [ "...", { "stat": "p99" } ],
313
+ [ "...", "HelloLaterWorker" ],
314
+ [ "...", { "stat": "p99" } ]
323
315
  ],
324
316
  "view": "timeSeries",
325
317
  "stacked": false,
326
318
  "region": "us-east-1",
327
- "title": "Delayed Jobs To Be Executed In The Next 90 Seconds",
328
- "period": 60,
329
319
  "stat": "Average",
330
- "liveData": true
331
- }
332
- },
333
- {
334
- "height": 3,
335
- "width": 6,
336
- "y": 0,
337
- "x": 0,
338
- "type": "text",
339
- "properties": {
340
- "markdown": "\n# Incoming Jobs\n"
341
- }
342
- },
343
- {
344
- "height": 3,
345
- "width": 6,
346
- "y": 9,
347
- "x": 0,
348
- "type": "text",
349
- "properties": {
350
- "markdown": "\n# Active Jobs\n"
320
+ "period": 60,
321
+ "title": "Job Duration by Worker"
351
322
  }
352
323
  },
353
- {
324
+ <% current_y += 6 %>
325
+
326
+ { <% "Delayed Jobs" %>
354
327
  "height": 3,
355
328
  "width": 6,
356
- "y": 24,
329
+ "y": <%= current_y %>,
357
330
  "x": 0,
358
331
  "type": "text",
359
332
  "properties": {
360
333
  "markdown": "\n# Delayed Jobs\n"
361
334
  }
362
335
  },
363
- {
336
+ { <% "Messages To Be Scheduled" %>
364
337
  "height": 3,
365
338
  "width": 3,
366
- "y": 0,
339
+ "y": <%= current_y %>,
367
340
  "x": 6,
368
341
  "type": "metric",
369
342
  "properties": {
370
343
  "metrics": [
371
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
344
+ [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
372
345
  ],
373
346
  "view": "singleValue",
374
347
  "region": "us-east-1",
375
- "stat": "Sum",
348
+ "stat": "Average",
376
349
  "period": 60,
377
- "title": "Messages Per Minute"
350
+ "title": "Messages To Be Scheduled"
378
351
  }
379
352
  },
380
- {
353
+ { <% "Delayed Job Scheduler Duration" %>
381
354
  "height": 3,
382
- "width": 3,
383
- "y": 9,
384
- "x": 6,
355
+ "width": 15,
356
+ "y": <%= current_y %>,
357
+ "x": 9,
385
358
  "type": "metric",
386
359
  "properties": {
387
360
  "metrics": [
388
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:service}-${self:custom.stage}-active-jobs", { "label": "Messages Per Minute" } ]
361
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", "Resource", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "label": "p10" } ],
362
+ [ "...", { "label": "p50", "stat": "p50" } ],
363
+ [ "...", { "label": "p99", "stat": "p99" } ],
364
+ [ "...", { "label": "Average", "stat": "Average" } ]
389
365
  ],
390
366
  "view": "singleValue",
391
367
  "region": "us-east-1",
392
- "stat": "Sum",
368
+ "stat": "p10",
393
369
  "period": 60,
394
- "title": "Messages Per Minute"
370
+ "title": "Handler Duration"
395
371
  }
396
372
  },
397
- {
373
+
374
+
375
+
376
+ <% current_y += 3 %>
377
+ { <% "Delayed Job SchedulerError count and success rate (%)" %>
398
378
  "height": 3,
399
- "width": 15,
400
- "y": 0,
401
- "x": 9,
379
+ "width": 6,
380
+ "y": <%= current_y %>,
381
+ "x": 18,
402
382
  "type": "metric",
403
383
  "properties": {
384
+ "period": 60,
404
385
  "metrics": [
405
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
406
- [ "...", { "label": "p50", "stat": "p50" } ],
407
- [ "...", { "label": "p99", "stat": "p99" } ],
408
- [ "...", { "label": "Average", "stat": "Average" } ]
386
+ [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
387
+ [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
388
+ [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
409
389
  ],
410
- "view": "singleValue",
411
390
  "region": "us-east-1",
412
- "stat": "p10",
413
- "period": 60,
414
- "title": "Handler Duration"
391
+ "title": "Delayed Job SchedulerError count and success rate (%)",
392
+ "yAxis": {
393
+ "right": {
394
+ "max": 100
395
+ }
396
+ },
397
+ "view": "timeSeries",
398
+ "stacked": false,
399
+ "liveData": true
415
400
  }
416
401
  },
417
- {
402
+ { <% "Delayed Job Scheduler Duration" %>
418
403
  "height": 3,
419
- "width": 15,
420
- "y": 9,
404
+ "width": 9,
405
+ "y": <%= current_y %>,
421
406
  "x": 9,
422
407
  "type": "metric",
423
408
  "properties": {
409
+ "period": 60,
424
410
  "metrics": [
425
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", "Resource", "${self:service}-${self:provider.stage}-active_job_handler", { "label": "p10" } ],
426
- [ "...", { "label": "p50", "stat": "p50" } ],
427
- [ "...", { "label": "p99", "stat": "p99" } ],
428
- [ "...", { "label": "Average", "stat": "Average" } ]
411
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Minimum" } ],
412
+ [ "...", { "stat": "Average" } ],
413
+ [ "...", { "stat": "Maximum" } ]
429
414
  ],
430
- "view": "singleValue",
431
415
  "region": "us-east-1",
432
- "stat": "p10",
433
- "period": 60,
434
- "title": "Handler Duration"
416
+ "title": "Delayed Job Scheduler Duration",
417
+ "view": "timeSeries",
418
+ "stacked": false,
419
+ "liveData": true
435
420
  }
436
421
  },
437
- {
422
+ { <% "Delayed Jobs To Be Executed In The Next 90 Seconds" %>
438
423
  "height": 3,
439
- "width": 3,
440
- "y": 24,
441
- "x": 6,
424
+ "width": 9,
425
+ "y": <%= current_y %>,
426
+ "x": 0,
442
427
  "type": "metric",
443
428
  "properties": {
444
429
  "metrics": [
445
430
  [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
446
431
  ],
447
- "view": "singleValue",
432
+ "view": "timeSeries",
433
+ "stacked": false,
448
434
  "region": "us-east-1",
449
- "stat": "Average",
435
+ "title": "Delayed Jobs To Be Executed In The Next 90 Seconds",
450
436
  "period": 60,
451
- "title": "Messages To Be Scheduled"
437
+ "stat": "Average",
438
+ "liveData": true
452
439
  }
453
440
  },
454
- {
441
+
442
+
443
+ <% current_y += 3 %>
444
+ { <% "Scheduled Job Table Read Capacity Units" %>
455
445
  "height": 3,
456
- "width": 15,
457
- "y": 24,
458
- "x": 9,
446
+ "width": 9,
447
+ "y": <%= current_y %>,
448
+ "x": 0,
459
449
  "type": "metric",
460
450
  "properties": {
461
451
  "metrics": [
462
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", "Resource", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "label": "p10" } ],
463
- [ "...", { "label": "p50", "stat": "p50" } ],
464
- [ "...", { "label": "p99", "stat": "p99" } ],
465
- [ "...", { "label": "Average", "stat": "Average" } ]
452
+ [ { "expression": "m2/PERIOD(m2)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
453
+ [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
454
+ [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
455
+ [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
466
456
  ],
467
- "view": "singleValue",
457
+ "view": "timeSeries",
458
+ "stacked": false,
468
459
  "region": "us-east-1",
469
- "stat": "p10",
460
+ "title": "Scheduled Job Table Read Capacity Units",
470
461
  "period": 60,
471
- "title": "Handler Duration"
462
+ "stat": "Sum",
463
+ "liveData": true
464
+ }
465
+ },
466
+ { <% "Delayd Job Schedule Concurrent executions" %>
467
+ "height": 3,
468
+ "width": 6,
469
+ "y": <%= current_y %>,
470
+ "x": 18,
471
+ "type": "metric",
472
+ "properties": {
473
+ "period": 60,
474
+ "metrics": [
475
+ [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Maximum" } ]
476
+ ],
477
+ "region": "us-east-1",
478
+ "title": "Delayd Job Schedule Concurrent executions",
479
+ "view": "timeSeries",
480
+ "stacked": false,
481
+ "liveData": true
472
482
  }
473
483
  },
474
- {
484
+ { <% "Scheduled Job Table Write Capacity Units" %>
475
485
  "height": 3,
476
486
  "width": 9,
477
- "y": 30,
487
+ "y": <%= current_y %>,
478
488
  "x": 9,
479
489
  "type": "metric",
480
490
  "properties": {
@@ -493,26 +503,53 @@ Resources:
493
503
  "liveData": true
494
504
  }
495
505
  },
496
- {
497
- "height": 6,
498
- "width": 24,
499
- "y": 18,
506
+
507
+
508
+
509
+ <% current_y += 3 %>
510
+ { <% "Scheduled Job Table Latency" %>
511
+ "height": 3,
512
+ "width": 18,
513
+ "y": <%= current_y %>,
500
514
  "x": 0,
501
515
  "type": "metric",
502
516
  "properties": {
503
517
  "metrics": [
504
- [ "rails-lambda-experiment", "Duration", "WorkerClassName", "HelloWorker" ],
505
- [ "...", { "stat": "p99" } ],
506
- [ "...", "HelloLaterWorker" ],
507
- [ "...", { "stat": "p99" } ]
518
+ [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "PutItem", { "yAxis": "left" } ],
519
+ [ "...", "Query" ],
520
+ [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
521
+ [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
522
+ [ ".", "ThrottledRequests", ".", ".", ".", ".", { "yAxis": "right", "visible": false } ]
508
523
  ],
509
524
  "view": "timeSeries",
510
525
  "stacked": false,
511
526
  "region": "us-east-1",
512
527
  "stat": "Average",
513
528
  "period": 60,
514
- "title": "Job Duration by Worker"
529
+ "title": "Scheduled Job Table Latency",
530
+ "liveData": true
531
+ }
532
+ },
533
+ { <% "Scheduled Job Table Throttled Operations" %>
534
+ "height": 3,
535
+ "width": 6,
536
+ "y": <%= current_y %>,
537
+ "x": 18,
538
+ "type": "metric",
539
+ "properties": {
540
+ "metrics": [
541
+ [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "DeleteItem" ],
542
+ [ "...", "PutItem" ]
543
+ ],
544
+ "view": "timeSeries",
545
+ "stacked": false,
546
+ "region": "us-east-1",
547
+ "stat": "Sum",
548
+ "period": 60,
549
+ "title": "Scheduled Job Table Throttled Operations",
550
+ "liveData": true
515
551
  }
516
552
  }
553
+
517
554
  ]
518
555
  }
@@ -5,12 +5,12 @@ runtime: ruby2.7
5
5
 
6
6
  # These values will be applied to any funktor handlers that don't specify custom values.
7
7
  # TODO - These still need to be wired up to do anything.
8
- handler_defaults:
8
+ handlerDefaults:
9
9
  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
10
10
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
11
11
  # expect to spend for one message. We default to a high number here to allow for the
12
12
  # times when things go weird.
13
- timeout: 900
13
+ timeout: 300
14
14
 
15
15
  # reservedConcurrency represents the maximum number of concurrent executions.
16
16
  # Usually you'll want to leave this as null so that handlers can scale infinitely
@@ -44,20 +44,17 @@ handler_defaults:
44
44
  logRetentionInDays: 30
45
45
 
46
46
  # Incoming Job Handler
47
- # TODO - These still need to be wired up to do anything.
48
- incoming_job_handler:
47
+ incomingJobHandler:
49
48
  # If your jobs are bursty AND time-sensitive you might want to have some lambdas pre-provisioned
50
49
  # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
51
50
  # provisionedConcurrency: 4
52
51
 
53
52
 
54
- # TODO - These still need to be wired up to do anything.
55
53
  queues:
56
54
  - default:
57
55
  # Set queue specific config options here
58
56
  # memorySize: 512
59
- # TODO - Is it advisable to use FIFO queuues with Funktor?
60
- # TODO - Maybe this isn't really even supported by CloudFormation?
57
+ # TODO - Is it advisable to use FIFO queuues with Funktor? Maybe this isn't really even supported by CloudFormation?
61
58
  # fifo: false
62
59
  - singleThread:
63
60
  reservedConcurrency: 1
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.2.2"
2
+ VERSION = "0.2.3"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-06-16 00:00:00.000000000 Z
11
+ date: 2021-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs