funktor 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 38d383dcdafdb71a93c3ab881ec7772e8694fe0a8ca913a11a27ab634b068349
4
- data.tar.gz: 92e17fab7ca35700c1721dbd9f674d14a1dd8a033486ed1af9d65236cad305cc
3
+ metadata.gz: 700b659a76af0a0e7e6a21c91a91553f4d5ff5899140ef18313bd181d97de4e1
4
+ data.tar.gz: f9d311402f6a0686e11c0d30131135fc5372bf58cc8e6c32c0f5a90f649861d7
5
5
  SHA512:
6
- metadata.gz: 1643f95d5f02f83272c2304e6c22cda5c91bb39f4f109cd4d363d9b667294679dbfa4634962162193dd9c2ea90da57f21048a41d2b5710a80f19c91631136f8d
7
- data.tar.gz: 559c5693bf8a958b22e0b6b189f72ca43d758701498ccb85c52c3cf3a9e7c2b148d26d17efdea72089328e393c1ad671b3d03b6f8dcee1ac39498fc6cf6f7535
6
+ metadata.gz: 47e9682f6d0f96e9ce5d393574642b8361049c91f07dc2b9154c4492c0b10257bc651e22ea5512a1f904cd2a895c9435d9722c559b756086618f1ea58983298a
7
+ data.tar.gz: a498a2854d47e3c2b0bfd9f1bb2d2feb31b5bd3b8ac1f34c79a5efd127cb81d77862147cd6d27528589710676b748bd744bdd3ace35ebfdb118d4983a3414ac0
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- funktor (0.2.2)
4
+ funktor (0.2.3)
5
5
  activesupport
6
6
  aws-sdk-sqs (~> 1.37)
7
7
  thor
@@ -18,8 +18,8 @@ GEM
18
18
  addressable (2.7.0)
19
19
  public_suffix (>= 2.0.2, < 5.0)
20
20
  aws-eventstream (1.1.1)
21
- aws-partitions (1.465.0)
22
- aws-sdk-core (3.114.1)
21
+ aws-partitions (1.468.0)
22
+ aws-sdk-core (3.114.3)
23
23
  aws-eventstream (~> 1, >= 1.0.2)
24
24
  aws-partitions (~> 1, >= 1.239.0)
25
25
  aws-sigv4 (~> 1.1)
@@ -29,7 +29,7 @@ GEM
29
29
  aws-sigv4 (~> 1.1)
30
30
  aws-sigv4 (1.2.3)
31
31
  aws-eventstream (~> 1, >= 1.0.2)
32
- concurrent-ruby (1.1.8)
32
+ concurrent-ruby (1.1.9)
33
33
  crack (0.4.5)
34
34
  rexml
35
35
  diff-lcs (1.4.4)
@@ -15,7 +15,7 @@ module Funktor
15
15
 
16
16
  class_option :file, :aliases => "-f",
17
17
  :type => :string, :desc => "The funktor init file.",
18
- :default => "funktor.yml"
18
+ :default => "funktor_init.yml"
19
19
 
20
20
  desc <<~DESC
21
21
  Description:
@@ -127,6 +127,29 @@ module Funktor
127
127
  @work_queue_config
128
128
  end
129
129
 
130
+ def queue_config(queue_name)
131
+ funktor_config["queues"].each do |queue_details|
132
+ if queue_details.keys.first == queue_name
133
+ return queue_details.values.first
134
+ end
135
+ end
136
+ return nil
137
+ end
138
+
139
+ def incoming_config_value(config_name)
140
+ funktor_config.dig("incomingJobHandler", config_name) ||
141
+ funktor_config.dig("handlerDefaults", config_name) ||
142
+ "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
143
+ end
144
+
145
+ def queue_config_value(queue_name, config_name)
146
+ puts funktor_config
147
+ queue_config(queue_name)&.dig(config_name) ||
148
+ funktor_config.dig("handlerDefaults", config_name) ||
149
+ "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
150
+
151
+ end
152
+
130
153
  end
131
154
  end
132
155
  end
@@ -3,7 +3,5 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
3
3
 
4
4
  ruby '~> 2.7'
5
5
 
6
- source "https://gem.fury.io/jagthedrummer/" do
7
- gem 'funktor'
8
- end
6
+ gem 'funktor'
9
7
 
@@ -3,19 +3,23 @@ incomingJobHandler:
3
3
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
4
4
  # expect to spend for one message. The incoming job handler usually will be pretty fast,
5
5
  # but we default to a high number here to allow for the times when things go weird.
6
- timeout: 300
6
+ timeout: <%= incoming_config_value 'timeout' %>
7
7
  # reservedConcurrency represents the maximum number of concurrent executions.
8
8
  # For the incoming job handler you probably don't want to limit it because you
9
9
  # want to get things onto work queues as quickly as possible.
10
- reservedConcurrency: null
10
+ reservedConcurrency: <%= incoming_config_value 'reservedConcurrency' %>
11
11
  # provisionedConcurrency represents the number of lambda functions that will always
12
12
  # be available. For the incoming jobs handler you probably don't need to set this
13
13
  # unless your jobs are very bursty AND very time sensitive.
14
- provisionedConcurrency: null
14
+ provisionedConcurrency: <%= incoming_config_value 'provisionedConcurrency' %>
15
15
  # Use memory_size to adjust the reousrces (both memory and CPU) available.
16
16
  # For the incoming jobs handler you probably don't need this to be too large,
17
17
  # but if you're seeing large delays at this stage it might help to bump it up.
18
- memorySize: 256
18
+ memorySize: <%= incoming_config_value 'memorySize' %>
19
+ batchSize: <%= incoming_config_value 'batchSize' %>
20
+ maximumBatchingWindow: <%= incoming_config_value 'maximumBatchingWindow' %>
21
+ visibilityTimeout: <%= incoming_config_value 'visibilityTimeout' %>
22
+ logRetentionInDays: <%= incoming_config_value 'logRetentionInDays' %>
19
23
 
20
24
  <%- queue_names.each do |queue_name| -%>
21
25
  <%= queue_name.camelize %>QueueHandler:
@@ -23,23 +27,26 @@ incomingJobHandler:
23
27
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
24
28
  # expect to spend for one message. The active job handler may be slow if your jobs are
25
29
  # doing a lot of work, so we default to the maximum here.
26
- timeout: 300 # TODO pull this (and other values) from funktor_init.yml
30
+ timeout: <%= queue_config_value queue_name, 'timeout' %>
27
31
  # reservedConcurrency represents the maximum number of concurrent executions.
28
32
  # For the active job handler you may want to limit it if you have resource limitations
29
33
  # like database connections that you need to avoid exhausting.
30
- reservedConcurrency: null
34
+ reservedConcurrency: <%= queue_config_value queue_name, 'reservedConcurrency' %>
31
35
  # provisionedConcurrency represents the number of lambda functions that will always
32
36
  # be available. For the active job handler you probably don't need to set this
33
37
  # unless your jobs are very bursty AND very time sensitive.
34
- provisionedConcurrency: null
38
+ provisionedConcurrency: <%= queue_config_value queue_name, 'provisionedConcurrency' %>
35
39
  # Use memory_size to adjust the reousrces (both memory and CPU) available.
36
40
  # For the active jobs handler you'll want this to be at least as large as the memory
37
41
  # required to actually do your jobs. You can choose an even higher number to increase
38
42
  # the available CPU to make the jobs run faster.
39
- memorySize: 256
40
- <%- end -%>
41
-
43
+ memorySize: <%= queue_config_value queue_name, 'memorySize' %>
44
+ batchSize: <%= queue_config_value queue_name, 'batchSize' %>
45
+ maximumBatchingWindow: <%= queue_config_value queue_name, 'maximumBatchingWindow' %>
46
+ visibilityTimeout: <%= queue_config_value queue_name, 'visibilityTimeout' %>
47
+ logRetentionInDays: <%= queue_config_value queue_name, 'logRetentionInDays' %>
42
48
 
49
+ <%- end -%>
43
50
 
44
51
 
45
52
  # You shouldn't need to mess with these under most circumstances. But you could if you want to change
@@ -51,6 +58,7 @@ dashboardName: ${self:service}-${self:custom.stage}-dashboard
51
58
  <%- queue_names.each do |queue_name| -%>
52
59
  <%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
53
60
  <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
61
+ <%= queue_name.camelize %>QueueHandlerName: ${self:service}-${self:custom.stage}-<%= queue_name.camelize %>QueueHandler
54
62
  <%- end -%>
55
63
 
56
64
 
@@ -6,123 +6,82 @@ Resources:
6
6
  DashboardBody: >
7
7
  {
8
8
  "widgets": [
9
- {
10
- "height": 6,
11
- "width": 9,
12
- "y": 3,
9
+
10
+ { <% "Incoming Jobs" %>
11
+ "height": 3,
12
+ "width": 6,
13
+ "y": 0,
13
14
  "x": 0,
14
- "type": "metric",
15
+ "type": "text",
15
16
  "properties": {
16
- "metrics": [
17
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
18
- [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
19
- [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
20
- [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
21
- [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
22
- [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
23
- [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
24
- ],
25
- "view": "timeSeries",
26
- "stacked": false,
27
- "region": "us-east-1",
28
- "title": "Incoming Job Queue (Async & scheduled jobs land here first)",
29
- "period": 60,
30
- "stat": "Sum",
31
- "setPeriodToTimeRange": true,
32
- "liveData": true
17
+ "markdown": "\n# Incoming Jobs\n"
33
18
  }
34
19
  },
35
- {
36
- "height": 6,
37
- "width": 9,
38
- "y": 12,
39
- "x": 0,
20
+ { <% "Incoming Job Queue Messages per minute" %>
21
+ "height": 3,
22
+ "width": 3,
23
+ "y": 0,
24
+ "x": 6,
40
25
  "type": "metric",
41
26
  "properties": {
42
27
  "metrics": [
43
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.DefaultQueueName}", { "label": "Received" } ],
44
- [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
45
- [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-DefaultQueueHandler", "Resource", "${self:service}-${self:provider.stage}-DefaultQueueHandler", { "label": "Handler Invocations" } ],
46
- [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.DefaultQueueName}", { "label": "Pending?" } ],
47
- [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
48
- [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
49
- [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
28
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
50
29
  ],
51
- "view": "timeSeries",
52
- "stacked": false,
30
+ "view": "singleValue",
53
31
  "region": "us-east-1",
54
- "title": "Active Job Queue (Async jobs go here immediately, scheduled jobs land here in the minute before they're scheduled)",
55
- "period": 60,
56
32
  "stat": "Sum",
57
- "liveData": true
33
+ "period": 60,
34
+ "title": "Messages Per Minute"
58
35
  }
59
36
  },
60
- {
37
+ { <% "Incoming Job Handler Duration" %>
61
38
  "height": 3,
62
- "width": 9,
63
- "y": 30,
64
- "x": 0,
39
+ "width": 15,
40
+ "y": 0,
41
+ "x": 9,
65
42
  "type": "metric",
66
43
  "properties": {
67
44
  "metrics": [
68
- [ { "expression": "m2/PERIOD(m2)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
69
- [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
70
- [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
71
- [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
45
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
46
+ [ "...", { "label": "p50", "stat": "p50" } ],
47
+ [ "...", { "label": "p99", "stat": "p99" } ],
48
+ [ "...", { "label": "Average", "stat": "Average" } ]
72
49
  ],
73
- "view": "timeSeries",
74
- "stacked": false,
50
+ "view": "singleValue",
75
51
  "region": "us-east-1",
76
- "title": "Scheduled Job Table Read Capacity Units",
52
+ "stat": "p10",
77
53
  "period": 60,
78
- "stat": "Sum",
79
- "liveData": true
54
+ "title": "Handler Duration"
80
55
  }
81
56
  },
82
- {
83
- "height": 3,
84
- "width": 18,
85
- "y": 33,
57
+
58
+ { <% "Incoming Job Queue (Async & scheduled jobs land here first)" %>
59
+ "height": 6,
60
+ "width": 9,
61
+ "y": 3,
86
62
  "x": 0,
87
63
  "type": "metric",
88
64
  "properties": {
89
65
  "metrics": [
90
- [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "PutItem", { "yAxis": "left" } ],
91
- [ "...", "Query" ],
92
- [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
93
- [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
94
- [ ".", "ThrottledRequests", ".", ".", ".", ".", { "yAxis": "right", "visible": false } ]
66
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
67
+ [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
68
+ [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
69
+ [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
70
+ [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
71
+ [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
72
+ [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
95
73
  ],
96
74
  "view": "timeSeries",
97
75
  "stacked": false,
98
76
  "region": "us-east-1",
99
- "stat": "Average",
77
+ "title": "Incoming Job Queue (Async & scheduled jobs land here first)",
100
78
  "period": 60,
101
- "title": "Scheduled Job Table Latency",
102
- "liveData": true
103
- }
104
- },
105
- {
106
- "height": 3,
107
- "width": 6,
108
- "y": 33,
109
- "x": 18,
110
- "type": "metric",
111
- "properties": {
112
- "metrics": [
113
- [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "DeleteItem" ],
114
- [ "...", "PutItem" ]
115
- ],
116
- "view": "timeSeries",
117
- "stacked": false,
118
- "region": "us-east-1",
119
79
  "stat": "Sum",
120
- "period": 60,
121
- "title": "Scheduled Job Table Throttled Operations",
80
+ "setPeriodToTimeRange": true,
122
81
  "liveData": true
123
82
  }
124
83
  },
125
- {
84
+ { <% "Incoming Job Handler Duration in Milliseconds" %>
126
85
  "height": 6,
127
86
  "width": 9,
128
87
  "y": 3,
@@ -142,7 +101,7 @@ Resources:
142
101
  "liveData": true
143
102
  }
144
103
  },
145
- {
104
+ { <% "Incoming Job Handler Error count and success rate (%)" %>
146
105
  "height": 3,
147
106
  "width": 6,
148
107
  "y": 3,
@@ -167,7 +126,9 @@ Resources:
167
126
  "liveData": true
168
127
  }
169
128
  },
170
- {
129
+
130
+
131
+ { <% "Incoming Job Handler Concurrent Executions" %>
171
132
  "height": 3,
172
133
  "width": 6,
173
134
  "y": 6,
@@ -185,104 +146,127 @@ Resources:
185
146
  "liveData": true
186
147
  }
187
148
  },
188
- {
189
- "height": 6,
190
- "width": 9,
191
- "y": 12,
192
- "x": 9,
149
+
150
+
151
+
152
+ <% current_y = 9 %>
153
+ <%- queue_names.each do |queue_name| -%>
154
+ { <% "Active Jobs" %>
155
+ "height": 3,
156
+ "width": 6,
157
+ "y": <%= current_y %>,
158
+ "x": 0,
159
+ "type": "text",
160
+ "properties": {
161
+ "markdown": "\n# <%= queue_name.camelize %> Queue\n"
162
+ }
163
+ },
164
+ { <% "Active Jobs messages per minut" %>
165
+ "height": 3,
166
+ "width": 3,
167
+ "y": <%= current_y %>,
168
+ "x": 6,
193
169
  "type": "metric",
194
170
  "properties": {
195
- "period": 60,
196
171
  "metrics": [
197
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", { "stat": "Minimum" } ],
198
- [ "...", { "stat": "Average" } ],
199
- [ "...", { "stat": "Maximum" } ]
172
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName",
173
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueName}", { "label": "Messages Per Minute" } ]
200
174
  ],
175
+ "view": "singleValue",
201
176
  "region": "us-east-1",
202
- "title": "Active Job Handler Duration in Milliseconds",
203
- "view": "timeSeries",
204
- "stacked": false,
205
- "liveData": true
177
+ "stat": "Sum",
178
+ "period": 60,
179
+ "title": "Messages Per Minute"
206
180
  }
207
181
  },
208
- {
182
+ { <% "Active Job Handler Duration" %>
209
183
  "height": 3,
210
- "width": 6,
211
- "y": 12,
212
- "x": 18,
184
+ "width": 15,
185
+ "y": <%= current_y %>,
186
+ "x": 9,
213
187
  "type": "metric",
214
188
  "properties": {
215
- "period": 60,
216
189
  "metrics": [
217
- [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
218
- [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
219
- [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
190
+ [ "AWS/Lambda", "Duration", "FunctionName",
191
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", "Resource",
192
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "label": "p10" } ],
193
+ [ "...", { "label": "p50", "stat": "p50" } ],
194
+ [ "...", { "label": "p99", "stat": "p99" } ],
195
+ [ "...", { "label": "Average", "stat": "Average" } ]
220
196
  ],
197
+ "view": "singleValue",
221
198
  "region": "us-east-1",
222
- "title": "Active Job HandlerError count and success rate (%)",
223
- "yAxis": {
224
- "right": {
225
- "max": 100
226
- }
227
- },
228
- "view": "timeSeries",
229
- "stacked": false,
230
- "liveData": true
199
+ "stat": "p10",
200
+ "period": 60,
201
+ "title": "Handler Duration"
231
202
  }
232
203
  },
233
- {
234
- "height": 3,
235
- "width": 6,
236
- "y": 15,
237
- "x": 18,
204
+ { <% "Active Job Queue" %>
205
+ "height": 6,
206
+ "width": 9,
207
+ "y": <%= current_y + 3 %>,
208
+ "x": 0,
238
209
  "type": "metric",
239
210
  "properties": {
240
- "period": 60,
241
211
  "metrics": [
242
- [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", { "stat": "Maximum" } ]
212
+ [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName",
213
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueName}",
214
+ { "label": "Received" } ],
215
+ [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
216
+ [ "AWS/Lambda", "Invocations", "FunctionName",
217
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", "Resource",
218
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "label": "Handler Invocations" } ],
219
+ [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName",
220
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueName}", { "label": "Pending?" } ],
221
+ [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
222
+ [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
223
+ [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
243
224
  ],
244
- "region": "us-east-1",
245
- "title": "Active Job Handler Concurrent executions",
246
225
  "view": "timeSeries",
247
226
  "stacked": false,
227
+ "region": "us-east-1",
228
+ "title": "<%= queue_name.camelize %> Queue (Async jobs go here immediately, scheduled jobs land here in the minute before they're scheduled)",
229
+ "period": 60,
230
+ "stat": "Sum",
248
231
  "liveData": true
249
232
  }
250
233
  },
251
- {
252
- "height": 3,
234
+ { <% "Active Job Handler Duration in Milliseconds" %>
235
+ "height": 6,
253
236
  "width": 9,
254
- "y": 27,
237
+ "y": <%= current_y + 3 %>,
255
238
  "x": 9,
256
239
  "type": "metric",
257
240
  "properties": {
258
241
  "period": 60,
259
242
  "metrics": [
260
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Minimum" } ],
243
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "stat": "Minimum" } ],
261
244
  [ "...", { "stat": "Average" } ],
262
245
  [ "...", { "stat": "Maximum" } ]
263
246
  ],
264
247
  "region": "us-east-1",
265
- "title": "Delayed Job Scheduler Duration",
248
+ "title": "<%= queue_name.camelize %> Queue Handler Duration in Milliseconds",
266
249
  "view": "timeSeries",
267
250
  "stacked": false,
268
251
  "liveData": true
269
252
  }
270
253
  },
271
- {
254
+ { <% "Active Job HandlerError count and success rate (%)" %>
272
255
  "height": 3,
273
256
  "width": 6,
274
- "y": 27,
257
+ "y": <%= current_y + 3 %>,
275
258
  "x": 18,
276
259
  "type": "metric",
277
260
  "properties": {
278
261
  "period": 60,
279
262
  "metrics": [
280
- [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
263
+ [ "AWS/Lambda", "Errors", "FunctionName",
264
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
281
265
  [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
282
266
  [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
283
267
  ],
284
268
  "region": "us-east-1",
285
- "title": "Delayed Job SchedulerError count and success rate (%)",
269
+ "title": "<%= queue_name.camelize %> Queue Handler Error count and success rate (%)",
286
270
  "yAxis": {
287
271
  "right": {
288
272
  "max": 100
@@ -293,188 +277,214 @@ Resources:
293
277
  "liveData": true
294
278
  }
295
279
  },
296
- {
280
+ { <% "Active Job Handler Concurrent executions" %>
297
281
  "height": 3,
298
282
  "width": 6,
299
- "y": 30,
283
+ "y": <%= current_y + 6 %>,
300
284
  "x": 18,
301
285
  "type": "metric",
302
286
  "properties": {
303
287
  "period": 60,
304
288
  "metrics": [
305
- [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Maximum" } ]
289
+ [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName",
290
+ "${self:custom.funktor.<%= queue_name.camelize %>QueueHandlerName}", { "stat": "Maximum" } ]
306
291
  ],
307
292
  "region": "us-east-1",
308
- "title": "Delayd Job Schedule Concurrent executions",
293
+ "title": "<%= queue_name.camelize %> Queue Handler Concurrent executions",
309
294
  "view": "timeSeries",
310
295
  "stacked": false,
311
296
  "liveData": true
312
297
  }
313
298
  },
314
- {
315
- "height": 3,
316
- "width": 9,
317
- "y": 27,
299
+
300
+ <% current_y += 9 %>
301
+ <%- end -%>
302
+
303
+ { <% "Job Duration By Worker" %>
304
+ "height": 6,
305
+ "width": 24,
306
+ "y": <%= current_y %>,
318
307
  "x": 0,
319
308
  "type": "metric",
320
309
  "properties": {
321
310
  "metrics": [
322
- [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
311
+ [ "rails-lambda-experiment", "Duration", "WorkerClassName", "HelloWorker" ],
312
+ [ "...", { "stat": "p99" } ],
313
+ [ "...", "HelloLaterWorker" ],
314
+ [ "...", { "stat": "p99" } ]
323
315
  ],
324
316
  "view": "timeSeries",
325
317
  "stacked": false,
326
318
  "region": "us-east-1",
327
- "title": "Delayed Jobs To Be Executed In The Next 90 Seconds",
328
- "period": 60,
329
319
  "stat": "Average",
330
- "liveData": true
331
- }
332
- },
333
- {
334
- "height": 3,
335
- "width": 6,
336
- "y": 0,
337
- "x": 0,
338
- "type": "text",
339
- "properties": {
340
- "markdown": "\n# Incoming Jobs\n"
341
- }
342
- },
343
- {
344
- "height": 3,
345
- "width": 6,
346
- "y": 9,
347
- "x": 0,
348
- "type": "text",
349
- "properties": {
350
- "markdown": "\n# Active Jobs\n"
320
+ "period": 60,
321
+ "title": "Job Duration by Worker"
351
322
  }
352
323
  },
353
- {
324
+ <% current_y += 6 %>
325
+
326
+ { <% "Delayed Jobs" %>
354
327
  "height": 3,
355
328
  "width": 6,
356
- "y": 24,
329
+ "y": <%= current_y %>,
357
330
  "x": 0,
358
331
  "type": "text",
359
332
  "properties": {
360
333
  "markdown": "\n# Delayed Jobs\n"
361
334
  }
362
335
  },
363
- {
336
+ { <% "Messages To Be Scheduled" %>
364
337
  "height": 3,
365
338
  "width": 3,
366
- "y": 0,
339
+ "y": <%= current_y %>,
367
340
  "x": 6,
368
341
  "type": "metric",
369
342
  "properties": {
370
343
  "metrics": [
371
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
344
+ [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
372
345
  ],
373
346
  "view": "singleValue",
374
347
  "region": "us-east-1",
375
- "stat": "Sum",
348
+ "stat": "Average",
376
349
  "period": 60,
377
- "title": "Messages Per Minute"
350
+ "title": "Messages To Be Scheduled"
378
351
  }
379
352
  },
380
- {
353
+ { <% "Delayed Job Scheduler Duration" %>
381
354
  "height": 3,
382
- "width": 3,
383
- "y": 9,
384
- "x": 6,
355
+ "width": 15,
356
+ "y": <%= current_y %>,
357
+ "x": 9,
385
358
  "type": "metric",
386
359
  "properties": {
387
360
  "metrics": [
388
- [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:service}-${self:custom.stage}-active-jobs", { "label": "Messages Per Minute" } ]
361
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", "Resource", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "label": "p10" } ],
362
+ [ "...", { "label": "p50", "stat": "p50" } ],
363
+ [ "...", { "label": "p99", "stat": "p99" } ],
364
+ [ "...", { "label": "Average", "stat": "Average" } ]
389
365
  ],
390
366
  "view": "singleValue",
391
367
  "region": "us-east-1",
392
- "stat": "Sum",
368
+ "stat": "p10",
393
369
  "period": 60,
394
- "title": "Messages Per Minute"
370
+ "title": "Handler Duration"
395
371
  }
396
372
  },
397
- {
373
+
374
+
375
+
376
+ <% current_y += 3 %>
377
+ { <% "Delayed Job SchedulerError count and success rate (%)" %>
398
378
  "height": 3,
399
- "width": 15,
400
- "y": 0,
401
- "x": 9,
379
+ "width": 6,
380
+ "y": <%= current_y %>,
381
+ "x": 18,
402
382
  "type": "metric",
403
383
  "properties": {
384
+ "period": 60,
404
385
  "metrics": [
405
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
406
- [ "...", { "label": "p50", "stat": "p50" } ],
407
- [ "...", { "label": "p99", "stat": "p99" } ],
408
- [ "...", { "label": "Average", "stat": "Average" } ]
386
+ [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
387
+ [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
388
+ [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
409
389
  ],
410
- "view": "singleValue",
411
390
  "region": "us-east-1",
412
- "stat": "p10",
413
- "period": 60,
414
- "title": "Handler Duration"
391
+ "title": "Delayed Job SchedulerError count and success rate (%)",
392
+ "yAxis": {
393
+ "right": {
394
+ "max": 100
395
+ }
396
+ },
397
+ "view": "timeSeries",
398
+ "stacked": false,
399
+ "liveData": true
415
400
  }
416
401
  },
417
- {
402
+ { <% "Delayed Job Scheduler Duration" %>
418
403
  "height": 3,
419
- "width": 15,
420
- "y": 9,
404
+ "width": 9,
405
+ "y": <%= current_y %>,
421
406
  "x": 9,
422
407
  "type": "metric",
423
408
  "properties": {
409
+ "period": 60,
424
410
  "metrics": [
425
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-active_job_handler", "Resource", "${self:service}-${self:provider.stage}-active_job_handler", { "label": "p10" } ],
426
- [ "...", { "label": "p50", "stat": "p50" } ],
427
- [ "...", { "label": "p99", "stat": "p99" } ],
428
- [ "...", { "label": "Average", "stat": "Average" } ]
411
+ [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Minimum" } ],
412
+ [ "...", { "stat": "Average" } ],
413
+ [ "...", { "stat": "Maximum" } ]
429
414
  ],
430
- "view": "singleValue",
431
415
  "region": "us-east-1",
432
- "stat": "p10",
433
- "period": 60,
434
- "title": "Handler Duration"
416
+ "title": "Delayed Job Scheduler Duration",
417
+ "view": "timeSeries",
418
+ "stacked": false,
419
+ "liveData": true
435
420
  }
436
421
  },
437
- {
422
+ { <% "Delayed Jobs To Be Executed In The Next 90 Seconds" %>
438
423
  "height": 3,
439
- "width": 3,
440
- "y": 24,
441
- "x": 6,
424
+ "width": 9,
425
+ "y": <%= current_y %>,
426
+ "x": 0,
442
427
  "type": "metric",
443
428
  "properties": {
444
429
  "metrics": [
445
430
  [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
446
431
  ],
447
- "view": "singleValue",
432
+ "view": "timeSeries",
433
+ "stacked": false,
448
434
  "region": "us-east-1",
449
- "stat": "Average",
435
+ "title": "Delayed Jobs To Be Executed In The Next 90 Seconds",
450
436
  "period": 60,
451
- "title": "Messages To Be Scheduled"
437
+ "stat": "Average",
438
+ "liveData": true
452
439
  }
453
440
  },
454
- {
441
+
442
+
443
+ <% current_y += 3 %>
444
+ { <% "Scheduled Job Table Read Capacity Units" %>
455
445
  "height": 3,
456
- "width": 15,
457
- "y": 24,
458
- "x": 9,
446
+ "width": 9,
447
+ "y": <%= current_y %>,
448
+ "x": 0,
459
449
  "type": "metric",
460
450
  "properties": {
461
451
  "metrics": [
462
- [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", "Resource", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "label": "p10" } ],
463
- [ "...", { "label": "p50", "stat": "p50" } ],
464
- [ "...", { "label": "p99", "stat": "p99" } ],
465
- [ "...", { "label": "Average", "stat": "Average" } ]
452
+ [ { "expression": "m2/PERIOD(m2)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
453
+ [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
454
+ [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
455
+ [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
466
456
  ],
467
- "view": "singleValue",
457
+ "view": "timeSeries",
458
+ "stacked": false,
468
459
  "region": "us-east-1",
469
- "stat": "p10",
460
+ "title": "Scheduled Job Table Read Capacity Units",
470
461
  "period": 60,
471
- "title": "Handler Duration"
462
+ "stat": "Sum",
463
+ "liveData": true
464
+ }
465
+ },
466
+ { <% "Delayd Job Schedule Concurrent executions" %>
467
+ "height": 3,
468
+ "width": 6,
469
+ "y": <%= current_y %>,
470
+ "x": 18,
471
+ "type": "metric",
472
+ "properties": {
473
+ "period": 60,
474
+ "metrics": [
475
+ [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Maximum" } ]
476
+ ],
477
+ "region": "us-east-1",
478
+ "title": "Delayd Job Schedule Concurrent executions",
479
+ "view": "timeSeries",
480
+ "stacked": false,
481
+ "liveData": true
472
482
  }
473
483
  },
474
- {
484
+ { <% "Scheduled Job Table Write Capacity Units" %>
475
485
  "height": 3,
476
486
  "width": 9,
477
- "y": 30,
487
+ "y": <%= current_y %>,
478
488
  "x": 9,
479
489
  "type": "metric",
480
490
  "properties": {
@@ -493,26 +503,53 @@ Resources:
493
503
  "liveData": true
494
504
  }
495
505
  },
496
- {
497
- "height": 6,
498
- "width": 24,
499
- "y": 18,
506
+
507
+
508
+
509
+ <% current_y += 3 %>
510
+ { <% "Scheduled Job Table Latency" %>
511
+ "height": 3,
512
+ "width": 18,
513
+ "y": <%= current_y %>,
500
514
  "x": 0,
501
515
  "type": "metric",
502
516
  "properties": {
503
517
  "metrics": [
504
- [ "rails-lambda-experiment", "Duration", "WorkerClassName", "HelloWorker" ],
505
- [ "...", { "stat": "p99" } ],
506
- [ "...", "HelloLaterWorker" ],
507
- [ "...", { "stat": "p99" } ]
518
+ [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "PutItem", { "yAxis": "left" } ],
519
+ [ "...", "Query" ],
520
+ [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
521
+ [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
522
+ [ ".", "ThrottledRequests", ".", ".", ".", ".", { "yAxis": "right", "visible": false } ]
508
523
  ],
509
524
  "view": "timeSeries",
510
525
  "stacked": false,
511
526
  "region": "us-east-1",
512
527
  "stat": "Average",
513
528
  "period": 60,
514
- "title": "Job Duration by Worker"
529
+ "title": "Scheduled Job Table Latency",
530
+ "liveData": true
531
+ }
532
+ },
533
+ { <% "Scheduled Job Table Throttled Operations" %>
534
+ "height": 3,
535
+ "width": 6,
536
+ "y": <%= current_y %>,
537
+ "x": 18,
538
+ "type": "metric",
539
+ "properties": {
540
+ "metrics": [
541
+ [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "DeleteItem" ],
542
+ [ "...", "PutItem" ]
543
+ ],
544
+ "view": "timeSeries",
545
+ "stacked": false,
546
+ "region": "us-east-1",
547
+ "stat": "Sum",
548
+ "period": 60,
549
+ "title": "Scheduled Job Table Throttled Operations",
550
+ "liveData": true
515
551
  }
516
552
  }
553
+
517
554
  ]
518
555
  }
@@ -5,12 +5,12 @@ runtime: ruby2.7
5
5
 
6
6
  # These values will be applied to any funktor handlers that don't specify custom values.
7
7
  # TODO - These still need to be wired up to do anything.
8
- handler_defaults:
8
+ handlerDefaults:
9
9
  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
10
10
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
11
11
  # expect to spend for one message. We default to a high number here to allow for the
12
12
  # times when things go weird.
13
- timeout: 900
13
+ timeout: 300
14
14
 
15
15
  # reservedConcurrency represents the maximum number of concurrent executions.
16
16
  # Usually you'll want to leave this as null so that handlers can scale infinitely
@@ -44,20 +44,17 @@ handler_defaults:
44
44
  logRetentionInDays: 30
45
45
 
46
46
  # Incoming Job Handler
47
- # TODO - These still need to be wired up to do anything.
48
- incoming_job_handler:
47
+ incomingJobHandler:
49
48
  # If your jobs are bursty AND time-sensitive you might want to have some lambdas pre-provisioned
50
49
  # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
51
50
  # provisionedConcurrency: 4
52
51
 
53
52
 
54
- # TODO - These still need to be wired up to do anything.
55
53
  queues:
56
54
  - default:
57
55
  # Set queue specific config options here
58
56
  # memorySize: 512
59
- # TODO - Is it advisable to use FIFO queuues with Funktor?
60
- # TODO - Maybe this isn't really even supported by CloudFormation?
57
+ # TODO - Is it advisable to use FIFO queuues with Funktor? Maybe this isn't really even supported by CloudFormation?
61
58
  # fifo: false
62
59
  - singleThread:
63
60
  reservedConcurrency: 1
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.2.2"
2
+ VERSION = "0.2.3"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-06-16 00:00:00.000000000 Z
11
+ date: 2021-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs