funktor 0.2.5 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/README.md +7 -28
- data/lib/funktor/cli/init.rb +1 -3
- data/lib/funktor/cli/templates/config/boot.rb +16 -0
- data/lib/funktor/cli/templates/config/environment.yml +1 -0
- data/lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml +4 -4
- data/lib/funktor/cli/templates/config/funktor.yml +6 -5
- data/lib/funktor/cli/templates/config/package.yml +1 -0
- data/lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml +9 -9
- data/lib/funktor/cli/templates/config/resources/incoming_job_queue.yml +2 -2
- data/lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb +3 -0
- data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb +1 -10
- data/lib/funktor/job.rb +1 -1
- data/lib/funktor/middleware/metrics.rb +1 -1
- data/lib/funktor/testing.rb +1 -1
- data/lib/funktor/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 16ad166a922c31ac7de1008d48fe34824203b0d325e7faae2f7ede140fc0f836
|
|
4
|
+
data.tar.gz: fafbd5ca0858ecb87c2c9e87c8efa52accc9262f922d6f3f1d4b311b3c8c470b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b40e4deee38ed584b93c863150d2d6c45dfc3aea1ac3ed0d7aa4d249c5661b527f252cf1b700545f54e80f98bf6158345c01b218f175eef117ce31626c5f5b21
|
|
7
|
+
data.tar.gz: 240839bae2964df499664ec8c21a44d3c8c4cf46d3d6df5cb798af95ba3b454deee6a41d5b4b0a3d71fc8c04e36bdfdb0d787f6168e71ef1cb516370e1c0d369
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
funktor (0.2.
|
|
4
|
+
funktor (0.2.10)
|
|
5
5
|
activesupport
|
|
6
6
|
aws-sdk-sqs (~> 1.37)
|
|
7
7
|
thor
|
|
@@ -18,7 +18,7 @@ GEM
|
|
|
18
18
|
addressable (2.7.0)
|
|
19
19
|
public_suffix (>= 2.0.2, < 5.0)
|
|
20
20
|
aws-eventstream (1.1.1)
|
|
21
|
-
aws-partitions (1.
|
|
21
|
+
aws-partitions (1.472.0)
|
|
22
22
|
aws-sdk-core (3.115.0)
|
|
23
23
|
aws-eventstream (~> 1, >= 1.0.2)
|
|
24
24
|
aws-partitions (~> 1, >= 1.239.0)
|
data/README.md
CHANGED
|
@@ -34,41 +34,20 @@ npm install -g serverless
|
|
|
34
34
|
|
|
35
35
|
Then you can initialize a new app by doing:
|
|
36
36
|
|
|
37
|
-
|
|
38
37
|
```bash
|
|
38
|
+
funktor bootstrap my-funktor-app
|
|
39
|
+
cd my-funktor-app
|
|
39
40
|
funktor init
|
|
40
41
|
```
|
|
41
42
|
|
|
43
|
+
Then you sh
|
|
44
|
+
|
|
42
45
|
This will create a `funktor` directory that is ready to deploy to AWS. If you've already configured
|
|
43
46
|
your aws tools via `~/.aws/credentials` you should be ready to deploy.
|
|
44
47
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
`funktor/resources` contains a few files that provision some AWS resources that are used by `funktor`.
|
|
50
|
-
* An SQS Queue for the "incoming jobs queue"
|
|
51
|
-
* A Dynamo DB table to allow queueing of jobs more than 15 minutes in the future (Funktor Pro)
|
|
52
|
-
* One or more SQS Queues for active jobs (currently there is only the default queue, support for additional queues is coming soon)
|
|
53
|
-
* An IAM User with permission to push jobs to the incoming jobs queue
|
|
54
|
-
* A CloudWatch dashboard to let you keep tabs on your application
|
|
55
|
-
|
|
56
|
-
`funktor/lambda_handlers` contains some scripts that receive events from Lambda, then invoke `funktor` to
|
|
57
|
-
do various things:
|
|
58
|
-
* `active_job_handler.rb` executes your jobs
|
|
59
|
-
* `delayed_job_scheduler.rb` (Funktor Pro) pulls delayed jobs out of DynamoDB and places them on the active job queue.
|
|
60
|
-
* `incoming_job_handler.rb` receives incoming jobs and pushes them to DynamoDB for delayed execution or to the active job queue as appropriate.
|
|
61
|
-
|
|
62
|
-
`funktor/function_definitions` contains details about hooking up the `lambda_handlers` to events.
|
|
63
|
-
|
|
64
|
-
`funktor/iam_permissions` contains some details about giving your lambda functions the appropriate permissions
|
|
65
|
-
to interact with SQS.
|
|
66
|
-
|
|
67
|
-
`funktor/workers` is where your workers will live.
|
|
68
|
-
|
|
69
|
-
`funktor/Gemfile` is the `Gemfile` that contains the gems that are needed for your workers to execute
|
|
70
|
-
jobs. This should be the minimal set of gems you can get away with so that cold start times remain reasonable.
|
|
71
|
-
This file will already contain `funktor`. (Don't remove it or `funktor` won't work!)
|
|
48
|
+
See the [wiki](https://github.com/Octo-Labs/funktor/wiki)
|
|
49
|
+
for more info, especially the
|
|
50
|
+
[section about getting started in a stand alone project](https://github.com/Octo-Labs/funktor/wiki/Getting-started-in-a-stand-alone-project).
|
|
72
51
|
|
|
73
52
|
## Deploying
|
|
74
53
|
|
data/lib/funktor/cli/init.rb
CHANGED
|
@@ -30,14 +30,13 @@ module Funktor
|
|
|
30
30
|
name
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
33
|
def funktor_config_yml
|
|
36
34
|
#template "funktor_config.yml", File.join("funktor_config.yml")
|
|
37
35
|
template File.join("config", "funktor.yml"), File.join("config", "funktor.yml")
|
|
38
36
|
template File.join("config", "ruby_layer.yml"), File.join("config", "ruby_layer.yml")
|
|
39
37
|
template File.join("config", "package.yml"), File.join("config", "package.yml")
|
|
40
38
|
template File.join("config", "environment.yml"), File.join("config", "environment.yml")
|
|
39
|
+
template File.join("config", "boot.rb"), File.join("config", "boot.rb")
|
|
41
40
|
end
|
|
42
41
|
|
|
43
42
|
def package_json
|
|
@@ -172,7 +171,6 @@ module Funktor
|
|
|
172
171
|
end
|
|
173
172
|
|
|
174
173
|
def queue_config_value(queue_name, config_name)
|
|
175
|
-
puts funktor_config
|
|
176
174
|
queue_config(queue_name)&.dig(config_name) ||
|
|
177
175
|
funktor_config.dig("handlerDefaults", config_name) ||
|
|
178
176
|
"null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Point at our Gemfile
|
|
2
|
+
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
|
|
3
|
+
|
|
4
|
+
# load rubygems & bundler
|
|
5
|
+
require "rubygems"
|
|
6
|
+
require 'bundler/setup'
|
|
7
|
+
|
|
8
|
+
# Set up gems listed in the Gemfile.
|
|
9
|
+
Bundler.require(:default, :production)
|
|
10
|
+
|
|
11
|
+
# Load all ruby files in the app directory
|
|
12
|
+
Dir.glob( File.join('..', 'app', '**', '*.rb'), base: File.dirname(__FILE__) ).each do |ruby_file|
|
|
13
|
+
puts "require_relative #{ruby_file}"
|
|
14
|
+
require_relative ruby_file
|
|
15
|
+
end
|
|
16
|
+
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
handler: lambda_event_handlers/incoming_job_handler.call
|
|
2
|
-
timeout: ${self:custom.funktor.
|
|
3
|
-
reservedConcurrency: ${self:custom.funktor.
|
|
4
|
-
provisionedConcurrency: ${self:custom.funktor.
|
|
5
|
-
memorySize: ${self:custom.funktor.
|
|
2
|
+
timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
|
|
3
|
+
reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
|
|
4
|
+
provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
|
|
5
|
+
memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
|
|
6
6
|
events:
|
|
7
7
|
- sqs:
|
|
8
8
|
arn:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
IncomingJobHandler:
|
|
2
2
|
# timeout is how long the handler can possibly run. Up to 10 messages may be delivered
|
|
3
3
|
# to a handler at one time, so you'll want this to be at least 10x the maximum time you
|
|
4
4
|
# expect to spend for one message. The incoming job handler usually will be pretty fast,
|
|
@@ -51,10 +51,11 @@ incomingJobHandler:
|
|
|
51
51
|
|
|
52
52
|
# You shouldn't need to mess with these under most circumstances. But you could if you want to change
|
|
53
53
|
# the name of some of your resources in AWS.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
IncomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
|
|
55
|
+
IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
|
|
56
|
+
IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
|
|
57
|
+
IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
|
|
58
|
+
DashboardName: ${self:service}-${self:custom.stage}-dashboard
|
|
58
59
|
<%- queue_names.each do |queue_name| -%>
|
|
59
60
|
<%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
|
|
60
61
|
<%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
|
|
@@ -2,7 +2,7 @@ Resources:
|
|
|
2
2
|
FunktorDashboard:
|
|
3
3
|
Type: AWS::CloudWatch::Dashboard
|
|
4
4
|
Properties:
|
|
5
|
-
DashboardName: ${self:custom.funktor.
|
|
5
|
+
DashboardName: ${self:custom.funktor.DashboardName}
|
|
6
6
|
DashboardBody: >
|
|
7
7
|
{
|
|
8
8
|
"widgets": [
|
|
@@ -25,7 +25,7 @@ Resources:
|
|
|
25
25
|
"type": "metric",
|
|
26
26
|
"properties": {
|
|
27
27
|
"metrics": [
|
|
28
|
-
[ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.
|
|
28
|
+
[ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Messages Per Minute" } ]
|
|
29
29
|
],
|
|
30
30
|
"view": "singleValue",
|
|
31
31
|
"region": "us-east-1",
|
|
@@ -42,7 +42,7 @@ Resources:
|
|
|
42
42
|
"type": "metric",
|
|
43
43
|
"properties": {
|
|
44
44
|
"metrics": [
|
|
45
|
-
[ "AWS/Lambda", "Duration", "FunctionName", "${self:
|
|
45
|
+
[ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "p10" } ],
|
|
46
46
|
[ "...", { "label": "p50", "stat": "p50" } ],
|
|
47
47
|
[ "...", { "label": "p99", "stat": "p99" } ],
|
|
48
48
|
[ "...", { "label": "Average", "stat": "Average" } ]
|
|
@@ -63,10 +63,10 @@ Resources:
|
|
|
63
63
|
"type": "metric",
|
|
64
64
|
"properties": {
|
|
65
65
|
"metrics": [
|
|
66
|
-
[ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.
|
|
66
|
+
[ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Received" } ],
|
|
67
67
|
[ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
|
|
68
|
-
[ "AWS/Lambda", "Invocations", "FunctionName", "${self:
|
|
69
|
-
[ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.
|
|
68
|
+
[ "AWS/Lambda", "Invocations", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "Handler Invocations" } ],
|
|
69
|
+
[ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Pending?" } ],
|
|
70
70
|
[ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
|
|
71
71
|
[ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
|
|
72
72
|
[ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
|
|
@@ -90,7 +90,7 @@ Resources:
|
|
|
90
90
|
"properties": {
|
|
91
91
|
"period": 60,
|
|
92
92
|
"metrics": [
|
|
93
|
-
[ "AWS/Lambda", "Duration", "FunctionName", "${self:
|
|
93
|
+
[ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Minimum" } ],
|
|
94
94
|
[ "...", { "stat": "Average" } ],
|
|
95
95
|
[ "...", { "stat": "Maximum" } ]
|
|
96
96
|
],
|
|
@@ -110,7 +110,7 @@ Resources:
|
|
|
110
110
|
"properties": {
|
|
111
111
|
"period": 60,
|
|
112
112
|
"metrics": [
|
|
113
|
-
[ "AWS/Lambda", "Errors", "FunctionName", "${self:
|
|
113
|
+
[ "AWS/Lambda", "Errors", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
|
|
114
114
|
[ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
|
|
115
115
|
[ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
|
|
116
116
|
],
|
|
@@ -137,7 +137,7 @@ Resources:
|
|
|
137
137
|
"properties": {
|
|
138
138
|
"period": 60,
|
|
139
139
|
"metrics": [
|
|
140
|
-
[ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:
|
|
140
|
+
[ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Maximum" } ]
|
|
141
141
|
],
|
|
142
142
|
"region": "us-east-1",
|
|
143
143
|
"title": "Incoming Job Handler Concurrent Executions",
|
|
@@ -2,7 +2,7 @@ Resources:
|
|
|
2
2
|
IncomingJobQueue:
|
|
3
3
|
Type: AWS::SQS::Queue
|
|
4
4
|
Properties:
|
|
5
|
-
QueueName: ${self:custom.funktor.
|
|
5
|
+
QueueName: ${self:custom.funktor.IncomingJobQueueName}
|
|
6
6
|
VisibilityTimeout: 300
|
|
7
7
|
RedrivePolicy:
|
|
8
8
|
deadLetterTargetArn:
|
|
@@ -11,7 +11,7 @@ Resources:
|
|
|
11
11
|
IncomingJobDeadLetterQueue:
|
|
12
12
|
Type: AWS::SQS::Queue
|
|
13
13
|
Properties:
|
|
14
|
-
QueueName: ${self:custom.funktor.
|
|
14
|
+
QueueName: ${self:custom.funktor.IncomingDeadJobQueueName}
|
|
15
15
|
|
|
16
16
|
Outputs:
|
|
17
17
|
IncomingJobQueueUrl:
|
|
@@ -1,13 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
# Bundler is hard to make work because AWS includes some gems in the basic ruby runtime.
|
|
4
|
-
# We're probably going to need to use containers...
|
|
5
|
-
#require "rubygems"
|
|
6
|
-
#require "bundler/setup"
|
|
7
|
-
#Bundler.require(:default)
|
|
8
|
-
|
|
9
|
-
# TODO : Ideally this wouldn't be needed
|
|
10
|
-
require_relative '../workers/hello_worker'
|
|
1
|
+
require_relative '../config/boot'
|
|
11
2
|
|
|
12
3
|
$handler = Funktor::ActiveJobHandler.new
|
|
13
4
|
|
data/lib/funktor/job.rb
CHANGED
|
@@ -19,7 +19,7 @@ module Funktor
|
|
|
19
19
|
"Timestamp": Time.now.strftime('%s%3N').to_i,
|
|
20
20
|
"CloudWatchMetrics": [
|
|
21
21
|
{
|
|
22
|
-
"Namespace":
|
|
22
|
+
"Namespace": ENV['FUNKTOR_APP_NAME'],
|
|
23
23
|
"Dimensions": [["WorkerClassName"]],
|
|
24
24
|
"Metrics": [ # CPU, Memory, Duration, etc...
|
|
25
25
|
{
|
data/lib/funktor/testing.rb
CHANGED
data/lib/funktor/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: funktor
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.10
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jeremy Green
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-06-
|
|
11
|
+
date: 2021-06-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: aws-sdk-sqs
|
|
@@ -116,6 +116,7 @@ files:
|
|
|
116
116
|
- lib/funktor/cli/init.rb
|
|
117
117
|
- lib/funktor/cli/templates/Gemfile
|
|
118
118
|
- lib/funktor/cli/templates/app/workers/hello_worker.rb
|
|
119
|
+
- lib/funktor/cli/templates/config/boot.rb
|
|
119
120
|
- lib/funktor/cli/templates/config/environment.yml
|
|
120
121
|
- lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml
|
|
121
122
|
- lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml
|