funktor 0.7.1 → 0.7.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +9 -9
- data/README.md +0 -2
- data/funktor-testapp/app/workers/audit_worker.rb +5 -1
- data/funktor-testapp/funktor_config/funktor.yml +1 -0
- data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml +12 -12
- data/funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml +8 -0
- data/lib/funktor/activity_tracker.rb +5 -1
- data/lib/funktor/cli/init.rb +34 -3
- data/lib/funktor/cli/templates/funktor_config/funktor.yml +1 -0
- data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml +4 -4
- data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml +8 -0
- data/lib/funktor/cli/templates/package.json +5 -5
- data/lib/funktor/counter.rb +5 -1
- data/lib/funktor/middleware/metrics.rb +5 -1
- data/lib/funktor/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6e7890531c38acfc5bc6f24550ee96f1df4ce78bea2f329300dcd70f4bb3e357
|
|
4
|
+
data.tar.gz: a46dbd83e9ab5521868053b737f2e5b047bef2a4012041a12072e1d5b19cf034
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3f108fea2135fd1675f62f4b211baa330207ca7be525851e141fe069d2ef042d17afeb88b3cabd4708917cf39cb9178df0634b4b7e05d36ed21ea6198118839c
|
|
7
|
+
data.tar.gz: a077b5c7d2429cfeb5a8fe2ff27fac22afbf129c29ce9ea35db091da0992a914e2d5b381c59ab6dd5d7919a2875996d594f4063b721aa6811777df7227d26807
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
funktor (0.7.
|
|
4
|
+
funktor (0.7.3)
|
|
5
5
|
activesupport
|
|
6
6
|
aws-sdk-dynamodb (~> 1.62)
|
|
7
7
|
aws-sdk-sqs (~> 1.37)
|
|
@@ -21,20 +21,20 @@ GEM
|
|
|
21
21
|
zeitwerk (~> 2.3)
|
|
22
22
|
addressable (2.8.0)
|
|
23
23
|
public_suffix (>= 2.0.2, < 5.0)
|
|
24
|
-
aws-eventstream (1.
|
|
25
|
-
aws-partitions (1.
|
|
26
|
-
aws-sdk-core (3.
|
|
24
|
+
aws-eventstream (1.2.0)
|
|
25
|
+
aws-partitions (1.494.0)
|
|
26
|
+
aws-sdk-core (3.120.0)
|
|
27
27
|
aws-eventstream (~> 1, >= 1.0.2)
|
|
28
28
|
aws-partitions (~> 1, >= 1.239.0)
|
|
29
29
|
aws-sigv4 (~> 1.1)
|
|
30
30
|
jmespath (~> 1.0)
|
|
31
|
-
aws-sdk-dynamodb (1.
|
|
32
|
-
aws-sdk-core (~> 3, >= 3.
|
|
31
|
+
aws-sdk-dynamodb (1.63.0)
|
|
32
|
+
aws-sdk-core (~> 3, >= 3.120.0)
|
|
33
33
|
aws-sigv4 (~> 1.1)
|
|
34
|
-
aws-sdk-sqs (1.
|
|
35
|
-
aws-sdk-core (~> 3, >= 3.
|
|
34
|
+
aws-sdk-sqs (1.44.0)
|
|
35
|
+
aws-sdk-core (~> 3, >= 3.120.0)
|
|
36
36
|
aws-sigv4 (~> 1.1)
|
|
37
|
-
aws-sigv4 (1.
|
|
37
|
+
aws-sigv4 (1.3.0)
|
|
38
38
|
aws-eventstream (~> 1, >= 1.0.2)
|
|
39
39
|
byebug (11.1.3)
|
|
40
40
|
coderay (1.1.3)
|
data/README.md
CHANGED
|
@@ -21,13 +21,17 @@ class AuditWorker
|
|
|
21
21
|
puts "So long from the #{self.class.name}, and thanks for all the fish!"
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
def metric_namespace
|
|
25
|
+
[ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
|
|
26
|
+
end
|
|
27
|
+
|
|
24
28
|
def metric_hash(time_diff)
|
|
25
29
|
{
|
|
26
30
|
"_aws": {
|
|
27
31
|
"Timestamp": Time.now.strftime('%s%3N').to_i,
|
|
28
32
|
"CloudWatchMetrics": [
|
|
29
33
|
{
|
|
30
|
-
"Namespace":
|
|
34
|
+
"Namespace": metric_namespace,
|
|
31
35
|
"Dimensions": [["WorkerClassName"]],
|
|
32
36
|
"Metrics": [ # CPU, Memory, Duration, etc...
|
|
33
37
|
{
|
|
@@ -103,6 +103,7 @@ IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
|
|
|
103
103
|
IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
|
|
104
104
|
IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
|
|
105
105
|
DashboardName: ${self:service}-${self:custom.stage}-dashboard
|
|
106
|
+
DashboardNamespace: ${self:service}-${self:custom.stage}
|
|
106
107
|
DefaultQueueName: ${self:service}-${self:custom.stage}-default
|
|
107
108
|
DefaultDeadJobQueueName: ${self:service}-${self:custom.stage}-default-dead
|
|
108
109
|
DefaultQueueHandlerName: ${self:service}-${self:custom.stage}-DefaultQueueHandler
|
|
@@ -29,13 +29,13 @@ Resources:
|
|
|
29
29
|
"type": "metric",
|
|
30
30
|
"properties": {
|
|
31
31
|
"metrics": [
|
|
32
|
-
[ "funktor
|
|
32
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "AuditWorker" ],
|
|
33
33
|
[ "...", { "stat": "p99" } ],
|
|
34
|
-
[ "funktor
|
|
34
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "GreetingsWorker" ],
|
|
35
35
|
[ "...", { "stat": "p99" } ],
|
|
36
|
-
[ "funktor
|
|
36
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "HelloWorker" ],
|
|
37
37
|
[ "...", { "stat": "p99" } ],
|
|
38
|
-
[ "funktor
|
|
38
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "SingleThreadAuditWorker" ],
|
|
39
39
|
[ "...", { "stat": "p99" } ]
|
|
40
40
|
],
|
|
41
41
|
"view": "timeSeries",
|
|
@@ -55,13 +55,13 @@ Resources:
|
|
|
55
55
|
"type": "metric",
|
|
56
56
|
"properties": {
|
|
57
57
|
"metrics": [
|
|
58
|
-
[ "funktor
|
|
58
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "AuditWorker" ],
|
|
59
59
|
[ ".", "failed", ".", "." ],
|
|
60
|
-
[ "funktor
|
|
60
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "GreetingsWorker" ],
|
|
61
61
|
[ ".", "failed", ".", "." ],
|
|
62
|
-
[ "funktor
|
|
62
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "HelloWorker" ],
|
|
63
63
|
[ ".", "failed", ".", "." ],
|
|
64
|
-
[ "funktor
|
|
64
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "SingleThreadAuditWorker" ],
|
|
65
65
|
[ ".", "failed", ".", "." ]
|
|
66
66
|
],
|
|
67
67
|
"view": "timeSeries",
|
|
@@ -81,9 +81,9 @@ Resources:
|
|
|
81
81
|
"type": "metric",
|
|
82
82
|
"properties": {
|
|
83
83
|
"metrics": [
|
|
84
|
-
[ "funktor
|
|
84
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "Queue", "default" ],
|
|
85
85
|
[ "...", { "stat": "p99" } ],
|
|
86
|
-
[ "funktor
|
|
86
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "Queue", "low_concurrency" ],
|
|
87
87
|
[ "...", { "stat": "p99" } ]
|
|
88
88
|
],
|
|
89
89
|
"view": "timeSeries",
|
|
@@ -102,9 +102,9 @@ Resources:
|
|
|
102
102
|
"type": "metric",
|
|
103
103
|
"properties": {
|
|
104
104
|
"metrics": [
|
|
105
|
-
[ "funktor
|
|
105
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "Queue", "default" ],
|
|
106
106
|
[ ".", "failed", ".", "." ],
|
|
107
|
-
[ "funktor
|
|
107
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "Queue", "low_concurrency" ],
|
|
108
108
|
[ ".", "failed", ".", "." ]
|
|
109
109
|
],
|
|
110
110
|
"view": "timeSeries",
|
|
@@ -17,6 +17,14 @@ Resources:
|
|
|
17
17
|
- dynamodb:*
|
|
18
18
|
Resource:
|
|
19
19
|
- "Fn::GetAtt": [ JobsTable, Arn ]
|
|
20
|
+
- Effect: Allow
|
|
21
|
+
Action:
|
|
22
|
+
- dynamodb:Query
|
|
23
|
+
Resource:
|
|
24
|
+
Fn::Join:
|
|
25
|
+
- ""
|
|
26
|
+
- - "Fn::GetAtt": [ JobsTable, Arn ]
|
|
27
|
+
- "/index/categoryIndex"
|
|
20
28
|
|
|
21
29
|
IncomingJobQueueUserAccessKey:
|
|
22
30
|
Type: AWS::IAM::AccessKey
|
|
@@ -77,13 +77,17 @@ module Funktor
|
|
|
77
77
|
@dynamodb_client ||= ::Aws::DynamoDB::Client.new
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
+
def metric_namespace
|
|
81
|
+
[ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
|
|
82
|
+
end
|
|
83
|
+
|
|
80
84
|
def put_metric_to_stdout(key, value)
|
|
81
85
|
data = {
|
|
82
86
|
"_aws": {
|
|
83
87
|
"Timestamp": Time.now.strftime('%s%3N').to_i,
|
|
84
88
|
"CloudWatchMetrics": [
|
|
85
89
|
{
|
|
86
|
-
"Namespace":
|
|
90
|
+
"Namespace": metric_namespace,
|
|
87
91
|
"Dimensions": [["functionVersion"]],
|
|
88
92
|
"Metrics": [ # CPU, Memory, Duration, etc...
|
|
89
93
|
{
|
data/lib/funktor/cli/init.rb
CHANGED
|
@@ -40,15 +40,46 @@ module Funktor
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
def package_json
|
|
43
|
-
|
|
43
|
+
if File.exist?("package.json")
|
|
44
|
+
package_data = File.open("package.json").read
|
|
45
|
+
if package_data =~ /serverless-ruby-layer/
|
|
46
|
+
say "serverless-ruby-layer is already installed in package.json"
|
|
47
|
+
else
|
|
48
|
+
if File.exist?("package-lock.json")
|
|
49
|
+
run "npm install serverless-ruby-layer@1.4.0 --save-dev"
|
|
50
|
+
# TODO - Add handers for yarn and what not
|
|
51
|
+
elsif File.exist?("yarn.lock")
|
|
52
|
+
run "yarn add --dev serverless-ruby-layer@1.4.0"
|
|
53
|
+
else
|
|
54
|
+
say "You should install serverless-ruby-layer version 1.4.0 using yor package manager of choice."
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
else
|
|
58
|
+
template "package.json", File.join("package.json")
|
|
59
|
+
end
|
|
44
60
|
end
|
|
45
61
|
|
|
46
62
|
def gemfile
|
|
47
|
-
|
|
63
|
+
if File.exist?("Gemfile")
|
|
64
|
+
gem_data = File.open("Gemfile").read
|
|
65
|
+
if gem_data =~ /funktor/
|
|
66
|
+
say "funktor is already installed in Gemfile"
|
|
67
|
+
else
|
|
68
|
+
run "bundle add funktor"
|
|
69
|
+
end
|
|
70
|
+
else
|
|
71
|
+
template "Gemfile", File.join("Gemfile")
|
|
72
|
+
end
|
|
48
73
|
end
|
|
49
74
|
|
|
50
75
|
def gitignore
|
|
51
|
-
|
|
76
|
+
# TODO clean this up so that we have a single source of truth for .gitignor stuff
|
|
77
|
+
if File.exist?(".gitignore")
|
|
78
|
+
append_to_file ".gitignore", ".serverless"
|
|
79
|
+
append_to_file ".gitignore", "node_modules"
|
|
80
|
+
else
|
|
81
|
+
template "gitignore", File.join(".gitignore")
|
|
82
|
+
end
|
|
52
83
|
end
|
|
53
84
|
|
|
54
85
|
def workers
|
|
@@ -81,6 +81,7 @@ IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
|
|
|
81
81
|
IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
|
|
82
82
|
IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
|
|
83
83
|
DashboardName: ${self:service}-${self:custom.stage}-dashboard
|
|
84
|
+
DashboardNamespace: ${self:service}-${self:custom.stage}
|
|
84
85
|
<%- queue_names.each do |queue_name| -%>
|
|
85
86
|
<%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
|
|
86
87
|
<%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
|
|
@@ -30,7 +30,7 @@ Resources:
|
|
|
30
30
|
"properties": {
|
|
31
31
|
"metrics": [
|
|
32
32
|
<%- app_worker_names.each do |worker_name| -%>
|
|
33
|
-
[ "
|
|
33
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ],
|
|
34
34
|
[ "...", { "stat": "p99" } ]<%= worker_name == app_worker_names.last ? "" : "," %>
|
|
35
35
|
<%- end -%>
|
|
36
36
|
],
|
|
@@ -52,7 +52,7 @@ Resources:
|
|
|
52
52
|
"properties": {
|
|
53
53
|
"metrics": [
|
|
54
54
|
<%- app_worker_names.each do |worker_name| -%>
|
|
55
|
-
[ "
|
|
55
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ],
|
|
56
56
|
[ ".", "failed", ".", "." ]<%= worker_name == app_worker_names.last ? "" : "," %>
|
|
57
57
|
<%- end -%>
|
|
58
58
|
],
|
|
@@ -74,7 +74,7 @@ Resources:
|
|
|
74
74
|
"properties": {
|
|
75
75
|
"metrics": [
|
|
76
76
|
<%- queue_names.each do |queue_name| -%>
|
|
77
|
-
[ "
|
|
77
|
+
[ "${self:custom.funktor.DashboardNamespace}", "Duration", "Queue", "<%= queue_name.underscore %>" ],
|
|
78
78
|
[ "...", { "stat": "p99" } ]<%= queue_name == queue_names.last ? "" : "," %>
|
|
79
79
|
<%- end -%>
|
|
80
80
|
],
|
|
@@ -95,7 +95,7 @@ Resources:
|
|
|
95
95
|
"properties": {
|
|
96
96
|
"metrics": [
|
|
97
97
|
<%- queue_names.each do |queue_name| -%>
|
|
98
|
-
[ "
|
|
98
|
+
[ "${self:custom.funktor.DashboardNamespace}", "processed", "Queue", "<%= queue_name.underscore %>" ],
|
|
99
99
|
[ ".", "failed", ".", "." ]<%= queue_name == queue_names.last ? "" : "," %>
|
|
100
100
|
<%- end -%>
|
|
101
101
|
],
|
|
@@ -17,6 +17,14 @@ Resources:
|
|
|
17
17
|
- dynamodb:*
|
|
18
18
|
Resource:
|
|
19
19
|
- "Fn::GetAtt": [ JobsTable, Arn ]
|
|
20
|
+
- Effect: Allow
|
|
21
|
+
Action:
|
|
22
|
+
- dynamodb:Query
|
|
23
|
+
Resource:
|
|
24
|
+
Fn::Join:
|
|
25
|
+
- ""
|
|
26
|
+
- - "Fn::GetAtt": [ JobsTable, Arn ]
|
|
27
|
+
- "/index/categoryIndex"
|
|
20
28
|
|
|
21
29
|
IncomingJobQueueUserAccessKey:
|
|
22
30
|
Type: AWS::IAM::AccessKey
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
|
-
"name":"<%= name %>",
|
|
3
|
-
"description":"",
|
|
4
|
-
"version":"0.1.0",
|
|
5
|
-
"devDependencies":{
|
|
6
|
-
"serverless-ruby-layer":"^1.4.0"
|
|
2
|
+
"name": "<%= name %>",
|
|
3
|
+
"description": "",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"devDependencies": {
|
|
6
|
+
"serverless-ruby-layer": "^1.4.0"
|
|
7
7
|
}
|
|
8
8
|
}
|
data/lib/funktor/counter.rb
CHANGED
|
@@ -17,13 +17,17 @@ module Funktor
|
|
|
17
17
|
Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(job))
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
+
def metric_namespace
|
|
21
|
+
[ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
|
|
22
|
+
end
|
|
23
|
+
|
|
20
24
|
def metric_hash(job)
|
|
21
25
|
{
|
|
22
26
|
"_aws": {
|
|
23
27
|
"Timestamp": Time.now.strftime('%s%3N').to_i,
|
|
24
28
|
"CloudWatchMetrics": [
|
|
25
29
|
{
|
|
26
|
-
"Namespace":
|
|
30
|
+
"Namespace": metric_namespace,
|
|
27
31
|
"Dimensions": [["WorkerClassName"], ["Queue"]],
|
|
28
32
|
"Metrics": [ # CPU, Memory, Duration, etc...
|
|
29
33
|
{
|
|
@@ -18,13 +18,17 @@ module Funktor
|
|
|
18
18
|
Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(time_diff, job))
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
+
def metric_namespace
|
|
22
|
+
[ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
|
|
23
|
+
end
|
|
24
|
+
|
|
21
25
|
def metric_hash(time_diff_in_seconds, job)
|
|
22
26
|
{
|
|
23
27
|
"_aws": {
|
|
24
28
|
"Timestamp": Time.now.strftime('%s%3N').to_i,
|
|
25
29
|
"CloudWatchMetrics": [
|
|
26
30
|
{
|
|
27
|
-
"Namespace":
|
|
31
|
+
"Namespace": metric_namespace,
|
|
28
32
|
"Dimensions": [["WorkerClassName"], ["Queue"]],
|
|
29
33
|
"Metrics": [ # CPU, Memory, Duration, etc...
|
|
30
34
|
{
|
data/lib/funktor/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: funktor
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.7.
|
|
4
|
+
version: 0.7.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jeremy Green
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-05-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: aws-sdk-sqs
|