funktor 0.4.6 → 0.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/.tool-versions +2 -0
  3. data/Gemfile.lock +24 -5
  4. data/funktor-testapp/.envrc +1 -0
  5. data/funktor-testapp/.gitignore +7 -0
  6. data/funktor-testapp/Gemfile +25 -0
  7. data/funktor-testapp/Gemfile.lock +51 -0
  8. data/funktor-testapp/app/services/job_flood.rb +38 -0
  9. data/funktor-testapp/app/workers/audit_worker.rb +49 -0
  10. data/funktor-testapp/app/workers/greetings_worker.rb +3 -0
  11. data/funktor-testapp/app/workers/hello_worker.rb +18 -0
  12. data/funktor-testapp/app/workers/single_thread_audit_worker.rb +3 -0
  13. data/funktor-testapp/deploy-dev.sh +5 -0
  14. data/funktor-testapp/funktor_config/boot.rb +17 -0
  15. data/funktor-testapp/funktor_config/environment.yml +15 -0
  16. data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml +13 -0
  17. data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml +13 -0
  18. data/funktor-testapp/funktor_config/function_definitions/job_activator.yml +7 -0
  19. data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml +13 -0
  20. data/funktor-testapp/funktor_config/function_definitions/random_job_generator.yml +18 -0
  21. data/funktor-testapp/funktor_config/funktor.yml +114 -0
  22. data/funktor-testapp/funktor_config/iam_permissions/activity_table.yml +5 -0
  23. data/funktor-testapp/funktor_config/iam_permissions/default_queue.yml +8 -0
  24. data/funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml +8 -0
  25. data/funktor-testapp/funktor_config/iam_permissions/jobs_table.yml +5 -0
  26. data/funktor-testapp/funktor_config/iam_permissions/jobs_table_secondary_index.yml +8 -0
  27. data/funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml +8 -0
  28. data/funktor-testapp/funktor_config/iam_permissions/ssm.yml +5 -0
  29. data/funktor-testapp/funktor_config/package.yml +11 -0
  30. data/funktor-testapp/funktor_config/resources/activity_table.yml +22 -0
  31. data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml +809 -0
  32. data/funktor-testapp/funktor_config/resources/default_queue.yml +22 -0
  33. data/funktor-testapp/funktor_config/resources/incoming_job_queue.yml +22 -0
  34. data/funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml +26 -0
  35. data/funktor-testapp/funktor_config/resources/jobs_table.yml +56 -0
  36. data/funktor-testapp/funktor_config/resources/low_concurrency_queue.yml +22 -0
  37. data/funktor-testapp/funktor_config/ruby_layer.yml +11 -0
  38. data/funktor-testapp/funktor_init.yml +69 -0
  39. data/funktor-testapp/lambda_event_handlers/default_queue_handler.rb +8 -0
  40. data/funktor-testapp/lambda_event_handlers/incoming_job_handler.rb +8 -0
  41. data/funktor-testapp/lambda_event_handlers/job_activator.rb +8 -0
  42. data/funktor-testapp/lambda_event_handlers/low_concurrency_queue_handler.rb +8 -0
  43. data/funktor-testapp/lambda_event_handlers/random_job_generator.rb +35 -0
  44. data/funktor-testapp/package-lock.json +248 -0
  45. data/funktor-testapp/package.json +8 -0
  46. data/funktor-testapp/serverless.yml +66 -0
  47. data/funktor.gemspec +4 -1
  48. data/lib/active_job/queue_adapters/funktor_adapter.rb +3 -3
  49. data/lib/funktor/activity_tracker.rb +106 -0
  50. data/lib/funktor/cli/bootstrap.rb +0 -1
  51. data/lib/funktor/cli/init.rb +13 -0
  52. data/lib/funktor/cli/templates/app/workers/hello_worker.rb +1 -1
  53. data/lib/funktor/cli/templates/funktor_config/environment.yml +4 -0
  54. data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml +3 -1
  55. data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml +7 -0
  56. data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml +3 -1
  57. data/lib/funktor/cli/templates/funktor_config/funktor.yml +32 -6
  58. data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml +5 -0
  59. data/lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml +5 -0
  60. data/lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml +8 -0
  61. data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml +22 -0
  62. data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml +13 -12
  63. data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml +2 -2
  64. data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml +56 -0
  65. data/lib/funktor/cli/templates/funktor_config/resources/work_queue.yml +2 -2
  66. data/lib/funktor/cli/templates/funktor_init.yml.tt +16 -16
  67. data/lib/funktor/cli/templates/lambda_event_handlers/job_activator.rb +8 -0
  68. data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb +1 -1
  69. data/lib/funktor/cli/templates/serverless.yml +3 -2
  70. data/lib/funktor/counter.rb +4 -1
  71. data/lib/funktor/incoming_job_handler.rb +54 -18
  72. data/lib/funktor/job.rb +57 -7
  73. data/lib/funktor/job_activator.rb +124 -0
  74. data/lib/funktor/job_pusher.rb +0 -2
  75. data/lib/funktor/middleware/metrics.rb +8 -3
  76. data/lib/funktor/shard_utils.rb +6 -0
  77. data/lib/funktor/testing.rb +52 -29
  78. data/lib/funktor/version.rb +1 -1
  79. data/lib/funktor/web/application.rb +139 -0
  80. data/lib/funktor/web/views/index.erb +3 -0
  81. data/lib/funktor/web/views/layout.erb +58 -0
  82. data/lib/funktor/web/views/processing.erb +29 -0
  83. data/lib/funktor/web/views/queued.erb +29 -0
  84. data/lib/funktor/web/views/retries.erb +35 -0
  85. data/lib/funktor/web/views/scheduled.erb +26 -0
  86. data/lib/funktor/web/views/stats.erb +9 -0
  87. data/lib/funktor/web/views/table_stats_with_buttons.erb +11 -0
  88. data/lib/funktor/web.rb +1 -0
  89. data/lib/funktor/work_queue_handler.rb +101 -0
  90. data/lib/funktor/worker/funktor_options.rb +3 -1
  91. data/lib/funktor/worker.rb +8 -18
  92. data/lib/funktor.rb +52 -20
  93. metadata +109 -3
  94. data/lib/funktor/active_job_handler.rb +0 -58
@@ -0,0 +1,8 @@
1
+ {
2
+ "name": "funktor-testapp",
3
+ "description": "",
4
+ "version": "0.1.0",
5
+ "devDependencies": {
6
+ "serverless-ruby-layer": "^1.4.0"
7
+ }
8
+ }
@@ -0,0 +1,66 @@
1
+ # Welcome to Funktor & Serverless!
2
+ #
3
+ # This file is the main config file for your service.
4
+ # It's already configured to run Funktor, you just have to deploy it.
5
+ #
6
+ # For more info about Funktor check the wiki:
7
+ # https://github.com/Octo-Labs/funktor/wiki
8
+ #
9
+ # For more about serverless, check their docs:
10
+ # docs.serverless.com
11
+ #
12
+ # Happy Coding!
13
+
14
+ # The name of your service. All your AWS resources will contain this name.
15
+ service: funktor-testapp
16
+
17
+ # This causes serverless to throw an error early if the config is bad, instead of waiting for CloudFormation to try and fail to deploy it.
18
+ configValidationMode: error
19
+
20
+ # Pin the serverless framework to the 2.x line
21
+ frameworkVersion: '2'
22
+
23
+ provider:
24
+ name: aws
25
+ runtime: ruby2.7
26
+ lambdaHashingVersion: 20201221
27
+ environment: ${file(funktor_config/environment.yml)}
28
+ versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
29
+ logRetentionInDays: 7
30
+ iamRoleStatements:
31
+ - ${file(funktor_config/iam_permissions/activity_table.yml)}
32
+ - ${file(funktor_config/iam_permissions/default_queue.yml)}
33
+ - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
34
+ - ${file(funktor_config/iam_permissions/ssm.yml)}
35
+ - ${file(funktor_config/iam_permissions/low_concurrency_queue.yml)}
36
+ - ${file(funktor_config/iam_permissions/jobs_table.yml)}
37
+ - ${file(funktor_config/iam_permissions/jobs_table_secondary_index.yml)}
38
+
39
+
40
+ custom:
41
+ # Our stage is based on what is passed in when running serverless
42
+ # commands. Or fallsback to what we have set in the provider section.
43
+ stage: ${self:provider.stage, 'dev'}
44
+ funktor: ${file(funktor_config/funktor.yml)}
45
+ rubyLayer: ${file(funktor_config/ruby_layer.yml)}
46
+
47
+ package: ${file(funktor_config/package.yml)}
48
+
49
+ functions:
50
+ IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
51
+ DefaultQueueHandler: ${file(funktor_config/function_definitions/default_queue_handler.yml)}
52
+ JobActivator: ${file(funktor_config/function_definitions/job_activator.yml)}
53
+ LowConcurrencyQueueHandler: ${file(funktor_config/function_definitions/low_concurrency_queue_handler.yml)}
54
+ RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
55
+
56
+ resources:
57
+ - ${file(funktor_config/resources/activity_table.yml)}
58
+ - ${file(funktor_config/resources/incoming_job_queue_user.yml)}
59
+ - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
60
+ - ${file(funktor_config/resources/default_queue.yml)}
61
+ - ${file(funktor_config/resources/incoming_job_queue.yml)}
62
+ - ${file(funktor_config/resources/low_concurrency_queue.yml)}
63
+ - ${file(funktor_config/resources/jobs_table.yml)}
64
+
65
+ plugins:
66
+ - serverless-ruby-layer
data/funktor.gemspec CHANGED
@@ -30,11 +30,14 @@ Gem::Specification.new do |spec|
30
30
  spec.require_paths = ["lib"]
31
31
 
32
32
  spec.add_dependency 'aws-sdk-sqs', '~> 1.37'
33
+ spec.add_dependency 'aws-sdk-dynamodb', '~> 1.62'
33
34
  spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
34
- spec.add_dependency "thor" # Thor drives the CLI
35
+ spec.add_dependency "thor" # Thor drives the CLI TODO - should this just be a dev dependency?
35
36
 
36
37
  spec.add_development_dependency 'activejob', '>= 5.1.5'
37
38
  spec.add_development_dependency 'simplecov'
38
39
  spec.add_development_dependency 'webmock'
39
40
  spec.add_development_dependency 'pry-byebug'
41
+ spec.add_development_dependency 'sinatra'
42
+ spec.add_development_dependency 'timecop'
40
43
  end
@@ -39,11 +39,11 @@ module ActiveJob
39
39
  include Funktor::Worker
40
40
 
41
41
  def perform(job_data)
42
- puts "job_data = #{job_data.class} #{job_data}"
42
+ Funktor.logger.debug "job_data = #{job_data.class} #{job_data}"
43
43
  if job_data.is_a?(Array)
44
- puts "it's an array"
44
+ Funktor.logger.debug "it's an array"
45
45
  job_data = job_data.first
46
- puts "job_data = #{job_data.class} #{job_data}"
46
+ Funktor.logger.debug "job_data = #{job_data.class} #{job_data}"
47
47
  end
48
48
  Base.execute job_data
49
49
  end
@@ -0,0 +1,106 @@
1
+ require 'json'
2
+ require 'aws-sdk-dynamodb'
3
+
4
+ module Funktor
5
+ class ActivityTracker
6
+
7
+ INCR_KEYS = {
8
+ incoming: 'incoming',
9
+ queued: 'queued',
10
+ scheduled: 'scheduled',
11
+ processingStarted: 'processing',
12
+ processingComplete: 'complete',
13
+ processingFailed: 'failed',
14
+ bailingOut: 'failed',
15
+ retrying: 'retries',
16
+ retryActivated: 'queued',
17
+ scheduledJobActivated: nil,
18
+ scheduledJobDeleted: 'scheduledJobDeleted',
19
+ retryDeleted: 'retryDeleted'
20
+ #scheduledJobPushedToActive: 'active',
21
+ #activeJobPushed: 'active',
22
+ #scheduledJobPushed: 'scheduled'
23
+ }
24
+
25
+ DECR_KEYS = {
26
+ incoming: nil,
27
+ queued: nil,
28
+ scheduled: nil,
29
+ processingStarted: 'queued',
30
+ processingComplete: 'processing',
31
+ processingFailed: nil,
32
+ bailingOut: 'processing',
33
+ retrying: 'processing',
34
+ retryActivated: 'retries',
35
+ scheduledJobActivated: 'scheduled',
36
+ scheduledJobDeleted: 'scheduled',
37
+ retryDeleted: 'retries'
38
+ #scheduledJobPushedToActive: 'scheduled',
39
+ #activeJobPushed: nil,
40
+ #scheduledJobPushed: nil
41
+ }
42
+
43
+ def track(activity, job)
44
+ Funktor.logger.debug "starting track activity for #{activity}"
45
+ incrKey = nil || INCR_KEYS[activity.to_sym]
46
+ decrKey = nil || DECR_KEYS[activity.to_sym]
47
+ if incrKey
48
+ increment_key(incrKey)
49
+ end
50
+ if decrKey
51
+ decrement_key(decrKey)
52
+ end
53
+ Funktor.logger.debug "ending track activity for #{activity}"
54
+ end
55
+
56
+ def increment_key(key)
57
+ #put_metric_to_stdout(key, 1)
58
+ dynamodb_client.update_item({
59
+ table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
60
+ key: { category: 'stat', statName: key },
61
+ expression_attribute_values:{ ":start": 0, ":inc": 1 },
62
+ update_expression: "SET stat_value = if_not_exists(stat_value, :start) + :inc",
63
+ })
64
+ end
65
+
66
+ def decrement_key(key)
67
+ #put_metric_to_stdout(key, -1)
68
+ dynamodb_client.update_item({
69
+ table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
70
+ key: { category: 'stat', statName: key },
71
+ expression_attribute_values:{ ":start": 0, ":inc": 1 },
72
+ update_expression: "SET stat_value = if_not_exists(stat_value, :start) - :inc",
73
+ })
74
+ end
75
+
76
+ def dynamodb_client
77
+ @dynamodb_client ||= ::Aws::DynamoDB::Client.new
78
+ end
79
+
80
+ def put_metric_to_stdout(key, value)
81
+ data = {
82
+ "_aws": {
83
+ "Timestamp": Time.now.strftime('%s%3N').to_i,
84
+ "CloudWatchMetrics": [
85
+ {
86
+ "Namespace": "rails-lambda-experiment",
87
+ "Dimensions": [["functionVersion"]],
88
+ "Metrics": [ # CPU, Memory, Duration, etc...
89
+ {
90
+ "Name": key,
91
+ "Unit": "Count"
92
+ }
93
+ ]
94
+ }
95
+ ]
96
+ },
97
+ "functionVersion": "LATEST",
98
+ #"count": value,
99
+ #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
100
+ }
101
+ data[key] = value
102
+ puts(data.to_json)
103
+ end
104
+
105
+ end
106
+ end
@@ -24,7 +24,6 @@ module Funktor
24
24
 
25
25
  def funktor_yml
26
26
  # TODO - Should we camelize the app name before writing it into the config? (CloudFormation names get weird with underscores and dashes.)
27
- puts funktor_file_target
28
27
  template "templates/funktor_init.yml", funktor_file_target
29
28
  end
30
29
 
@@ -65,6 +65,8 @@ module Funktor
65
65
  @work_queue_config = queue_details.values.first
66
66
  template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
67
67
  end
68
+ template File.join("funktor_config", "resources", "jobs_table.yml"), File.join("funktor_config", "resources", "jobs_table.yml")
69
+ template File.join("funktor_config", "resources", "activity_table.yml"), File.join("funktor_config", "resources", "activity_table.yml")
68
70
  end
69
71
 
70
72
  def iam_permissions
@@ -75,10 +77,14 @@ module Funktor
75
77
  @work_queue_config = queue_details.values.first
76
78
  template File.join("funktor_config", "iam_permissions", "work_queue.yml"), File.join("funktor_config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
77
79
  end
80
+ template File.join("funktor_config", "iam_permissions", "jobs_table.yml"), File.join("funktor_config", "iam_permissions", "jobs_table.yml")
81
+ template File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml"), File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml")
82
+ template File.join("funktor_config", "iam_permissions", "activity_table.yml"), File.join("funktor_config", "iam_permissions", "activity_table.yml")
78
83
  end
79
84
 
80
85
  def function_definitions
81
86
  template File.join("funktor_config", "function_definitions", "incoming_job_handler.yml"), File.join("funktor_config", "function_definitions", "incoming_job_handler.yml")
87
+ template File.join("funktor_config", "function_definitions", "job_activator.yml"), File.join("funktor_config", "function_definitions", "job_activator.yml")
82
88
  queues.each do |queue_details|
83
89
  @work_queue_name = queue_details.keys.first
84
90
  @work_queue_config = queue_details.values.first
@@ -88,6 +94,7 @@ module Funktor
88
94
 
89
95
  def lambda_handlers
90
96
  template File.join("lambda_event_handlers", "incoming_job_handler.rb"), File.join("lambda_event_handlers", "incoming_job_handler.rb")
97
+ template File.join("lambda_event_handlers", "job_activator.rb"), File.join("lambda_event_handlers", "job_activator.rb")
91
98
  queues.each do |queue_details|
92
99
  @work_queue_name = queue_details.keys.first
93
100
  @work_queue_config = queue_details.values.first
@@ -170,6 +177,12 @@ module Funktor
170
177
  "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
171
178
  end
172
179
 
180
+ def activator_config_value(config_name)
181
+ funktor_config.dig("jobActivator", config_name) ||
182
+ funktor_config.dig("handlerDefaults", config_name) ||
183
+ "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
184
+ end
185
+
173
186
  def queue_config_value(queue_name, config_name)
174
187
  queue_config(queue_name)&.dig(config_name) ||
175
188
  funktor_config.dig("handlerDefaults", config_name) ||
@@ -2,7 +2,7 @@ class HelloWorker
2
2
  include Funktor::Worker
3
3
 
4
4
  def perform(*args)
5
- puts "Greetings from the HelloWorker!"
5
+ Funktor.logger.debug "Greetings from the HelloWorker!"
6
6
  end
7
7
  end
8
8
 
@@ -9,3 +9,7 @@ FUNKTOR_INCOMING_JOB_QUEUE:
9
9
  FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
10
10
  Ref: <%= queue_name.camelize %>Queue
11
11
  <%- end -%>
12
+ FUNKTOR_ACTIVITY_TABLE:
13
+ Ref: ActivityTable
14
+ FUNKTOR_JOBS_TABLE:
15
+ Ref: JobsTable
@@ -1,10 +1,12 @@
1
1
  handler: lambda_event_handlers/incoming_job_handler.call
2
- timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
2
+ timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
3
3
  reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
4
4
  provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
5
5
  memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
6
6
  events:
7
7
  - sqs:
8
+ batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}
9
+ maximumBatchingWindow: ${self:custom.funktor.IncomingJobHandler.maximumBatchingWindow, 0}
8
10
  arn:
9
11
  Fn::GetAtt:
10
12
  - IncomingJobQueue
@@ -0,0 +1,7 @@
1
+ handler: lambda_event_handlers/job_activator.call
2
+ timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
3
+ reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
4
+ provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
5
+ memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
6
+ events:
7
+ - schedule: rate(1 minute)
@@ -1,10 +1,12 @@
1
1
  handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
2
- timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.timeout, 900}
2
+ timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
3
3
  reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
4
4
  provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
5
5
  memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
6
6
  events:
7
7
  - sqs:
8
+ batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}
9
+ maximumBatchingWindow: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.maximumBatchingWindow, 0}
8
10
  arn:
9
11
  Fn::GetAtt:
10
12
  - <%= work_queue_name.camelize %>Queue
@@ -3,7 +3,7 @@ IncomingJobHandler:
3
3
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
4
4
  # expect to spend for one message. The incoming job handler usually will be pretty fast,
5
5
  # but we default to a high number here to allow for the times when things go weird.
6
- timeout: <%= incoming_config_value 'timeout' %>
6
+ functionTimeout: <%= incoming_config_value 'functionTimeout' %>
7
7
  # reservedConcurrency represents the maximum number of concurrent executions.
8
8
  # For the incoming job handler you probably don't want to limit it because you
9
9
  # want to get things onto work queues as quickly as possible.
@@ -19,7 +19,32 @@ IncomingJobHandler:
19
19
  batchSize: <%= incoming_config_value 'batchSize' %>
20
20
  maximumBatchingWindow: <%= incoming_config_value 'maximumBatchingWindow' %>
21
21
  visibilityTimeout: <%= incoming_config_value 'visibilityTimeout' %>
22
- logRetentionInDays: <%= incoming_config_value 'logRetentionInDays' %>
22
+ maxReceiveCount: <%= incoming_config_value 'maxReceiveCount' %>
23
+
24
+ JobActivator:
25
+ # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
26
+ # to a handler at one time, so you'll want this to be at least 10x the maximum time you
27
+ # expect to spend for one message. The job activator usually will be pretty fast,
28
+ # but we default to a high number here to allow for the times when things go weird.
29
+ functionTimeout: <%= activator_config_value 'functionTimeout' %>
30
+ # reservedConcurrency represents the maximum number of concurrent executions.
31
+ # For the job activator you probably don't want to limit it because you
32
+ # want to get things onto work queues as quickly as possible when they're ready.
33
+ reservedConcurrency: <%= activator_config_value 'reservedConcurrency' %>
34
+ # provisionedConcurrency represents the number of lambda functions that will always
35
+ # be available. For the job activator you probably don't need to set this
36
+ # since it will be running on a schedule
37
+ provisionedConcurrency: <%= activator_config_value 'provisionedConcurrency' %>
38
+ # Use memory_size to adjust the reousrces (both memory and CPU) available.
39
+ # For the job activator you probably don't need this to be too large,
40
+ # but if you're seeing large delays at this stage it might help to bump it up.
41
+ memorySize: <%= activator_config_value 'memorySize' %>
42
+ batchSize: <%= activator_config_value 'batchSize' %>
43
+ maximumBatchingWindow: <%= activator_config_value 'maximumBatchingWindow' %>
44
+ visibilityTimeout: <%= activator_config_value 'visibilityTimeout' %>
45
+ maxReceiveCount: <%= activator_config_value 'maxReceiveCount' %>
46
+
47
+
23
48
 
24
49
  <%- queue_names.each do |queue_name| -%>
25
50
  <%= queue_name.camelize %>QueueHandler:
@@ -27,7 +52,7 @@ IncomingJobHandler:
27
52
  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
28
53
  # expect to spend for one message. The active job handler may be slow if your jobs are
29
54
  # doing a lot of work, so we default to the maximum here.
30
- timeout: <%= queue_config_value queue_name, 'timeout' %>
55
+ functionTimeout: <%= queue_config_value queue_name, 'functionTimeout' %>
31
56
  # reservedConcurrency represents the maximum number of concurrent executions.
32
57
  # For the active job handler you may want to limit it if you have resource limitations
33
58
  # like database connections that you need to avoid exhausting.
@@ -44,7 +69,7 @@ IncomingJobHandler:
44
69
  batchSize: <%= queue_config_value queue_name, 'batchSize' %>
45
70
  maximumBatchingWindow: <%= queue_config_value queue_name, 'maximumBatchingWindow' %>
46
71
  visibilityTimeout: <%= queue_config_value queue_name, 'visibilityTimeout' %>
47
- logRetentionInDays: <%= queue_config_value queue_name, 'logRetentionInDays' %>
72
+ maxReceiveCount: <%= queue_config_value queue_name, 'maxReceiveCount' %>
48
73
 
49
74
  <%- end -%>
50
75
 
@@ -61,5 +86,6 @@ DashboardName: ${self:service}-${self:custom.stage}-dashboard
61
86
  <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
62
87
  <%= queue_name.camelize %>QueueHandlerName: ${self:service}-${self:custom.stage}-<%= queue_name.camelize %>QueueHandler
63
88
  <%- end -%>
64
-
65
-
89
+ JobsTableName: ${self:service}-${self:custom.stage}-jobs
90
+ ActivityTableName: ${self:service}-${self:custom.stage}-activity
91
+ JobActivatorName: ${self:service}-${self:custom.stage}-JobActivator
@@ -0,0 +1,5 @@
1
+ Effect: Allow
2
+ Action:
3
+ - dynamodb:*
4
+ Resource:
5
+ - "Fn::GetAtt": [ ActivityTable, Arn ]
@@ -0,0 +1,5 @@
1
+ Effect: Allow
2
+ Action:
3
+ - dynamodb:*
4
+ Resource:
5
+ - "Fn::GetAtt": [ JobsTable, Arn ]
@@ -0,0 +1,8 @@
1
+ Effect: Allow
2
+ Action:
3
+ - dynamodb:Query
4
+ Resource:
5
+ Fn::Join:
6
+ - ""
7
+ - - "Fn::GetAtt": [ JobsTable, Arn ]
8
+ - "/index/performAtIndex"
@@ -0,0 +1,22 @@
1
+ Resources:
2
+ ActivityTable:
3
+ Type: AWS::DynamoDB::Table
4
+ Properties:
5
+ # Generate a name based on the stage
6
+ TableName: ${self:custom.funktor.ActivityTableName}
7
+ BillingMode: PAY_PER_REQUEST
8
+ AttributeDefinitions:
9
+ - AttributeName: category
10
+ AttributeType: S
11
+ - AttributeName: statName
12
+ AttributeType: S
13
+ KeySchema:
14
+ - AttributeName: category
15
+ KeyType: HASH
16
+ - AttributeName: statName
17
+ KeyType: RANGE
18
+
19
+ Outputs:
20
+ ActivityTable:
21
+ Value:
22
+ Ref: ActivityTable