funktor 0.5.0 → 0.6.3

data/funktor-testapp/funktor_config/resources/default_queue.yml

@@ -3,11 +3,11 @@ Resources:
     Type: AWS::SQS::Queue
     Properties:
       QueueName: ${self:custom.funktor.DefaultQueueName}
-      VisibilityTimeout: 300
+      VisibilityTimeout: ${self:custom.funktor.DefaultQueueHandler.visibilityTimeout}
       RedrivePolicy:
         deadLetterTargetArn:
           "Fn::GetAtt": [ DefaultDeadLetterQueue, Arn ]
-        maxReceiveCount: 5
+        maxReceiveCount: ${self:custom.funktor.DefaultQueueHandler.maxReceiveCount}
   DefaultDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:

data/funktor-testapp/funktor_config/resources/incoming_job_queue.yml

@@ -3,11 +3,11 @@ Resources:
     Type: AWS::SQS::Queue
     Properties:
       QueueName: ${self:custom.funktor.IncomingJobQueueName}
-      VisibilityTimeout: 300
+      VisibilityTimeout: ${self:custom.funktor.IncomingJobHandler.visibilityTimeout}
       RedrivePolicy:
         deadLetterTargetArn:
           "Fn::GetAtt": [ IncomingJobDeadLetterQueue, Arn ]
-        maxReceiveCount: 5
+        maxReceiveCount: ${self:custom.funktor.IncomingJobHandler.maxReceiveCount}
   IncomingJobDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:

data/funktor-testapp/funktor_config/resources/jobs_table.yml

@@ -8,11 +8,11 @@ Resources:
           AttributeType: N
         - AttributeName: jobId
           AttributeType: S
-        #- AttributeName: category
-          #AttributeType: S
+        - AttributeName: category
+          AttributeType: S
         - AttributeName: performAt
           AttributeType: S
-        - AttributeName: dummy
+        - AttributeName: queueable
           AttributeType: S
       KeySchema:
         - AttributeName: jobShard
@@ -22,7 +22,19 @@ Resources:
       GlobalSecondaryIndexes:
         - IndexName: performAtIndex
           KeySchema:
-          - AttributeName: dummy
+          - AttributeName: queueable
+            KeyType: HASH
+          - AttributeName: performAt
+            KeyType: RANGE
+          Projection:
+            NonKeyAttributes:
+            - jobId
+            - payload
+            - category
+            ProjectionType: INCLUDE
+        - IndexName: categoryIndex
+          KeySchema:
+          - AttributeName: category
             KeyType: HASH
           - AttributeName: performAt
             KeyType: RANGE

data/funktor-testapp/funktor_config/resources/low_concurrency_queue.yml

@@ -0,0 +1,22 @@
+Resources:
+  LowConcurrencyQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.LowConcurrencyQueueName}
+      VisibilityTimeout: ${self:custom.funktor.LowConcurrencyQueueHandler.visibilityTimeout}
+      RedrivePolicy:
+        deadLetterTargetArn:
+          "Fn::GetAtt": [ LowConcurrencyDeadLetterQueue, Arn ]
+        maxReceiveCount: ${self:custom.funktor.LowConcurrencyQueueHandler.maxReceiveCount}
+  LowConcurrencyDeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.LowConcurrencyDeadJobQueueName}
+
+Outputs:
+  LowConcurrencyQueueUrl:
+    Value:
+      Ref: LowConcurrencyQueue
+  LowConcurrencyDeadLetterQueueUrl:
+    Value:
+      Ref: LowConcurrencyDeadLetterQueue

data/funktor-testapp/funktor_init.yml

@@ -10,7 +10,7 @@ handlerDefaults:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. We default to a high number here to allow for the
   # times when things go weird.
-  timeout: 300
+  functionTimeout: 300
 
   # reservedConcurrency represents the maximum number of concurrent executions.
   # Usually you'll want to leave this as null so that handlers can scale infinitely
@@ -29,19 +29,21 @@ handlerDefaults:
   memorySize: 256
 
   # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
-  batchSize: 10
+  batchSize: 1
 
   # How many seconds should AWS wait for a batch to fill up before executing lambda?
   # For immediate execution set the batch size to 1.
-  maximumBatchingWindow : 1
+  maximumBatchingWindow : 0
 
   # Visibility timeout should only come into play in the case of Funktor errors.
   # Application level errors should be handled by Funktor retry mechanisms.
   # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
-  visibilityTimeout: 900
+  visibilityTimeout: 1800
 
-  # Set log rentention to save money
-  logRetentionInDays: 30
+  # Max recieve count affects how many times a job will retry that has been throttled at the SQS -> Lambda boundary.
+  # Amazon recommend this be at least 5, but I've found that higher numbers are better to avoid legit jobs ending
+  # up in the dead letter queue
+  maxReceiveCount: 20
 
 # Incoming Job Handler
 incomingJobHandler:
@@ -49,6 +51,8 @@ incomingJobHandler:
   # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
   # provisionedConcurrency: 4
 
+jobActivator:
+  # You probably don't need to adjust the defaults for this one.
 
 queues:
   - default:
@@ -56,6 +60,10 @@ queues:
       # memorySize: 512
       # TODO - Is it advisable to use FIFO queuues with Funktor? Maybe this isn't really even supported by CloudFormation?
       # fifo: false
-  - singleThread:
-      reservedConcurrency: 1
+  - lowConcurrency:
+      # BEWARE - Setting very low concurrency values (5 or lower) can contribute to "SQS Overpull", so you probably don't
+      # want to have any queues with extremely low concurrency.
+      # Details about "SQS Overpull" can be found in this article:
+      # https://zaccharles.medium.com/reproducing-the-sqs-trigger-and-lambda-concurrency-limit-issue-f4c09d384a18
+      reservedConcurrency: 10
 

data/funktor-testapp/serverless.yml

@@ -26,12 +26,13 @@ provider:
   lambdaHashingVersion: 20201221
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
+  logRetentionInDays: 7
   iamRoleStatements:
     - ${file(funktor_config/iam_permissions/activity_table.yml)}
     - ${file(funktor_config/iam_permissions/default_queue.yml)}
     - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
     - ${file(funktor_config/iam_permissions/ssm.yml)}
-    - ${file(funktor_config/iam_permissions/single_thread_queue.yml)}
+    - ${file(funktor_config/iam_permissions/low_concurrency_queue.yml)}
     - ${file(funktor_config/iam_permissions/jobs_table.yml)}
     - ${file(funktor_config/iam_permissions/jobs_table_secondary_index.yml)}
 
@@ -49,7 +50,7 @@ functions:
   IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
   DefaultQueueHandler: ${file(funktor_config/function_definitions/default_queue_handler.yml)}
   JobActivator: ${file(funktor_config/function_definitions/job_activator.yml)}
-  SingleThreadQueueHandler: ${file(funktor_config/function_definitions/single_thread_queue_handler.yml)}
+  LowConcurrencyQueueHandler: ${file(funktor_config/function_definitions/low_concurrency_queue_handler.yml)}
   RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
 
 resources:
@@ -58,7 +59,7 @@ resources:
   - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
   - ${file(funktor_config/resources/default_queue.yml)}
   - ${file(funktor_config/resources/incoming_job_queue.yml)}
-  - ${file(funktor_config/resources/single_thread_queue.yml)}
+  - ${file(funktor_config/resources/low_concurrency_queue.yml)}
   - ${file(funktor_config/resources/jobs_table.yml)}
 
 plugins:

data/funktor.gemspec

@@ -32,10 +32,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-sqs', '~> 1.37'
   spec.add_dependency 'aws-sdk-dynamodb', '~> 1.62'
   spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
-  spec.add_dependency "thor" # Thor drives the CLI
+  spec.add_dependency "thor" # Thor drives the CLI TODO - should this just be a dev dependency?
 
   spec.add_development_dependency 'activejob', '>= 5.1.5'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'webmock'
   spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'sinatra'
+  spec.add_development_dependency 'timecop'
 end

data/lib/funktor/activity_tracker.rb

@@ -14,7 +14,9 @@ module Funktor
       bailingOut: 'failed',
       retrying: 'retries',
       retryActivated: 'queued',
-      scheduledJobActivated: 'queued'
+      scheduledJobActivated: nil,
+      scheduledJobDeleted: 'scheduledJobDeleted',
+      retryDeleted: 'retryDeleted'
       #scheduledJobPushedToActive: 'active',
       #activeJobPushed: 'active',
       #scheduledJobPushed: 'scheduled'
@@ -30,7 +32,9 @@ module Funktor
       bailingOut: 'processing',
       retrying: 'processing',
       retryActivated: 'retries',
-      scheduledJobActivated: 'scheduled'
+      scheduledJobActivated: 'scheduled',
+      scheduledJobDeleted: 'scheduled',
+      retryDeleted: 'retries'
       #scheduledJobPushedToActive: 'scheduled',
       #activeJobPushed: nil,
       #scheduledJobPushed: nil

data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,10 +1,12 @@
 handler: lambda_event_handlers/incoming_job_handler.call
-timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
+timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
 events:
   - sqs:
+      batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}
+      maximumBatchingWindow: ${self:custom.funktor.IncomingJobHandler.maximumBatchingWindow, 0}
       arn:
         Fn::GetAtt:
           - IncomingJobQueue

data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml

@@ -1,8 +1,7 @@
 handler: lambda_event_handlers/job_activator.call
-timeout: ${self:custom.funktor.JobActivator.timeout, 30}
+timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
-#reservedConcurrency: 1
 events:
   - schedule: rate(1 minute)

data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml

@@ -1,10 +1,12 @@
 handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
-timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.timeout, 900}
+timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
 events:
   - sqs:
+      batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}
+      maximumBatchingWindow: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.maximumBatchingWindow, 0}
       arn:
         Fn::GetAtt:
           - <%= work_queue_name.camelize %>Queue

data/lib/funktor/cli/templates/funktor_config/funktor.yml

@@ -3,7 +3,7 @@ IncomingJobHandler:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
   # but we default to a high number here to allow for the times when things go weird.
-  timeout: <%= incoming_config_value 'timeout' %>
+  functionTimeout: <%= incoming_config_value 'functionTimeout' %>
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the incoming job handler you probably don't want to limit it because you
   # want to get things onto work queues as quickly as possible.
@@ -19,14 +19,14 @@ IncomingJobHandler:
   batchSize: <%= incoming_config_value 'batchSize' %>
   maximumBatchingWindow: <%= incoming_config_value 'maximumBatchingWindow' %>
   visibilityTimeout: <%= incoming_config_value 'visibilityTimeout' %>
-  logRetentionInDays: <%= incoming_config_value 'logRetentionInDays' %>
+  maxReceiveCount: <%= incoming_config_value 'maxReceiveCount' %>
 
 JobActivator:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The job activator usually will be pretty fast,
   # but we default to a high number here to allow for the times when things go weird.
-  timeout: <%= activator_config_value 'timeout' %>
+  functionTimeout: <%= activator_config_value 'functionTimeout' %>
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the job activator you probably don't want to limit it because you
   # want to get things onto work queues as quickly as possible when they're ready.
@@ -42,7 +42,7 @@ JobActivator:
   batchSize: <%= activator_config_value 'batchSize' %>
   maximumBatchingWindow: <%= activator_config_value 'maximumBatchingWindow' %>
   visibilityTimeout: <%= activator_config_value 'visibilityTimeout' %>
-  logRetentionInDays: <%= activator_config_value 'logRetentionInDays' %>
+  maxReceiveCount: <%= activator_config_value 'maxReceiveCount' %>
 
 
 
@@ -52,7 +52,7 @@ JobActivator:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The active job handler may be slow if your jobs are
   # doing a lot of work, so we default to the maximum here.
-  timeout: <%= queue_config_value queue_name, 'timeout' %>
+  functionTimeout: <%= queue_config_value queue_name, 'functionTimeout' %>
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the active job handler you may want to limit it if you have resource limitations
   # like database connections that you need to avoid exhausting.
@@ -69,7 +69,7 @@ JobActivator:
   batchSize: <%= queue_config_value queue_name, 'batchSize' %>
   maximumBatchingWindow: <%= queue_config_value queue_name, 'maximumBatchingWindow' %>
   visibilityTimeout: <%= queue_config_value queue_name, 'visibilityTimeout' %>
-  logRetentionInDays: <%= queue_config_value queue_name, 'logRetentionInDays' %>
+  maxReceiveCount: <%= queue_config_value queue_name, 'maxReceiveCount' %>
 
 <%- end -%>
 

data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml

@@ -74,7 +74,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- queue_names.each do |queue_name| -%>
-                            [ "<%= app_name %>", "Duration", "Queue", "<%= queue_name %>" ],
+                            [ "<%= app_name %>", "Duration", "Queue", "<%= queue_name.underscore %>" ],
                             [ "...", { "stat": "p99" } ]<%= queue_name == queue_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -95,7 +95,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- queue_names.each do |queue_name| -%>
-                            [ "<%= app_name %>", "processed", "Queue", "<%= queue_name %>" ],
+                            [ "<%= app_name %>", "processed", "Queue", "<%= queue_name.underscore %>" ],
                             [ ".", "failed", ".", "." ]<%= queue_name == queue_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -616,6 +616,7 @@ Resources:
                             [ "...", "Query" ],
                             [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
                             [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
+                            [ ".", "SuccessfulRequestLatency", ".", ".", ".", "UpdateItem" ],
                             [ ".", "ThrottledRequests", ".", ".", ".", ".", { "yAxis": "right", "visible": false } ]
                         ],
                         "view": "timeSeries",

data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml

@@ -3,11 +3,11 @@ Resources:
     Type: AWS::SQS::Queue
     Properties:
       QueueName: ${self:custom.funktor.IncomingJobQueueName}
-      VisibilityTimeout: 300
+      VisibilityTimeout: ${self:custom.funktor.IncomingJobHandler.visibilityTimeout}
       RedrivePolicy:
         deadLetterTargetArn:
           "Fn::GetAtt": [ IncomingJobDeadLetterQueue, Arn ]
-        maxReceiveCount: 5
+        maxReceiveCount: ${self:custom.funktor.IncomingJobHandler.maxReceiveCount}
   IncomingJobDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:

data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml

@@ -8,11 +8,11 @@ Resources:
           AttributeType: N
         - AttributeName: jobId
           AttributeType: S
-        #- AttributeName: category
-          #AttributeType: S
+        - AttributeName: category
+          AttributeType: S
         - AttributeName: performAt
           AttributeType: S
-        - AttributeName: dummy
+        - AttributeName: queueable
           AttributeType: S
       KeySchema:
         - AttributeName: jobShard
@@ -22,7 +22,19 @@ Resources:
       GlobalSecondaryIndexes:
         - IndexName: performAtIndex
           KeySchema:
-          - AttributeName: dummy
+          - AttributeName: queueable
+            KeyType: HASH
+          - AttributeName: performAt
+            KeyType: RANGE
+          Projection:
+            NonKeyAttributes:
+            - jobId
+            - payload
+            - category
+            ProjectionType: INCLUDE
+        - IndexName: categoryIndex
+          KeySchema:
+          - AttributeName: category
             KeyType: HASH
           - AttributeName: performAt
             KeyType: RANGE

data/lib/funktor/cli/templates/funktor_config/resources/work_queue.yml

@@ -3,11 +3,11 @@ Resources:
     Type: AWS::SQS::Queue
     Properties:
       QueueName: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueName}
-      VisibilityTimeout: 300
+      VisibilityTimeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.visibilityTimeout}
       RedrivePolicy:
         deadLetterTargetArn:
           "Fn::GetAtt": [ <%= work_queue_name.camelize %>DeadLetterQueue, Arn ]
-        maxReceiveCount: 5
+        maxReceiveCount: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.maxReceiveCount}
   <%= work_queue_name.camelize %>DeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:

data/lib/funktor/cli/templates/funktor_init.yml.tt

@@ -10,7 +10,7 @@ handlerDefaults:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. We default to a high number here to allow for the
   # times when things go weird.
-  timeout: 300
+  functionTimeout: 300
 
   # reservedConcurrency represents the maximum number of concurrent executions.
   # Usually you'll want to leave this as null so that handlers can scale infinitely
@@ -29,19 +29,21 @@ handlerDefaults:
   memorySize: 256
 
   # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
-  batchSize: 10
+  batchSize: 1
 
   # How many seconds should AWS wait for a batch to fill up before executing lambda?
   # For immediate execution set the batch size to 1.
-  maximumBatchingWindow : 1
+  maximumBatchingWindow : 0
 
   # Visibility timeout should only come into play in the case of Funktor errors.
   # Application level errors should be handled by Funktor retry mechanisms.
   # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
-  visibilityTimeout: 900
+  visibilityTimeout: 1800
 
-  # Set log rentention to save money
-  logRetentionInDays: 30
+  # Max recieve count affects how many times a job will retry that has been throttled at the SQS -> Lambda boundary.
+  # Amazon recommend this be at least 5, but I've found that higher numbers are better to avoid legit jobs ending
+  # up in the dead letter queue
+  maxReceiveCount: 20
 
 # Incoming Job Handler
 incomingJobHandler:
@@ -58,6 +60,10 @@ queues:
       # memorySize: 512
       # TODO - Is it advisable to use FIFO queuues with Funktor? Maybe this isn't really even supported by CloudFormation?
       # fifo: false
-  - singleThread:
-      reservedConcurrency: 1
+  - lowConcurrency:
+      # BEWARE - Setting very low concurrency values (5 or lower) can contribute to "SQS Overpull", so you probably don't
+      # want to have any queues with extremely low concurrency.
+      # Details about "SQS Overpull" can be found in this article:
+      # https://zaccharles.medium.com/reproducing-the-sqs-trigger-and-lambda-concurrency-limit-issue-f4c09d384a18
+      reservedConcurrency: 10