funktor 0.4.5 → 0.6.0

data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml

@@ -74,7 +74,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- queue_names.each do |queue_name| -%>
-                            [ "<%= app_name %>", "Duration", "Queue", "<%= queue_name %>" ],
+                            [ "<%= app_name %>", "Duration", "Queue", "<%= queue_name.underscore %>" ],
                             [ "...", { "stat": "p99" } ]<%= queue_name == queue_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -95,7 +95,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- queue_names.each do |queue_name| -%>
-                            [ "<%= app_name %>", "processed", "Queue", "<%= queue_name %>" ],
+                            [ "<%= app_name %>", "processed", "Queue", "<%= queue_name.underscore %>" ],
                             [ ".", "failed", ".", "." ]<%= queue_name == queue_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -438,7 +438,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
+                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "Query" ]
                         ],
                         "view": "singleValue",
                         "region": "us-east-1",
@@ -455,7 +455,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", "Resource", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "label": "p10" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.JobActivatorName}", "Resource", "${self:custom.funktor.JobActivatorName}", { "label": "p10" } ],
                             [ "...", { "label": "p50", "stat": "p50" } ],
                             [ "...", { "label": "p99", "stat": "p99" } ],
                             [ "...", { "label": "Average", "stat": "Average" } ]
@@ -480,7 +480,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
+                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:custom.funktor.JobActivatorName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
                             [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
                             [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
                         ],
@@ -505,7 +505,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Minimum" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.JobActivatorName}", { "stat": "Minimum" } ],
                             [ "...", { "stat": "Average" } ],
                             [ "...", { "stat": "Maximum" } ]
                         ],
@@ -524,7 +524,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
+                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "Query" ]
                         ],
                         "view": "timeSeries",
                         "stacked": false,
@@ -547,7 +547,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             [ { "expression": "m2/PERIOD(m2)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
-                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
+                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:custom.funktor.JobsTableName}", { "id": "m2", "visible": false } ],
                             [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
                             [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
                         ],
@@ -569,7 +569,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Maximum" } ]
+                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:custom.funktor.JobActivatorName}", { "stat": "Maximum" } ]
                         ],
                         "region": "us-east-1",
                         "title": "Delayd Job Schedule Concurrent executions",
@@ -587,7 +587,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             [ { "expression": "m4/PERIOD(m4)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
-                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
+                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:custom.funktor.JobsTableName}", { "id": "m2", "visible": false } ],
                             [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
                             [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
                         ],
@@ -612,10 +612,11 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "PutItem", { "yAxis": "left" } ],
+                            [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "PutItem", { "yAxis": "left" } ],
                             [ "...", "Query" ],
                             [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
                             [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
+                            [ ".", "SuccessfulRequestLatency", ".", ".", ".", "UpdateItem" ],
                             [ ".", "ThrottledRequests", ".", ".", ".", ".", { "yAxis": "right", "visible": false } ]
                         ],
                         "view": "timeSeries",
@@ -635,7 +636,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "DeleteItem" ],
+                            [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "DeleteItem" ],
                             [ "...", "PutItem" ]
                         ],
                         "view": "timeSeries",

data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml

@@ -3,11 +3,11 @@ Resources:
     Type: AWS::SQS::Queue
     Properties:
       QueueName: ${self:custom.funktor.IncomingJobQueueName}
-      VisibilityTimeout: 300
+      VisibilityTimeout: ${self:custom.funktor.IncomingJobHandler.visibilityTimeout}
       RedrivePolicy:
         deadLetterTargetArn:
           "Fn::GetAtt": [ IncomingJobDeadLetterQueue, Arn ]
-        maxReceiveCount: 5
+        maxReceiveCount: ${self:custom.funktor.IncomingJobHandler.maxReceiveCount}
   IncomingJobDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:

data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml

@@ -0,0 +1,56 @@
+Resources:
+  JobsTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: ${self:custom.funktor.JobsTableName}
+      AttributeDefinitions:
+        - AttributeName: jobShard
+          AttributeType: N
+        - AttributeName: jobId
+          AttributeType: S
+        - AttributeName: category
+          AttributeType: S
+        - AttributeName: performAt
+          AttributeType: S
+        - AttributeName: queueable
+          AttributeType: S
+      KeySchema:
+        - AttributeName: jobShard
+          KeyType: HASH
+        - AttributeName: jobId
+          KeyType: RANGE
+      GlobalSecondaryIndexes:
+        - IndexName: performAtIndex
+          KeySchema:
+          - AttributeName: queueable
+            KeyType: HASH
+          - AttributeName: performAt
+            KeyType: RANGE
+          Projection:
+            NonKeyAttributes:
+            - jobId
+            - payload
+            - category
+            ProjectionType: INCLUDE
+        - IndexName: categoryIndex
+          KeySchema:
+          - AttributeName: category
+            KeyType: HASH
+          - AttributeName: performAt
+            KeyType: RANGE
+          Projection:
+            NonKeyAttributes:
+            - jobId
+            - payload
+            - category
+            ProjectionType: INCLUDE
+      BillingMode: PAY_PER_REQUEST
+      # Set the capacity based on the stage
+      #ProvisionedThroughput:
+        #ReadCapacityUnits: ${self:custom.tableThroughput}
+        #WriteCapacityUnits: ${self:custom.tableThroughput}
+
+Outputs:
+  JobsTable:
+    Value:
+      Ref: JobsTable

data/lib/funktor/cli/templates/funktor_config/resources/work_queue.yml

@@ -3,11 +3,11 @@ Resources:
     Type: AWS::SQS::Queue
     Properties:
       QueueName: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueName}
-      VisibilityTimeout: 300
+      VisibilityTimeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.visibilityTimeout}
       RedrivePolicy:
         deadLetterTargetArn:
           "Fn::GetAtt": [ <%= work_queue_name.camelize %>DeadLetterQueue, Arn ]
-        maxReceiveCount: 5
+        maxReceiveCount: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.maxReceiveCount}
   <%= work_queue_name.camelize %>DeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:

data/lib/funktor/cli/templates/funktor_init.yml.tt

@@ -10,7 +10,7 @@ handlerDefaults:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. We default to a high number here to allow for the
   # times when things go weird.
-  timeout: 300
+  functionTimeout: 300
 
   # reservedConcurrency represents the maximum number of concurrent executions.
   # Usually you'll want to leave this as null so that handlers can scale infinitely
@@ -29,19 +29,21 @@ handlerDefaults:
   memorySize: 256
 
   # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
-  batchSize: 10
+  batchSize: 1
 
   # How many seconds should AWS wait for a batch to fill up before executing lambda?
   # For immediate execution set the batch size to 1.
-  maximumBatchingWindow : 1
+  maximumBatchingWindow : 0
 
   # Visibility timeout should only come into play in the case of Funktor errors.
   # Application level errors should be handled by Funktor retry mechanisms.
   # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
-  visibilityTimeout: 900
+  visibilityTimeout: 1800
 
-  # Set log rentention to save money
-  logRetentionInDays: 30
+  # Max recieve count affects how many times a job will retry that has been throttled at the SQS -> Lambda boundary.
+  # Amazon recommend this be at least 5, but I've found that higher numbers are better to avoid legit jobs ending
+  # up in the dead letter queue
+  maxReceiveCount: 20
 
 # Incoming Job Handler
 incomingJobHandler:
@@ -49,6 +51,8 @@ incomingJobHandler:
   # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
   # provisionedConcurrency: 4
 
+jobActivator:
+  # You probably don't need to adjust the defaults for this one.
 
 queues:
   - default:
@@ -56,14 +60,10 @@ queues:
       # memorySize: 512
       # TODO - Is it advisable to use FIFO queuues with Funktor? Maybe this isn't really even supported by CloudFormation?
       # fifo: false
-  - singleThread:
-      reservedConcurrency: 1
+  - lowConcurrency:
+      # BEWARE - Setting very low concurrency values (5 or lower) can contribute to "SQS Overpull", so you probably don't
+      # want to have any queues with extremely low concurrency.
+      # Details about "SQS Overpull" can be found in this article:
+      # https://zaccharles.medium.com/reproducing-the-sqs-trigger-and-lambda-concurrency-limit-issue-f4c09d384a18
+      reservedConcurrency: 10
 
-# TODO - Maybe this shouldn't be surfaced this early?
-# TODO - This still needs to be wired up to do anything.
-package:
-  patterns:
-    - Gemfile
-    - Gemfile.lock
-    - app/**
-    - funktor_config/**

data/lib/funktor/cli/templates/lambda_event_handlers/job_activator.rb

@@ -0,0 +1,8 @@
+require_relative '../funktor_config/boot'
+
+$handler = Funktor::JobActivator.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb

@@ -1,6 +1,6 @@
 require_relative '../funktor_config/boot'
 
-$handler = Funktor::ActiveJobHandler.new
+$handler = Funktor::WorkQueueHandler.new
 
 def call(event:, context:)
   $handler.call(event: event, context: context)

data/lib/funktor/cli/templates/serverless.yml

@@ -3,8 +3,8 @@
 # This file is the main config file for your service.
 # It's already configured to run Funktor, you just have to deploy it.
 #
-# For more info about Funktor:
-#    TODO
+# For more info about Funktor check the wiki:
+#    https://github.com/Octo-Labs/funktor/wiki
 #
 # For more about serverless, check their docs:
 #    docs.serverless.com
@@ -26,6 +26,7 @@ provider:
   lambdaHashingVersion: 20201221
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
+  logRetentionInDays: 7
   iamRoleStatements:
     <%- all_iam_permissions.each do |iam_permission| -%>
     - ${file(<%= iam_permission %>)}

data/lib/funktor/counter.rb

@@ -11,7 +11,10 @@ module Funktor
     end
 
     def put_metric_to_stdout(job)
-      puts Funktor.dump_json(metric_hash(job))
+      # NOTE : We use raw puts here instead of Funktor.logger.something to avoid getting extra
+      # timestamps or log level information in the log line. We need this specific format to
+      # be the only thing in the line so that CloudWatch can parse the logs and use the data.
+      Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(job))
     end
 
     def metric_hash(job)

data/lib/funktor/incoming_job_handler.rb

@@ -1,12 +1,17 @@
 require 'aws-sdk-sqs'
+require 'aws-sdk-dynamodb'
 require 'active_support/core_ext/string/inflections'
 
 module Funktor
   class IncomingJobHandler
 
+    def initialize
+      @tracker = Funktor::ActivityTracker.new
+    end
+
     def call(event:, context:)
       event = Funktor::Aws::Sqs::Event.new(event)
-      puts "event.jobs.count = #{event.jobs.count}"
+      Funktor.logger.debug "event.jobs.count = #{event.jobs.count}"
       event.jobs.each do |job|
         dispatch(job)
       end
@@ -16,32 +21,63 @@ module Funktor
       @sqs_client ||= ::Aws::SQS::Client.new
     end
 
+    def dynamodb_client
+      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+    end
+
     def dispatch(job)
       Funktor.incoming_job_handler_middleware.invoke(job) do
-        puts "pushing to active_job_queue for delay = #{job.delay}"
-        push_to_active_job_queue(job)
+        # TODO : This number should be configurable via ENV var
+        if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
+          Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
+          # We push to the jobs table first becauase the work queue handler will expect to be able
+          # to update the stats of a record that's already in the table.
+          # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
+          # work queue handler to be ok with a job that's not yet in the table?
+          push_to_jobs_table(job, "queued")
+          push_to_work_queue(job)
+          if job.is_retry?
+            @tracker.track(:retryActivated, job)
+          else
+            @tracker.track(:queued, job)
+          end
+        else
+          Funktor.logger.debug "pushing to jobs table for delay = #{job.delay}"
+          push_to_jobs_table(job, nil)
+          if job.is_retry?
+            # do nothing for tracking
+          else
+            @tracker.track(:scheduled, job)
+          end
+        end
+        @tracker.track(:incoming, job)
       end
     end
 
-    def active_job_queue
-      ENV['FUNKTOR_ACTIVE_JOB_QUEUE']
+    def push_to_work_queue(job)
+      Funktor.logger.debug "job = #{job.to_json}"
+      sqs_client.send_message({
+        queue_url: job.work_queue_url,
+        message_body: job.to_json,
+        delay_seconds: job.delay
+      })
     end
 
-    def queue_for_job(job)
-      queue_name = job.queue || 'default'
-      queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
-      puts "queue_constant = #{queue_constant}"
-      puts "ENV value = #{ENV[queue_constant]}"
-      ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
+    def delayed_job_table
+      ENV['FUNKTOR_JOBS_TABLE']
     end
 
-    def push_to_active_job_queue(job)
-      puts "job = #{job.to_json}"
-      sqs_client.send_message({
-        # TODO : How to get this URL...
-        queue_url: queue_for_job(job),
-        message_body: job.to_json,
-        delay_seconds: job.delay
+    def push_to_jobs_table(job, category = nil)
+      resp = dynamodb_client.put_item({
+        item: {
+          payload: job.to_json,
+          jobId: job.job_id,
+          performAt: job.perform_at.iso8601,
+          jobShard: job.shard,
+          queueable: category.present? ? "false" : "true",
+          category: category || (job.is_retry? ? "retry" : "scheduled")
+        },
+        table_name: delayed_job_table
       })
     end
 

data/lib/funktor/job.rb

@@ -1,5 +1,8 @@
+require_relative 'shard_utils'
+
 module Funktor
   class Job
+    include ShardUtils
     attr_accessor :job_string
     attr_accessor :job_data
     def initialize(job_string)
@@ -11,7 +14,15 @@ module Funktor
     end
 
     def queue
-      job_data["queue"]
+      job_data["queue"] || 'default'
+    end
+
+    def work_queue_url
+      queue_name = self.queue
+      queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
+      Funktor.logger.debug "queue_constant = #{queue_constant}"
+      Funktor.logger.debug "ENV value = #{ENV[queue_constant]}"
+      ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
     end
 
     def worker_class_name
@@ -22,6 +33,10 @@ module Funktor
       job_data["job_id"]
     end
 
+    def shard
+      calculate_shard(job_data["job_id"])
+    end
+
     def worker_params
       job_data["worker_params"]
     end
@@ -30,20 +45,51 @@ module Funktor
       job_data["retries"] || 0
     end
 
+    def is_retry?
+      job_data["retries"].present?
+    end
+
     def retries=(retries)
       job_data["retries"] = retries
     end
 
+    def perform_at
+      if job_data["perform_at"].present?
+        job_data["perform_at"].is_a?(Time) ? job_data["perform_at"] : Time.parse(job_data["perform_at"])
+      else
+        Time.now.utc
+      end
+    end
+
     def delay
-      # TODO - In Funktor Pro we need to override this method (or do something else) so that
-      # we can schedule jobs farther in the future than 15 minutes. We do this here in case a
-      # retry sequence goes too long.
-      jdelay = job_data["delay"] || 0
-      return jdelay < 900 ? jdelay : 900
+      delay = (perform_at - Time.now.utc).to_i
+      if delay < 0
+        delay = 0
+      end
+      return delay
     end
 
     def delay=(delay)
-      job_data["delay"] = delay
+      job_data["perform_at"] = Time.now.utc + delay
+    end
+
+    def error_class
+      job_data["error_class"]
+    end
+
+    def error_message
+      job_data["error_message"]
+    end
+
+    def error_backtrace
+      job_data["error_backtrace"].present? ? Funktor.parse_json(job_data["error_backtrace"]) : []
+    end
+
+    def error=(error)
+      # TODO We should maybe compress this?
+      job_data["error_class"] = error.class.name
+      job_data["error_message"] = error.message
+      job_data["error_backtrace"] = Funktor.dump_json(error.backtrace)
     end
 
     def execute
@@ -79,6 +125,10 @@ module Funktor
 
     def retry_queue_url
       worker_class&.custom_queue_url || ENV['FUNKTOR_INCOMING_JOB_QUEUE']
+    rescue NameError, TypeError
+      # In the web ui we may not have access to the the worker classes
+      # TODO : We should mayb handle this differently somehow? This just feels a bit icky...
+      ENV['FUNKTOR_INCOMING_JOB_QUEUE']
     end
   end
 end