funktor 0.4.7 → 0.5.0

data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml

@@ -438,7 +438,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
+                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "Query" ]
                         ],
                         "view": "singleValue",
                         "region": "us-east-1",
@@ -455,7 +455,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", "Resource", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "label": "p10" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.JobActivatorName}", "Resource", "${self:custom.funktor.JobActivatorName}", { "label": "p10" } ],
                             [ "...", { "label": "p50", "stat": "p50" } ],
                             [ "...", { "label": "p99", "stat": "p99" } ],
                             [ "...", { "label": "Average", "stat": "Average" } ]
@@ -480,7 +480,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
+                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:custom.funktor.JobActivatorName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
                             [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
                             [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
                         ],
@@ -505,7 +505,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Minimum" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.JobActivatorName}", { "stat": "Minimum" } ],
                             [ "...", { "stat": "Average" } ],
                             [ "...", { "stat": "Maximum" } ]
                         ],
@@ -524,7 +524,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "Query" ]
+                            [ "AWS/DynamoDB", "ReturnedItemCount", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "Query" ]
                         ],
                         "view": "timeSeries",
                         "stacked": false,
@@ -547,7 +547,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             [ { "expression": "m2/PERIOD(m2)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
-                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
+                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:custom.funktor.JobsTableName}", { "id": "m2", "visible": false } ],
                             [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
                             [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
                         ],
@@ -569,7 +569,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-delayed_job_scheduler", { "stat": "Maximum" } ]
+                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:custom.funktor.JobActivatorName}", { "stat": "Maximum" } ]
                         ],
                         "region": "us-east-1",
                         "title": "Delayd Job Schedule Concurrent executions",
@@ -587,7 +587,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             [ { "expression": "m4/PERIOD(m4)", "label": "Consumed Read Capacity Units", "id": "e1", "stat": "Sum", "region": "us-east-1" } ],
-                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", { "id": "m2", "visible": false } ],
+                            [ "AWS/DynamoDB", "ConsumedReadCapacityUnits", "TableName", "${self:custom.funktor.JobsTableName}", { "id": "m2", "visible": false } ],
                             [ ".", "ConsumedWriteCapacityUnits", ".", ".", { "yAxis": "left", "id": "m4", "visible": false } ],
                             [ ".", "WriteThrottleEvents", ".", ".", { "yAxis": "right", "id": "m5", "visible": false } ]
                         ],
@@ -612,7 +612,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "PutItem", { "yAxis": "left" } ],
+                            [ "AWS/DynamoDB", "SuccessfulRequestLatency", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "PutItem", { "yAxis": "left" } ],
                             [ "...", "Query" ],
                             [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
                             [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
@@ -635,7 +635,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:service}-${self:custom.stage}-delayed-jobs", "Operation", "DeleteItem" ],
+                            [ "AWS/DynamoDB", "ThrottledRequests", "TableName", "${self:custom.funktor.JobsTableName}", "Operation", "DeleteItem" ],
                             [ "...", "PutItem" ]
                         ],
                         "view": "timeSeries",

data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml

@@ -0,0 +1,44 @@
+Resources:
+  JobsTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: ${self:custom.funktor.JobsTableName}
+      AttributeDefinitions:
+        - AttributeName: jobShard
+          AttributeType: N
+        - AttributeName: jobId
+          AttributeType: S
+        #- AttributeName: category
+          #AttributeType: S
+        - AttributeName: performAt
+          AttributeType: S
+        - AttributeName: dummy
+          AttributeType: S
+      KeySchema:
+        - AttributeName: jobShard
+          KeyType: HASH
+        - AttributeName: jobId
+          KeyType: RANGE
+      GlobalSecondaryIndexes:
+        - IndexName: performAtIndex
+          KeySchema:
+          - AttributeName: dummy
+            KeyType: HASH
+          - AttributeName: performAt
+            KeyType: RANGE
+          Projection:
+            NonKeyAttributes:
+            - jobId
+            - payload
+            - category
+            ProjectionType: INCLUDE
+      BillingMode: PAY_PER_REQUEST
+      # Set the capacity based on the stage
+      #ProvisionedThroughput:
+        #ReadCapacityUnits: ${self:custom.tableThroughput}
+        #WriteCapacityUnits: ${self:custom.tableThroughput}
+
+Outputs:
+  JobsTable:
+    Value:
+      Ref: JobsTable

data/lib/funktor/cli/templates/funktor_init.yml.tt

@@ -49,6 +49,8 @@ incomingJobHandler:
   # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
   # provisionedConcurrency: 4
 
+jobActivator:
+  # You probably don't need to adjust the defaults for this one.
 
 queues:
   - default:
@@ -59,11 +61,3 @@ queues:
   - singleThread:
       reservedConcurrency: 1
 
-# TODO - Maybe this shouldn't be surfaced this early?
-# TODO - This still needs to be wired up to do anything.
-package:
-  patterns:
-    - Gemfile
-    - Gemfile.lock
-    - app/**
-    - funktor_config/**

data/lib/funktor/cli/templates/lambda_event_handlers/job_activator.rb

@@ -0,0 +1,8 @@
+require_relative '../funktor_config/boot'
+
+$handler = Funktor::JobActivator.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb

@@ -1,6 +1,6 @@
 require_relative '../funktor_config/boot'
 
-$handler = Funktor::ActiveJobHandler.new
+$handler = Funktor::WorkQueueHandler.new
 
 def call(event:, context:)
   $handler.call(event: event, context: context)

data/lib/funktor/cli/templates/serverless.yml

@@ -3,8 +3,8 @@
 # This file is the main config file for your service.
 # It's already configured to run Funktor, you just have to deploy it.
 #
-# For more info about Funktor:
-#    TODO
+# For more info about Funktor check the wiki:
+#    https://github.com/Octo-Labs/funktor/wiki
 #
 # For more about serverless, check their docs:
 #    docs.serverless.com

data/lib/funktor/counter.rb

@@ -11,7 +11,10 @@ module Funktor
     end
 
     def put_metric_to_stdout(job)
-      puts Funktor.dump_json(metric_hash(job))
+      # NOTE : We use raw puts here instead of Funktor.logger.something to avoid getting extra
+      # timestamps or log level information in the log line. We need this specific format to
+      # be the only thing in the line so that CloudWatch can parse the logs and use the data.
+      Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(job))
     end
 
     def metric_hash(job)

data/lib/funktor/incoming_job_handler.rb

@@ -1,12 +1,17 @@
 require 'aws-sdk-sqs'
+require 'aws-sdk-dynamodb'
 require 'active_support/core_ext/string/inflections'
 
 module Funktor
   class IncomingJobHandler
 
+    def initialize
+      @tracker = Funktor::ActivityTracker.new
+    end
+
     def call(event:, context:)
       event = Funktor::Aws::Sqs::Event.new(event)
-      puts "event.jobs.count = #{event.jobs.count}"
+      Funktor.logger.debug "event.jobs.count = #{event.jobs.count}"
       event.jobs.each do |job|
         dispatch(job)
       end
@@ -16,34 +21,69 @@ module Funktor
       @sqs_client ||= ::Aws::SQS::Client.new
     end
 
+    def dynamodb_client
+      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+    end
+
     def dispatch(job)
       Funktor.incoming_job_handler_middleware.invoke(job) do
-        puts "pushing to active_job_queue for delay = #{job.delay}"
-        push_to_active_job_queue(job)
+        # TODO : This number should be configurable via ENV var
+        if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
+          Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
+          push_to_work_queue(job)
+          if job.is_retry?
+            @tracker.track(:retryActivated, job)
+          else
+            @tracker.track(:queued, job)
+          end
+        else
+          Funktor.logger.debug "pushing to jobs table for delay = #{job.delay}"
+          push_to_jobs_table(job)
+          if job.is_retry?
+            # do nothing for tracking
+          else
+            @tracker.track(:scheduled, job)
+          end
+        end
+        @tracker.track(:incoming, job)
       end
     end
 
-    def active_job_queue
-      ENV['FUNKTOR_ACTIVE_JOB_QUEUE']
-    end
-
     def queue_for_job(job)
       queue_name = job.queue || 'default'
       queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
-      puts "queue_constant = #{queue_constant}"
-      puts "ENV value = #{ENV[queue_constant]}"
+      Funktor.logger.debug "queue_constant = #{queue_constant}"
+      Funktor.logger.debug "ENV value = #{ENV[queue_constant]}"
       ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
     end
 
-    def push_to_active_job_queue(job)
-      puts "job = #{job.to_json}"
+    def push_to_work_queue(job)
+      Funktor.logger.debug "job = #{job.to_json}"
       sqs_client.send_message({
-        # TODO : How to get this URL...
         queue_url: queue_for_job(job),
         message_body: job.to_json,
         delay_seconds: job.delay
       })
     end
 
+    def delayed_job_table
+      ENV['FUNKTOR_JOBS_TABLE']
+    end
+
+    def push_to_jobs_table(job)
+      perform_at = (Time.now + job.delay).utc
+      resp = dynamodb_client.put_item({
+        item: {
+          payload: job.to_json,
+          jobId: job.job_id,
+          performAt: perform_at.iso8601,
+          jobShard: job.shard,
+          dummy: "dummy",
+          category: job.is_retry? ? "retry" : "scheduled"
+        },
+        table_name: delayed_job_table
+      })
+    end
+
   end
 end

data/lib/funktor/job.rb

@@ -22,6 +22,11 @@ module Funktor
       job_data["job_id"]
     end
 
+    def shard
+      # TODO - Should the number of shards be configurable?
+      job_data["job_id"].hash % 64
+    end
+
     def worker_params
       job_data["worker_params"]
     end
@@ -30,16 +35,16 @@ module Funktor
       job_data["retries"] || 0
     end
 
+    def is_retry?
+      job_data["retries"].present?
+    end
+
     def retries=(retries)
       job_data["retries"] = retries
     end
 
     def delay
-      # TODO - In Funktor Pro we need to override this method (or do something else) so that
-      # we can schedule jobs farther in the future than 15 minutes. We do this here in case a
-      # retry sequence goes too long.
-      jdelay = job_data["delay"] || 0
-      return jdelay < 900 ? jdelay : 900
+      job_data["delay"] || 0
     end
 
     def delay=(delay)

data/lib/funktor/job_activator.rb

@@ -0,0 +1,98 @@
+require 'aws-sdk-dynamodb'
+require 'aws-sdk-sqs'
+
+module Funktor
+  class JobActivator
+
+    def initialize
+      @tracker = Funktor::ActivityTracker.new
+    end
+
+    def dynamodb_client
+      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+    end
+
+    def sqs_client
+      @sqs_client ||= ::Aws::SQS::Client.new
+    end
+
+    def active_job_queue
+      ENV['FUNKTOR_ACTIVE_JOB_QUEUE']
+    end
+
+    def delayed_job_table
+      ENV['FUNKTOR_JOBS_TABLE']
+    end
+
+    def jobs_to_activate
+      target_time = (Time.now + 90).utc
+      query_params = {
+        expression_attribute_values: {
+          ":dummy" => "dummy",
+          ":targetTime" => target_time.iso8601
+        },
+        key_condition_expression: "dummy = :dummy AND performAt < :targetTime",
+        projection_expression: "payload, performAt, jobId, jobShard",
+        table_name: delayed_job_table,
+        index_name: "performAtIndex"
+      }
+      resp = dynamodb_client.query(query_params)
+      return resp.items
+    end
+
+    def queue_for_job(job)
+      queue_name = job.queue || 'default'
+      queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
+      Funktor.logger.debug "queue_constant = #{queue_constant}"
+      Funktor.logger.debug "ENV value = #{ENV[queue_constant]}"
+      ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
+    end
+
+    def handle_item(item)
+      job = Funktor::Job.new(item["payload"])
+      Funktor.logger.debug "we created a job from payload"
+      Funktor.logger.debug item["payload"]
+      delay = (Time.parse(item["performAt"]) - Time.now.utc).to_i
+      if delay < 0
+        delay = 0
+      end
+      Funktor.logger.debug "jobShard = #{item['jobShard']}"
+      Funktor.logger.debug "jobId = #{item['jobId']}"
+      # First we delete the item from Dynamo to be sure that another scheduler hasn't gotten to it,
+      # and if that works then send to SQS. This is basically how Sidekiq scheduler works.
+      response = dynamodb_client.delete_item({
+        key: {
+          "jobShard" => item["jobShard"],
+          "jobId" => item["jobId"]
+        },
+        table_name: delayed_job_table,
+        return_values: "ALL_OLD"
+      })
+      if response.attributes # this means the record was still there
+        sqs_client.send_message({
+          # TODO : How to get this URL...
+          queue_url: queue_for_job(job),
+          message_body: item["payload"],
+          delay_seconds: delay
+        })
+        if job.is_retry?
+          @tracker.track(:retryActivated, job)
+        else
+          @tracker.track(:scheduledJobActivated, job)
+        end
+      end
+    end
+
+    def call(event:, context:)
+      handled_item_count = 0
+      jobs_to_activate.each do |item|
+        if context.get_remaining_time_in_millis < 5_000 # This lets us exit gracefully and resume on the next round instead of getting forcibly killed.
+          puts "Bailing out due to milliseconds remaining #{context.get_remaining_time_in_millis}"
+          break
+        end
+        handle_item(item)
+        handled_item_count += 1
+      end
+    end
+  end
+end

data/lib/funktor/job_pusher.rb

@@ -4,8 +4,6 @@ module Funktor
   class JobPusher
 
     def push(payload)
-      puts "payload ============"
-      pp payload
       job_id = SecureRandom.uuid
       payload[:job_id] = job_id
 

data/lib/funktor/middleware/metrics.rb

@@ -10,7 +10,12 @@ module Funktor
       end
 
       def put_metric_to_stdout(time_diff, job)
-        puts Funktor.dump_json(metric_hash(time_diff, job))
+        # NOTE : We use raw_logger here instead of Funktor.loggert o avoid getting extra
+        # timestamps or log level information in the log line. We need this specific format to
+        # be the only thing in the line so that CloudWatch can parse the logs and use the data.
+        # 'unknown' is a log level that will always be logged, no matter what is set in the
+        # runtime environment as far as log level.
+        Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(time_diff, job))
       end
 
       def metric_hash(time_diff_in_seconds, job)
@@ -43,8 +48,8 @@ module Funktor
   end
 end
 
-Funktor.configure_active_job_handler do |config|
-  config.active_job_handler_middleware do |chain|
+Funktor.configure_work_queue_handler do |config|
+  config.work_queue_handler_middleware do |chain|
     chain.add Funktor::Middleware::Metrics
   end
 end