funktor 0.4.6 → 0.6.1

data/funktor-testapp/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "funktor-testapp",
+  "description": "",
+  "version": "0.1.0",
+  "devDependencies": {
+    "serverless-ruby-layer": "^1.4.0"
+  }
+}

data/funktor-testapp/serverless.yml

@@ -0,0 +1,66 @@
+# Welcome to Funktor & Serverless!
+#
+# This file is the main config file for your service.
+# It's already configured to run Funktor, you just have to deploy it.
+#
+# For more info about Funktor check the wiki:
+#    https://github.com/Octo-Labs/funktor/wiki
+#
+# For more about serverless, check their docs:
+#    docs.serverless.com
+#
+# Happy Coding!
+
+# The name of your service. All your AWS resources will contain this name.
+service: funktor-testapp
+
+# This causes serverless to throw an error early if the config is bad, instead of waiting for CloudFormation to try and fail to deploy it.
+configValidationMode: error
+
+# Pin the serverless framework to the 2.x line
+frameworkVersion: '2'
+
+provider:
+  name: aws
+  runtime: ruby2.7
+  lambdaHashingVersion: 20201221
+  environment: ${file(funktor_config/environment.yml)}
+  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
+  logRetentionInDays: 7
+  iamRoleStatements:
+    - ${file(funktor_config/iam_permissions/activity_table.yml)}
+    - ${file(funktor_config/iam_permissions/default_queue.yml)}
+    - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
+    - ${file(funktor_config/iam_permissions/ssm.yml)}
+    - ${file(funktor_config/iam_permissions/low_concurrency_queue.yml)}
+    - ${file(funktor_config/iam_permissions/jobs_table.yml)}
+    - ${file(funktor_config/iam_permissions/jobs_table_secondary_index.yml)}
+
+
+custom:
+  # Our stage is based on what is passed in when running serverless
+  # commands. Or fallsback to what we have set in the provider section.
+  stage: ${self:provider.stage, 'dev'}
+  funktor: ${file(funktor_config/funktor.yml)}
+  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+
+package: ${file(funktor_config/package.yml)}
+
+functions:
+  IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
+  DefaultQueueHandler: ${file(funktor_config/function_definitions/default_queue_handler.yml)}
+  JobActivator: ${file(funktor_config/function_definitions/job_activator.yml)}
+  LowConcurrencyQueueHandler: ${file(funktor_config/function_definitions/low_concurrency_queue_handler.yml)}
+  RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
+
+resources:
+  - ${file(funktor_config/resources/activity_table.yml)}
+  - ${file(funktor_config/resources/incoming_job_queue_user.yml)}
+  - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
+  - ${file(funktor_config/resources/default_queue.yml)}
+  - ${file(funktor_config/resources/incoming_job_queue.yml)}
+  - ${file(funktor_config/resources/low_concurrency_queue.yml)}
+  - ${file(funktor_config/resources/jobs_table.yml)}
+
+plugins:
+  - serverless-ruby-layer

data/funktor.gemspec

@@ -30,11 +30,14 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'aws-sdk-sqs', '~> 1.37'
+  spec.add_dependency 'aws-sdk-dynamodb', '~> 1.62'
   spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
-  spec.add_dependency "thor" # Thor drives the CLI
+  spec.add_dependency "thor" # Thor drives the CLI TODO - should this just be a dev dependency?
 
   spec.add_development_dependency 'activejob', '>= 5.1.5'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'webmock'
   spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'sinatra'
+  spec.add_development_dependency 'timecop'
 end

data/lib/active_job/queue_adapters/funktor_adapter.rb

@@ -39,11 +39,11 @@ module ActiveJob
         include Funktor::Worker
 
         def perform(job_data)
-          puts "job_data = #{job_data.class} #{job_data}"
+          Funktor.logger.debug "job_data = #{job_data.class} #{job_data}"
           if job_data.is_a?(Array)
-            puts "it's an array"
+            Funktor.logger.debug "it's an array"
             job_data = job_data.first
-            puts "job_data = #{job_data.class} #{job_data}"
+            Funktor.logger.debug "job_data = #{job_data.class} #{job_data}"
           end
           Base.execute job_data
         end

data/lib/funktor/activity_tracker.rb

@@ -0,0 +1,106 @@
+require 'json'
+require 'aws-sdk-dynamodb'
+
+module Funktor
+  class ActivityTracker
+
+    INCR_KEYS = {
+      incoming: 'incoming',
+      queued: 'queued',
+      scheduled: 'scheduled',
+      processingStarted: 'processing',
+      processingComplete: 'complete',
+      processingFailed: 'failed',
+      bailingOut: 'failed',
+      retrying: 'retries',
+      retryActivated: 'queued',
+      scheduledJobActivated: nil,
+      scheduledJobDeleted: 'scheduledJobDeleted',
+      retryDeleted: 'retryDeleted'
+      #scheduledJobPushedToActive: 'active',
+      #activeJobPushed: 'active',
+      #scheduledJobPushed: 'scheduled'
+    }
+
+    DECR_KEYS = {
+      incoming: nil,
+      queued: nil,
+      scheduled: nil,
+      processingStarted: 'queued',
+      processingComplete: 'processing',
+      processingFailed: nil,
+      bailingOut: 'processing',
+      retrying: 'processing',
+      retryActivated: 'retries',
+      scheduledJobActivated: 'scheduled',
+      scheduledJobDeleted: 'scheduled',
+      retryDeleted: 'retries'
+      #scheduledJobPushedToActive: 'scheduled',
+      #activeJobPushed: nil,
+      #scheduledJobPushed: nil
+    }
+
+    def track(activity, job)
+      Funktor.logger.debug "starting track activity for #{activity}"
+      incrKey = nil || INCR_KEYS[activity.to_sym]
+      decrKey = nil || DECR_KEYS[activity.to_sym]
+      if incrKey
+        increment_key(incrKey)
+      end
+      if decrKey
+        decrement_key(decrKey)
+      end
+      Funktor.logger.debug "ending track activity for #{activity}"
+    end
+
+    def increment_key(key)
+      #put_metric_to_stdout(key, 1)
+      dynamodb_client.update_item({
+        table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
+        key: { category: 'stat', statName: key },
+        expression_attribute_values:{ ":start": 0, ":inc": 1 },
+        update_expression: "SET stat_value = if_not_exists(stat_value, :start) + :inc",
+      })
+    end
+
+    def decrement_key(key)
+      #put_metric_to_stdout(key, -1)
+      dynamodb_client.update_item({
+        table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
+        key: { category: 'stat', statName: key },
+        expression_attribute_values:{ ":start": 0, ":inc": 1 },
+        update_expression: "SET stat_value = if_not_exists(stat_value, :start) - :inc",
+      })
+    end
+
+    def dynamodb_client
+      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+    end
+
+    def put_metric_to_stdout(key, value)
+      data = {
+        "_aws": {
+          "Timestamp": Time.now.strftime('%s%3N').to_i,
+          "CloudWatchMetrics": [
+            {
+              "Namespace": "rails-lambda-experiment",
+              "Dimensions": [["functionVersion"]],
+              "Metrics": [ # CPU, Memory, Duration, etc...
+                           {
+                             "Name": key,
+                             "Unit": "Count"
+                           }
+              ]
+            }
+          ]
+        },
+        "functionVersion": "LATEST",
+        #"count": value,
+        #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
+      }
+      data[key] = value
+      puts(data.to_json)
+    end
+
+  end
+end

data/lib/funktor/cli/bootstrap.rb

@@ -24,7 +24,6 @@ module Funktor
 
       def funktor_yml
         # TODO - Should we camelize the app name before writing it into the config? (CloudFormation names get weird with underscores and dashes.)
-        puts funktor_file_target
         template "templates/funktor_init.yml", funktor_file_target
       end
 

data/lib/funktor/cli/init.rb

@@ -65,6 +65,8 @@ module Funktor
           @work_queue_config = queue_details.values.first
           template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
         end
+        template File.join("funktor_config", "resources", "jobs_table.yml"), File.join("funktor_config", "resources", "jobs_table.yml")
+        template File.join("funktor_config", "resources", "activity_table.yml"), File.join("funktor_config", "resources", "activity_table.yml")
       end
 
       def iam_permissions
@@ -75,10 +77,14 @@ module Funktor
           @work_queue_config = queue_details.values.first
           template File.join("funktor_config", "iam_permissions", "work_queue.yml"), File.join("funktor_config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
         end
+        template File.join("funktor_config", "iam_permissions", "jobs_table.yml"), File.join("funktor_config", "iam_permissions", "jobs_table.yml")
+        template File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml"), File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml")
+        template File.join("funktor_config", "iam_permissions", "activity_table.yml"), File.join("funktor_config", "iam_permissions", "activity_table.yml")
       end
 
       def function_definitions
         template File.join("funktor_config", "function_definitions", "incoming_job_handler.yml"), File.join("funktor_config", "function_definitions", "incoming_job_handler.yml")
+        template File.join("funktor_config", "function_definitions", "job_activator.yml"), File.join("funktor_config", "function_definitions", "job_activator.yml")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
@@ -88,6 +94,7 @@ module Funktor
 
       def lambda_handlers
         template File.join("lambda_event_handlers", "incoming_job_handler.rb"), File.join("lambda_event_handlers", "incoming_job_handler.rb")
+        template File.join("lambda_event_handlers", "job_activator.rb"), File.join("lambda_event_handlers", "job_activator.rb")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
@@ -170,6 +177,12 @@ module Funktor
           "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
       end
 
+      def activator_config_value(config_name)
+        funktor_config.dig("jobActivator", config_name) ||
+          funktor_config.dig("handlerDefaults", config_name) ||
+          "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
+      end
+
       def queue_config_value(queue_name, config_name)
         queue_config(queue_name)&.dig(config_name) ||
           funktor_config.dig("handlerDefaults", config_name) ||

data/lib/funktor/cli/templates/app/workers/hello_worker.rb

@@ -2,7 +2,7 @@ class HelloWorker
   include Funktor::Worker
 
   def perform(*args)
-    puts "Greetings from the HelloWorker!"
+    Funktor.logger.debug "Greetings from the HelloWorker!"
   end
 end
 

data/lib/funktor/cli/templates/funktor_config/environment.yml

@@ -9,3 +9,7 @@ FUNKTOR_INCOMING_JOB_QUEUE:
 FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
   Ref: <%= queue_name.camelize %>Queue
 <%- end -%>
+FUNKTOR_ACTIVITY_TABLE:
+  Ref: ActivityTable
+FUNKTOR_JOBS_TABLE:
+  Ref: JobsTable

data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,10 +1,12 @@
 handler: lambda_event_handlers/incoming_job_handler.call
-timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
+timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
 events:
   - sqs:
+      batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}
+      maximumBatchingWindow: ${self:custom.funktor.IncomingJobHandler.maximumBatchingWindow, 0}
       arn:
         Fn::GetAtt:
           - IncomingJobQueue

data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml

@@ -0,0 +1,7 @@
+handler: lambda_event_handlers/job_activator.call
+timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
+reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+events:
+  - schedule: rate(1 minute)

data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml

@@ -1,10 +1,12 @@
 handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
-timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.timeout, 900}
+timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
 events:
   - sqs:
+      batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}
+      maximumBatchingWindow: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.maximumBatchingWindow, 0}
       arn:
         Fn::GetAtt:
           - <%= work_queue_name.camelize %>Queue

data/lib/funktor/cli/templates/funktor_config/funktor.yml

@@ -3,7 +3,7 @@ IncomingJobHandler:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
   # but we default to a high number here to allow for the times when things go weird.
-  timeout: <%= incoming_config_value 'timeout' %>
+  functionTimeout: <%= incoming_config_value 'functionTimeout' %>
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the incoming job handler you probably don't want to limit it because you
   # want to get things onto work queues as quickly as possible.
@@ -19,7 +19,32 @@ IncomingJobHandler:
   batchSize: <%= incoming_config_value 'batchSize' %>
   maximumBatchingWindow: <%= incoming_config_value 'maximumBatchingWindow' %>
   visibilityTimeout: <%= incoming_config_value 'visibilityTimeout' %>
-  logRetentionInDays: <%= incoming_config_value 'logRetentionInDays' %>
+  maxReceiveCount: <%= incoming_config_value 'maxReceiveCount' %>
+
+JobActivator:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The job activator usually will be pretty fast,
+  # but we default to a high number here to allow for the times when things go weird.
+  functionTimeout: <%= activator_config_value 'functionTimeout' %>
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the job activator you probably don't want to limit it because you
+  # want to get things onto work queues as quickly as possible when they're ready.
+  reservedConcurrency: <%= activator_config_value 'reservedConcurrency' %>
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the job activator you probably don't need to set this
+  # since it will be running on a schedule
+  provisionedConcurrency: <%= activator_config_value 'provisionedConcurrency' %>
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the job activator you probably don't need this to be too large,
+  # but if you're seeing large delays at this stage it might help to bump it up.
+  memorySize: <%= activator_config_value 'memorySize' %>
+  batchSize: <%= activator_config_value 'batchSize' %>
+  maximumBatchingWindow: <%= activator_config_value 'maximumBatchingWindow' %>
+  visibilityTimeout: <%= activator_config_value 'visibilityTimeout' %>
+  maxReceiveCount: <%= activator_config_value 'maxReceiveCount' %>
+
+
 
 <%- queue_names.each do |queue_name| -%>
 <%= queue_name.camelize %>QueueHandler:
@@ -27,7 +52,7 @@ IncomingJobHandler:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The active job handler may be slow if your jobs are
   # doing a lot of work, so we default to the maximum here.
-  timeout: <%= queue_config_value queue_name, 'timeout' %>
+  functionTimeout: <%= queue_config_value queue_name, 'functionTimeout' %>
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the active job handler you may want to limit it if you have resource limitations
   # like database connections that you need to avoid exhausting.
@@ -44,7 +69,7 @@ IncomingJobHandler:
   batchSize: <%= queue_config_value queue_name, 'batchSize' %>
   maximumBatchingWindow: <%= queue_config_value queue_name, 'maximumBatchingWindow' %>
   visibilityTimeout: <%= queue_config_value queue_name, 'visibilityTimeout' %>
-  logRetentionInDays: <%= queue_config_value queue_name, 'logRetentionInDays' %>
+  maxReceiveCount: <%= queue_config_value queue_name, 'maxReceiveCount' %>
 
 <%- end -%>
 
@@ -61,5 +86,6 @@ DashboardName: ${self:service}-${self:custom.stage}-dashboard
 <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
 <%= queue_name.camelize %>QueueHandlerName: ${self:service}-${self:custom.stage}-<%= queue_name.camelize %>QueueHandler
 <%- end -%>
-
-
+JobsTableName: ${self:service}-${self:custom.stage}-jobs
+ActivityTableName: ${self:service}-${self:custom.stage}-activity
+JobActivatorName: ${self:service}-${self:custom.stage}-JobActivator

data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml

@@ -0,0 +1,5 @@
+Effect: Allow
+Action:
+  - dynamodb:*
+Resource:
+  - "Fn::GetAtt": [ ActivityTable, Arn ]

data/lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml

@@ -0,0 +1,5 @@
+Effect: Allow
+Action:
+  - dynamodb:*
+Resource:
+  - "Fn::GetAtt": [ JobsTable, Arn ]

data/lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml

@@ -0,0 +1,8 @@
+Effect: Allow
+Action:
+  - dynamodb:Query
+Resource:
+  Fn::Join:
+    - ""
+    - - "Fn::GetAtt": [ JobsTable, Arn ]
+      - "/index/performAtIndex"

data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml

@@ -0,0 +1,22 @@
+Resources:
+  ActivityTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      # Generate a name based on the stage
+      TableName: ${self:custom.funktor.ActivityTableName}
+      BillingMode: PAY_PER_REQUEST
+      AttributeDefinitions:
+        - AttributeName: category
+          AttributeType: S
+        - AttributeName: statName
+          AttributeType: S
+      KeySchema:
+        - AttributeName: category
+          KeyType: HASH
+        - AttributeName: statName
+          KeyType: RANGE
+
+Outputs:
+  ActivityTable:
+    Value:
+      Ref: ActivityTable