funktor 0.4.7 → 0.5.0

data/funktor-testapp/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "funktor-testapp",
+  "description": "",
+  "version": "0.1.0",
+  "devDependencies": {
+    "serverless-ruby-layer": "^1.4.0"
+  }
+}

data/funktor-testapp/serverless.yml

@@ -0,0 +1,65 @@
+# Welcome to Funktor & Serverless!
+#
+# This file is the main config file for your service.
+# It's already configured to run Funktor, you just have to deploy it.
+#
+# For more info about Funktor check the wiki:
+#    https://github.com/Octo-Labs/funktor/wiki
+#
+# For more about serverless, check their docs:
+#    docs.serverless.com
+#
+# Happy Coding!
+
+# The name of your service. All your AWS resources will contain this name.
+service: funktor-testapp
+
+# This causes serverless to throw an error early if the config is bad, instead of waiting for CloudFormation to try and fail to deploy it.
+configValidationMode: error
+
+# Pin the serverless framework to the 2.x line
+frameworkVersion: '2'
+
+provider:
+  name: aws
+  runtime: ruby2.7
+  lambdaHashingVersion: 20201221
+  environment: ${file(funktor_config/environment.yml)}
+  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
+  iamRoleStatements:
+    - ${file(funktor_config/iam_permissions/activity_table.yml)}
+    - ${file(funktor_config/iam_permissions/default_queue.yml)}
+    - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
+    - ${file(funktor_config/iam_permissions/ssm.yml)}
+    - ${file(funktor_config/iam_permissions/single_thread_queue.yml)}
+    - ${file(funktor_config/iam_permissions/jobs_table.yml)}
+    - ${file(funktor_config/iam_permissions/jobs_table_secondary_index.yml)}
+
+
+custom:
+  # Our stage is based on what is passed in when running serverless
+  # commands. Or fallsback to what we have set in the provider section.
+  stage: ${self:provider.stage, 'dev'}
+  funktor: ${file(funktor_config/funktor.yml)}
+  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+
+package: ${file(funktor_config/package.yml)}
+
+functions:
+  IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
+  DefaultQueueHandler: ${file(funktor_config/function_definitions/default_queue_handler.yml)}
+  JobActivator: ${file(funktor_config/function_definitions/job_activator.yml)}
+  SingleThreadQueueHandler: ${file(funktor_config/function_definitions/single_thread_queue_handler.yml)}
+  RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
+
+resources:
+  - ${file(funktor_config/resources/activity_table.yml)}
+  - ${file(funktor_config/resources/incoming_job_queue_user.yml)}
+  - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
+  - ${file(funktor_config/resources/default_queue.yml)}
+  - ${file(funktor_config/resources/incoming_job_queue.yml)}
+  - ${file(funktor_config/resources/single_thread_queue.yml)}
+  - ${file(funktor_config/resources/jobs_table.yml)}
+
+plugins:
+  - serverless-ruby-layer

data/funktor.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'aws-sdk-sqs', '~> 1.37'
+  spec.add_dependency 'aws-sdk-dynamodb', '~> 1.62'
   spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
   spec.add_dependency "thor" # Thor drives the CLI
 

data/lib/active_job/queue_adapters/funktor_adapter.rb

@@ -39,11 +39,11 @@ module ActiveJob
         include Funktor::Worker
 
         def perform(job_data)
-          puts "job_data = #{job_data.class} #{job_data}"
+          Funktor.logger.debug "job_data = #{job_data.class} #{job_data}"
           if job_data.is_a?(Array)
-            puts "it's an array"
+            Funktor.logger.debug "it's an array"
             job_data = job_data.first
-            puts "job_data = #{job_data.class} #{job_data}"
+            Funktor.logger.debug "job_data = #{job_data.class} #{job_data}"
           end
           Base.execute job_data
         end

data/lib/funktor.rb

@@ -8,6 +8,8 @@ require 'funktor/logger'
 require 'funktor/worker'
 require 'funktor/middleware_chain'
 require 'funktor/incoming_job_handler'
+require 'funktor/job_activator'
+require 'funktor/activity_tracker'
 
 require 'json'
 
@@ -33,14 +35,14 @@ module Funktor
     @job_pusher_chain
   end
 
-  def self.configure_active_job_handler
+  def self.configure_work_queue_handler
     yield self
   end
 
-  def self.active_job_handler_middleware
-    @active_job_handler_chain ||= MiddlewareChain.new
-    yield @active_job_handler_chain if block_given?
-    @active_job_handler_chain
+  def self.work_queue_handler_middleware
+    @work_queue_handler_chain ||= MiddlewareChain.new
+    yield @work_queue_handler_chain if block_given?
+    @work_queue_handler_chain
   end
 
   # TODO - Maybe we don't need this either? Maybe this should be a super dumb thing that also
@@ -58,6 +60,18 @@ module Funktor
     @incoming_job_handler_chain
   end
 
+  # TODO - Does this actually make any sense? Should the JobActivator even know about
+  # jobs/classes? Maybe it should be super dumb and just push JSON blobs around?
+  def self.configure_job_activator
+    yield self
+  end
+
+  def self.job_activator_middleware
+    @job_activator_chain ||= MiddlewareChain.new
+    yield @job_activator_chain if block_given?
+    @job_activator_chain
+  end
+
   def self.parse_json(string)
     JSON.parse(string)
   end
@@ -95,13 +109,31 @@ module Funktor
 
     @logger = logger
   end
+
+  # We have a raw_logger that doesn't add timestamps and what not. This is used to publish
+  # CloudWatch metrics that can be used in dashboards.
+  def self.raw_logger
+    @raw_logger ||= Funktor::Logger.new($stdout, level: options[:log_level], formatter: proc {|severity, datetime, progname, msg|
+      "#{msg}\n"
+    })
+  end
+
+  def self.raw_logger=(raw_logger)
+    if raw_logger.nil?
+      self.raw_logger.level = Logger::FATAL
+      return self.raw_logger
+    end
+
+    @raw_logger = raw_logger
+  end
 end
 
-# TODO - Should we require this by default or let people opt in?
 # TODO - Is it a code smell that we need to include these at the bottom, after
 # the main Funktor module is defined?
+#
+# TODO - Should we require metrics by default or let people opt in?
 require 'funktor/middleware/metrics'
 require 'funktor/error_handler'
-require 'funktor/active_job_handler'
+require 'funktor/work_queue_handler'
 
 require 'funktor/rails' if defined?(::Rails::Engine)

data/lib/funktor/activity_tracker.rb

@@ -0,0 +1,102 @@
+require 'json'
+require 'aws-sdk-dynamodb'
+
+module Funktor
+  class ActivityTracker
+
+    INCR_KEYS = {
+      incoming: 'incoming',
+      queued: 'queued',
+      scheduled: 'scheduled',
+      processingStarted: 'processing',
+      processingComplete: 'complete',
+      processingFailed: 'failed',
+      bailingOut: 'failed',
+      retrying: 'retries',
+      retryActivated: 'queued',
+      scheduledJobActivated: 'queued'
+      #scheduledJobPushedToActive: 'active',
+      #activeJobPushed: 'active',
+      #scheduledJobPushed: 'scheduled'
+    }
+
+    DECR_KEYS = {
+      incoming: nil,
+      queued: nil,
+      scheduled: nil,
+      processingStarted: 'queued',
+      processingComplete: 'processing',
+      processingFailed: nil,
+      bailingOut: 'processing',
+      retrying: 'processing',
+      retryActivated: 'retries',
+      scheduledJobActivated: 'scheduled'
+      #scheduledJobPushedToActive: 'scheduled',
+      #activeJobPushed: nil,
+      #scheduledJobPushed: nil
+    }
+
+    def track(activity, job)
+      Funktor.logger.debug "starting track activity for #{activity}"
+      incrKey = nil || INCR_KEYS[activity.to_sym]
+      decrKey = nil || DECR_KEYS[activity.to_sym]
+      if incrKey
+        increment_key(incrKey)
+      end
+      if decrKey
+        decrement_key(decrKey)
+      end
+      Funktor.logger.debug "ending track activity for #{activity}"
+    end
+
+    def increment_key(key)
+      #put_metric_to_stdout(key, 1)
+      dynamodb_client.update_item({
+        table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
+        key: { category: 'stat', statName: key },
+        expression_attribute_values:{ ":start": 0, ":inc": 1 },
+        update_expression: "SET stat_value = if_not_exists(stat_value, :start) + :inc",
+      })
+    end
+
+    def decrement_key(key)
+      #put_metric_to_stdout(key, -1)
+      dynamodb_client.update_item({
+        table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
+        key: { category: 'stat', statName: key },
+        expression_attribute_values:{ ":start": 0, ":inc": 1 },
+        update_expression: "SET stat_value = if_not_exists(stat_value, :start) - :inc",
+      })
+    end
+
+    def dynamodb_client
+      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+    end
+
+    def put_metric_to_stdout(key, value)
+      data = {
+        "_aws": {
+          "Timestamp": Time.now.strftime('%s%3N').to_i,
+          "CloudWatchMetrics": [
+            {
+              "Namespace": "rails-lambda-experiment",
+              "Dimensions": [["functionVersion"]],
+              "Metrics": [ # CPU, Memory, Duration, etc...
+                           {
+                             "Name": key,
+                             "Unit": "Count"
+                           }
+              ]
+            }
+          ]
+        },
+        "functionVersion": "LATEST",
+        #"count": value,
+        #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
+      }
+      data[key] = value
+      puts(data.to_json)
+    end
+
+  end
+end

data/lib/funktor/cli/bootstrap.rb

@@ -24,7 +24,6 @@ module Funktor
 
       def funktor_yml
         # TODO - Should we camelize the app name before writing it into the config? (CloudFormation names get weird with underscores and dashes.)
-        puts funktor_file_target
         template "templates/funktor_init.yml", funktor_file_target
       end
 

data/lib/funktor/cli/init.rb

@@ -65,6 +65,8 @@ module Funktor
           @work_queue_config = queue_details.values.first
           template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
         end
+        template File.join("funktor_config", "resources", "jobs_table.yml"), File.join("funktor_config", "resources", "jobs_table.yml")
+        template File.join("funktor_config", "resources", "activity_table.yml"), File.join("funktor_config", "resources", "activity_table.yml")
       end
 
       def iam_permissions
@@ -75,10 +77,14 @@ module Funktor
           @work_queue_config = queue_details.values.first
           template File.join("funktor_config", "iam_permissions", "work_queue.yml"), File.join("funktor_config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
         end
+        template File.join("funktor_config", "iam_permissions", "jobs_table.yml"), File.join("funktor_config", "iam_permissions", "jobs_table.yml")
+        template File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml"), File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml")
+        template File.join("funktor_config", "iam_permissions", "activity_table.yml"), File.join("funktor_config", "iam_permissions", "activity_table.yml")
       end
 
       def function_definitions
         template File.join("funktor_config", "function_definitions", "incoming_job_handler.yml"), File.join("funktor_config", "function_definitions", "incoming_job_handler.yml")
+        template File.join("funktor_config", "function_definitions", "job_activator.yml"), File.join("funktor_config", "function_definitions", "job_activator.yml")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
@@ -88,6 +94,7 @@ module Funktor
 
       def lambda_handlers
         template File.join("lambda_event_handlers", "incoming_job_handler.rb"), File.join("lambda_event_handlers", "incoming_job_handler.rb")
+        template File.join("lambda_event_handlers", "job_activator.rb"), File.join("lambda_event_handlers", "job_activator.rb")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
@@ -170,6 +177,12 @@ module Funktor
           "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
       end
 
+      def activator_config_value(config_name)
+        funktor_config.dig("jobActivator", config_name) ||
+          funktor_config.dig("handlerDefaults", config_name) ||
+          "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
+      end
+
       def queue_config_value(queue_name, config_name)
         queue_config(queue_name)&.dig(config_name) ||
           funktor_config.dig("handlerDefaults", config_name) ||

data/lib/funktor/cli/templates/app/workers/hello_worker.rb

@@ -2,7 +2,7 @@ class HelloWorker
   include Funktor::Worker
 
   def perform(*args)
-    puts "Greetings from the HelloWorker!"
+    Funktor.logger.debug "Greetings from the HelloWorker!"
   end
 end
 

data/lib/funktor/cli/templates/funktor_config/environment.yml

@@ -9,3 +9,7 @@ FUNKTOR_INCOMING_JOB_QUEUE:
 FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
   Ref: <%= queue_name.camelize %>Queue
 <%- end -%>
+FUNKTOR_ACTIVITY_TABLE:
+  Ref: ActivityTable
+FUNKTOR_JOBS_TABLE:
+  Ref: JobsTable

data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml

@@ -0,0 +1,8 @@
+handler: lambda_event_handlers/job_activator.call
+timeout: ${self:custom.funktor.JobActivator.timeout, 30}
+reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+#reservedConcurrency: 1
+events:
+  - schedule: rate(1 minute)

data/lib/funktor/cli/templates/funktor_config/funktor.yml

@@ -21,6 +21,31 @@ IncomingJobHandler:
   visibilityTimeout: <%= incoming_config_value 'visibilityTimeout' %>
   logRetentionInDays: <%= incoming_config_value 'logRetentionInDays' %>
 
+JobActivator:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The job activator usually will be pretty fast,
+  # but we default to a high number here to allow for the times when things go weird.
+  timeout: <%= activator_config_value 'timeout' %>
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the job activator you probably don't want to limit it because you
+  # want to get things onto work queues as quickly as possible when they're ready.
+  reservedConcurrency: <%= activator_config_value 'reservedConcurrency' %>
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the job activator you probably don't need to set this
+  # since it will be running on a schedule
+  provisionedConcurrency: <%= activator_config_value 'provisionedConcurrency' %>
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the job activator you probably don't need this to be too large,
+  # but if you're seeing large delays at this stage it might help to bump it up.
+  memorySize: <%= activator_config_value 'memorySize' %>
+  batchSize: <%= activator_config_value 'batchSize' %>
+  maximumBatchingWindow: <%= activator_config_value 'maximumBatchingWindow' %>
+  visibilityTimeout: <%= activator_config_value 'visibilityTimeout' %>
+  logRetentionInDays: <%= activator_config_value 'logRetentionInDays' %>
+
+
+
 <%- queue_names.each do |queue_name| -%>
 <%= queue_name.camelize %>QueueHandler:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
@@ -61,5 +86,6 @@ DashboardName: ${self:service}-${self:custom.stage}-dashboard
 <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
 <%= queue_name.camelize %>QueueHandlerName: ${self:service}-${self:custom.stage}-<%= queue_name.camelize %>QueueHandler
 <%- end -%>
-
-
+JobsTableName: ${self:service}-${self:custom.stage}-jobs
+ActivityTableName: ${self:service}-${self:custom.stage}-activity
+JobActivatorName: ${self:service}-${self:custom.stage}-JobActivator

data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml

@@ -0,0 +1,5 @@
+Effect: Allow
+Action:
+  - dynamodb:*
+Resource:
+  - "Fn::GetAtt": [ ActivityTable, Arn ]

data/lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml

@@ -0,0 +1,5 @@
+Effect: Allow
+Action:
+  - dynamodb:*
+Resource:
+  - "Fn::GetAtt": [ JobsTable, Arn ]

data/lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml

@@ -0,0 +1,8 @@
+Effect: Allow
+Action:
+  - dynamodb:Query
+Resource:
+  Fn::Join:
+    - ""
+    - - "Fn::GetAtt": [ JobsTable, Arn ]
+      - "/index/performAtIndex"

data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml

@@ -0,0 +1,22 @@
+Resources:
+  ActivityTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      # Generate a name based on the stage
+      TableName: ${self:custom.funktor.ActivityTableName}
+      BillingMode: PAY_PER_REQUEST
+      AttributeDefinitions:
+        - AttributeName: category
+          AttributeType: S
+        - AttributeName: statName
+          AttributeType: S
+      KeySchema:
+        - AttributeName: category
+          KeyType: HASH
+        - AttributeName: statName
+          KeyType: RANGE
+
+Outputs:
+  ActivityTable:
+    Value:
+      Ref: ActivityTable