funktor 0.2.1

data/lib/funktor/cli/templates/resources/incoming_job_queue.yml

@@ -0,0 +1,22 @@
+Resources:
+  IncomingJobQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.incomingJobQueueName}
+      VisibilityTimeout: 300
+      RedrivePolicy:
+        deadLetterTargetArn:
+          "Fn::GetAtt": [ IncomingJobDeadLetterQueue, Arn ]
+        maxReceiveCount: 5
+  IncomingJobDeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.incomingDeadJobQueueName}
+
+Outputs:
+  IncomingJobQueueUrl:
+    Value:
+      Ref: IncomingJobQueue
+  IncomingJobDeadLetterQueueUrl:
+    Value:
+      Ref: IncomingJobDeadLetterQueue

data/lib/funktor/cli/templates/resources/incoming_job_queue_user.yml

@@ -0,0 +1,26 @@
+Resources:
+  IncomingJobQueueUser:
+    Type: AWS::IAM::User
+    Properties:
+      Policies:
+        - PolicyName: incoming-job-queue-access
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - sqs:*
+              Resource:
+                - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
+
+  IncomingJobQueueUserAccessKey:
+    Type: AWS::IAM::AccessKey
+    Properties:
+      UserName: !Ref IncomingJobQueueUser
+
+
+Outputs:
+  AccessKeyID:
+    Value: !Ref IncomingJobQueueUserAccessKey
+  SecretAccessKey:
+    Value: !GetAtt IncomingJobQueueUserAccessKey.SecretAccessKey

data/lib/funktor/cli/templates/serverless.yml

@@ -0,0 +1,54 @@
+# Welcome to Funktor & Serverless!
+#
+# This file is the main config file for your service.
+# It's already configured to run Funktor, you just have to deploy it.
+#
+# For more info about Funktor:
+#    TODO
+#
+# For more about serverless, check their docs:
+#    docs.serverless.com
+#
+# Happy Coding!
+
+# The name of your service. All your AWS resources will contain this name.
+service: yourapp-funktor
+
+# This causes serverless to throw an error early if the config is bad, instead of waiting for CloudFormation to try and fail to deploy it.
+configValidationMode: error
+
+# Pin the serverless framework to the 2.x line
+frameworkVersion: '2'
+
+provider:
+  name: aws
+  runtime: ruby2.7
+  lambdaHashingVersion: 20201221
+  environment: ${file(config/environment.yml)}
+  iamRoleStatements:
+    - ${file(iam_permissions/ssm.yml)}
+    - ${file(iam_permissions/active_job_queue.yml)}
+    - ${file(iam_permissions/incoming_job_queue.yml)}
+
+
+custom:
+  # Our stage is based on what is passed in when running serverless
+  # commands. Or fallsback to what we have set in the provider section.
+  stage: ${self:provider.stage, 'dev'}
+  funktor: ${file(config/funktor.yml)}
+  rubyLayer: ${file(config/ruby_layer.yml)}
+
+package: ${file(config/package.yml)}
+
+functions:
+  incomingJobHandler: ${file(function_definitions/incoming_job_handler.yml)}
+  activeJobHandler: ${file(function_definitions/active_job_handler.yml)}
+
+resources:
+  - ${file(resources/active_job_queue.yml)}
+  - ${file(resources/incoming_job_queue.yml)}
+  - ${file(resources/incoming_job_queue_user.yml)}
+  - ${file(resources/cloudwatch_dashboard.yml)}
+
+plugins:
+  - /Users/jgreen/projects/serverless-ruby-layer

data/lib/funktor/cli/templates/workers/hello_worker.rb

@@ -0,0 +1,8 @@
+class HelloWorker
+  include Funktor::Worker
+
+  def perform(*args)
+    puts "Greetings from the HelloWorker!"
+  end
+end
+

data/lib/funktor/deploy/cli.rb

@@ -0,0 +1,42 @@
+require 'optparse'
+require 'funktor/deploy/serverless'
+
+module Funktor
+  module Deploy
+    class CLI
+      attr_reader :options
+      def initialize
+        @options = {
+          verbose: false,
+          file: 'funktor.yml',
+          tmp_dir_prefix: '.funktor',
+          stage: 'dev'
+        }
+      end
+
+      def parse(argv = ARGV)
+        OptionParser.new do |opts|
+          opts.on('-v', '--verbose', 'Display verbose output') do |verbose|
+            options[:verbose] = verbose
+          end
+          opts.on('-f', '--file=FILE', 'The path to the funktor.yml file to deploy') do |file|
+            options[:file] = file
+          end
+          opts.on('-t', '--tmp_dir_prefix=TMP_DIR_PREFIX', 'The prefix for the tmp dir. The stage will be appended.') do |tmp_dir_prefix|
+            options[:tmp_dir_prefix] = tmp_dir_prefix
+          end
+          opts.on('-s', '--stage=STAGE', 'The stage to deploy to. Defaults to "dev"') do |stage|
+            options[:stage] = stage
+          end
+          opts.on('-h') { puts opts; exit }
+          opts.parse!(argv)
+        end
+      end
+
+      def run
+        Funktor::Deploy::Serverless.new(**options).call
+      end
+    end
+  end
+end
+

data/lib/funktor/deploy/serverless.rb

@@ -0,0 +1,60 @@
+require 'yaml'
+
+module Funktor
+  module Deploy
+    class Serverless
+      attr_accessor :file, :tmp_dir_prefix, :verbose, :stage
+      def initialize(file:, tmp_dir_prefix:, verbose:, stage:)
+        @file = file
+        @tmp_dir_prefix = tmp_dir_prefix
+        @verbose = verbose
+        @stage = stage
+      end
+
+      def call
+        #puts "deploying file #{file} via tmp_dir_prefix #{tmp_dir_prefix} for stage #{stage}"
+        make_tmp_dir
+        create_serverless_file
+      end
+
+      def funktor_data
+        @funktor_data ||= squash_hash(YAML.load_file(file))
+      end
+
+      def squash_hash(hsh, stack=[])
+        hsh.reduce({}) do |res, (key, val)|
+          next_stack = [ *stack, key ]
+          if val.is_a?(Hash)
+            next res.merge(squash_hash(val, next_stack))
+          end
+          res.merge(next_stack.join(".").to_sym => val)
+        end
+      end
+
+      def make_tmp_dir
+        FileUtils.mkdir_p tmp_dir
+      end
+
+      def tmp_dir
+        "#{tmp_dir_prefix}_#{stage}"
+      end
+
+      def create_serverless_file
+        #puts "funktor_data = "
+        #puts funktor_data
+        template_source = File.open(serverless_file_source).read
+        file_content = template_source % funktor_data
+        File.open(serverless_file_destination, 'w') { |file| file.write(file_content) }
+      end
+
+      def serverless_file_source
+        File.expand_path("../serverless_templates/serverless.yml", __FILE__)
+      end
+
+      def serverless_file_destination
+        File.join tmp_dir, 'serverless.yml'
+      end
+    end
+  end
+end
+

data/lib/funktor/deploy/serverless_templates/serverless.yml

@@ -0,0 +1,156 @@
+# WARNING : You probably don't want to mess with this file directly.
+service: %{stack_name}
+# app and org for use with dashboard.serverless.com
+#app: your-app-name
+#org: your-org-name
+
+frameworkVersion: '2'
+
+provider:
+  name: aws
+  runtime: %{runtime}
+  lambdaHashingVersion: 20201221
+  ecr:
+    images:
+      funktorimage:
+        path: ./
+        # TODO : Expose buildArgs via funktor.yml?
+        buildArgs:
+          BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
+  # TODO : Expose environment to funktor.yml?
+  environment:
+    FUNKTOR_INCOMING_JOB_QUEUE:
+      Ref: IncomingJobQueue
+    FUNKTOR_ACTIVE_JOB_QUEUE:
+      Ref: ActiveJobQueue
+    FUNKTOR_DELAYED_JOB_TABLE:
+      Ref: DelayedJobTable
+    delayedJobTable:
+      Ref: DelayedJobTable
+  iamRoleStatements:
+    - Effect: Allow
+      Action:
+        - ssm:Get*
+      Resource:
+        - '*' # TODO : This should probably be more selective...
+    - Effect: Allow
+      Action:
+        - sqs:ReceiveMessage
+        - sqs:DeleteMessage
+        - sqs:SendMessage
+        - sqs:GetQueueAttributes
+      Resource:
+        - "Fn::GetAtt": [ ActiveJobQueue, Arn ]
+    - Effect: Allow
+      Action:
+        - sqs:ReceiveMessage
+        - sqs:DeleteMessage
+        - sqs:SendMessage
+        - sqs:GetQueueAttributes
+      Resource:
+        - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
+    - Effect: Allow
+      Action:
+        - sqs:ReceiveMessage
+        - sqs:DeleteMessage
+        - sqs:SendMessage
+        - sqs:GetQueueAttributes
+      Resource:
+        - "Fn::GetAtt": [ ActivityQueue, Arn ]
+    - Effect: Allow
+      Action:
+        - dynamodb:PutItem
+        - dynamodb:DeleteItem
+      Resource:
+        - "Fn::GetAtt": [ DelayedJobTable, Arn ]
+    - Effect: Allow
+      Action:
+        - dynamodb:*
+      Resource:
+        - "Fn::GetAtt": [ StatsTable, Arn ]
+    - Effect: Allow
+      Action:
+        - dynamodb:Query
+      Resource:
+        Fn::Join:
+          - ""
+          - - "Fn::GetAtt": [ DelayedJobTable, Arn ]
+            - "/index/performAtIndex"
+
+custom:
+  # Our stage is based on what is passed in when running serverless
+  # commands. Or fallsback to what we have set in the provider section.
+  stage: ${opt:stage, 'dev'}
+  incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+  incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
+  incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead-jobs
+  activeJobQueueName: ${self:service}-${self:custom.stage}-active-jobs
+  activityQueueName: ${self:service}-${self:custom.stage}-activity
+  activityDeadQueueName: ${self:service}-${self:custom.stage}-activity-dead
+  deadJobQueueName: ${self:service}-${self:custom.stage}-dead-jobs
+  delayedJobTableName: ${self:service}-${self:custom.stage}-delayed-jobs
+  statsTableName: ${self:service}-${self:custom.stage}-stats
+  dashboardName: ${self:service}-${self:custom.stage}-dashboard
+
+# you can define service wide environment variables here
+#  environment:
+#    variable1: value1
+
+functions:
+  # TODO - How could other functions be passed in from funktor.yml?
+  #random_job_generator:
+    ##handler: lambda_handlers/random_job_generator.RandomJobGenerator.call
+    #image:
+      #name: funktorimage
+      #command:
+        #- lambda_handlers/random_job_generator.RandomJobGenerator.call
+    #timeout: 170
+    #reservedConcurrency: 0
+    #events:
+      #- schedule: rate(1 minute)
+
+  delayed_job_activator:
+    image:
+      name: funktorimage
+      command:
+        - lambda_handlers/delayed_job_activator.call
+    timeout: %{delayed_job_activator.timeout_in_seconds}
+    # TODO - handle memory and concurrency
+    #reservedConcurrency: 1
+    events:
+      - schedule: %{delayed_job_activator.execution_schedule}
+
+  incoming_job_handler:
+    image:
+      name: funktorimage
+      command:
+        - lambda_handlers/incoming_job_handler.call
+    timeout: %{incoming_job_handler.timeout_in_seconds}
+    events:
+      - sqs:
+          arn:
+            Fn::GetAtt:
+              - IncomingJobQueue
+              - Arn
+
+  # TODO - We need one of these per work queue
+  active_job_handler:
+    image:
+      name: funktorimage
+      command:
+        - lambda_handlers/active_job_handler.call
+    timeout: 300
+    events:
+      - sqs:
+          arn:
+            Fn::GetAtt:
+              - ActiveJobQueue
+              - Arn
+
+# you can add CloudFormation resource templates here
+resources:
+  - ${file(resources/sqs-queue.yml)}
+  - ${file(resources/sqs-incoming-user.yml)}
+  - ${file(resources/dynamodb-table.yml)}
+  - ${file(resources/cloudfront-dashboard.yml)}
+

data/lib/funktor/fake_job_queue.rb

@@ -0,0 +1,15 @@
+module Funktor
+  module FakeJobQueue
+    def self.push(worker, payload)
+      jobs[worker.name].push({worker: worker, payload: payload})
+    end
+
+    def self.jobs
+      @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    def self.clear_all
+      jobs.clear
+    end
+  end
+end

data/lib/funktor/incoming_job_handler.rb

@@ -0,0 +1,39 @@
+require 'aws-sdk-sqs'
+
+module Funktor
+  class IncomingJobHandler
+
+    def call(event:, context:)
+      event = Funktor::Aws::Sqs::Event.new(event)
+      puts "event.jobs.count = #{event.jobs.count}"
+      event.jobs.each do |job|
+        dispatch(job)
+      end
+    end
+
+    def sqs_client
+      @sqs_client ||= ::Aws::SQS::Client.new
+    end
+
+    def dispatch(job)
+      Funktor.incoming_job_handler_middleware.invoke(job) do
+        puts "pushing to active_job_queue for delay = #{job.delay}"
+        push_to_active_job_queue(job)
+      end
+    end
+
+    def active_job_queue
+      ENV['FUNKTOR_ACTIVE_JOB_QUEUE']
+    end
+
+    def push_to_active_job_queue(job)
+      sqs_client.send_message({
+        # TODO : How to get this URL...
+        queue_url: active_job_queue,
+        message_body: job.to_json,
+        delay_seconds: job.delay
+      })
+    end
+
+  end
+end

data/lib/funktor/job.rb

@@ -0,0 +1,76 @@
+module Funktor
+  class Job
+    attr_accessor :job_string
+    attr_accessor :job_data
+    def initialize(job_string)
+      @job_string = job_string
+    end
+
+    def job_data
+      @job_data ||= Funktor.parse_json(job_string)
+    end
+
+    def worker_class_name
+      job_data["worker"]
+    end
+
+    def job_id
+      job_data["job_id"]
+    end
+
+    def worker_params
+      job_data["worker_params"]
+    end
+
+    def retries
+      job_data["retries"] || 0
+    end
+
+    def retries=(retries)
+      job_data["retries"] = retries
+    end
+
+    def delay
+      job_data["delay"]
+    end
+
+    def delay=(delay)
+      job_data["delay"] = delay
+    end
+
+    def execute
+      worker_class.new.perform(worker_params)
+    end
+
+    def worker_class
+      @klass ||= Object.const_get worker_class_name
+    end
+
+    def increment_retries
+      self.retries ||= 0
+      self.retries += 1
+      self.delay = seconds_to_delay(retries)
+    end
+
+    # delayed_job and sidekiq use the same basic formula
+    def seconds_to_delay(count)
+      (count**4) + 15 + (rand(30) * (count + 1))
+    end
+
+    def to_json(arg = nil)
+      Funktor.dump_json(job_data)
+    end
+
+    def retry_limit
+      25
+    end
+
+    def can_retry
+      self.retries < retry_limit
+    end
+
+    def retry_queue_url
+      worker_class&.custom_queue_url || ENV['FUNKTOR_INCOMING_JOB_QUEUE']
+    end
+  end
+end