funktor 0.2.1

data/lib/funktor.rb

@@ -0,0 +1,63 @@
+require "funktor/version"
+require 'funktor/aws/sqs/event'
+require 'funktor/aws/sqs/record'
+require 'funktor/job'
+require 'funktor/worker'
+require 'funktor/middleware_chain'
+require 'funktor/incoming_job_handler'
+require 'funktor/active_job_handler'
+
+require 'json'
+
+module Funktor
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.configure_job_pusher
+    yield self
+  end
+
+  def self.job_pusher_middleware
+    @job_pusher_chain ||= MiddlewareChain.new
+    yield @job_pusher_chain if block_given?
+    @job_pusher_chain
+  end
+
+  def self.configure_active_job_handler
+    yield self
+  end
+
+  def self.active_job_handler_middleware
+    @active_job_handler_chain ||= MiddlewareChain.new
+    yield @active_job_handler_chain if block_given?
+    @active_job_handler_chain
+  end
+
+  # TODO - Maybe we don't need this either? Maybe this should be a super dumb thing that also
+  # just pushed JSON around? Maybe we want to centralize middlewares in only two spots?
+  # 1. Job pushing.
+  # 2. Job execution.
+  # 🤔
+  def self.configure_incoming_job_handler
+    yield self
+  end
+
+  def self.incoming_job_handler_middleware
+    @incoming_job_handler_chain ||= MiddlewareChain.new
+    yield @incoming_job_handler_chain if block_given?
+    @incoming_job_handler_chain
+  end
+
+  def self.parse_json(string)
+    JSON.parse(string)
+  end
+
+  def self.dump_json(object)
+    JSON.generate(object)
+  end
+end
+
+# TODO - Should we require this by default or let people opt in?
+# Is it a code smell that we need to include it at the bottom, after
+# the main Funktor module is defined?
+require 'funktor/middleware/metrics'

data/lib/funktor/active_job_handler.rb

@@ -0,0 +1,52 @@
+require 'aws-sdk-sqs'
+
+module Funktor
+  class ActiveJobHandler
+
+    def call(event:, context:)
+      event = Funktor::Aws::Sqs::Event.new(event)
+      puts "event.jobs.count = #{event.jobs.count}"
+      event.jobs.each do |job|
+        dispatch(job)
+      end
+    end
+
+    def sqs_client
+      @sqs_client ||= ::Aws::SQS::Client.new
+    end
+
+    def dispatch(job)
+      begin
+        Funktor.active_job_handler_middleware.invoke(job) do
+          job.execute
+        end
+      # rescue Funktor::Job::InvalidJsonError # TODO Make this work
+      rescue Exception => e
+        puts "Error during processing: #{$!}"
+        puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
+        attempt_retry_or_bail(job)
+      end
+    end
+
+    def attempt_retry_or_bail(job)
+      if job.can_retry
+        trigger_retry(job)
+      else
+        puts "We retried max times. We're bailing on this one."
+        puts job.to_json
+      end
+    end
+
+    def trigger_retry(job)
+      job.increment_retries
+      puts "scheduling retry # #{job.retries} with delay of #{job.delay}"
+      puts job.to_json
+      sqs_client.send_message({
+        queue_url: job.retry_queue_url,
+        message_body: job.to_json,
+        delay_seconds: job.delay
+      })
+    end
+
+  end
+end

data/lib/funktor/aws/sqs/event.rb

@@ -0,0 +1,20 @@
+module Funktor
+  module Aws
+    module Sqs
+      class Event
+        attr_accessor :event_data
+        def initialize(event_data)
+          @event_data = event_data
+        end
+
+        def records
+          @records ||= @event_data["Records"].map{|record_data| Funktor::Aws::Sqs::Record.new(record_data) }
+        end
+
+        def jobs
+          records.map(&:job)
+        end
+      end
+    end
+  end
+end

data/lib/funktor/aws/sqs/record.rb

@@ -0,0 +1,14 @@
+module Funktor
+  module Aws
+    module Sqs
+      class Record
+        attr_accessor :record_data
+        attr_accessor :job
+        def initialize(record_data)
+          @record_data = record_data
+          @job = Funktor::Job.new(record_data["body"])
+        end
+      end
+    end
+  end
+end

data/lib/funktor/cli/application.rb

@@ -0,0 +1,23 @@
+require "thor"
+
+require_relative "./bootstrap"
+require_relative "./init"
+require_relative "./generate/base"
+
+module Funktor
+  module CLI
+    class Application < Thor
+      # This makes thor report the correct exit code in the event of a failure.
+      def self.exit_on_failure?
+        true
+      end
+
+      register(Funktor::CLI::Bootstrap, "bootstrap", "bootstrap [options]", "Bootstrap a new funktor application")
+      register(Funktor::CLI::Init, "init", "init [FRAMEWORK] [options]", "Initialize a new funktor directory")
+      register(Funktor::CLI::Generate::Base, "generate", "generate GENERATOR [args] [options]", "Generate new resources")
+
+      # Set up an alias so that "funktor g" is the same as "funktor generate"
+      map "g" => :generate
+    end
+  end
+end

data/lib/funktor/cli/bootstrap.rb

@@ -0,0 +1,35 @@
+module Funktor
+  module CLI
+    class Bootstrap < Thor::Group
+      include Thor::Actions
+
+      class_option :file, :aliases => "-f",
+        :type => :string, :desc => "The bootstrap file to generate.",
+        :default => "funktor.yml"
+      class_option :directory, :aliases => "-d",
+        :type => :string, :desc => "The directory to initialize",
+        :default => "funktor"
+
+      desc <<~DESC
+        Description:
+          Bootstrap a new funktor application by generating a funktor.yml file."
+      DESC
+
+      def self.source_root
+        File.dirname(__FILE__)
+      end
+
+      def funktor_yml
+        puts funktor_file_target
+        template "templates/funktor.yml", funktor_file_target
+      end
+
+      private
+      def funktor_file_target
+        File.join options[:directory], options[:file]
+      end
+
+    end
+  end
+end
+

data/lib/funktor/cli/generate/base.rb

@@ -0,0 +1,13 @@
+require_relative "./work_queue"
+
+module Funktor
+  module CLI
+    module Generate
+      class Base < Thor
+        register(WorkQueue, "work_queue", "work_queue NAME", "Generate new work queue and lambda handler")
+      end
+    end
+  end
+end
+
+

data/lib/funktor/cli/generate/work_queue.rb

@@ -0,0 +1,25 @@
+module Funktor
+  module CLI
+    module Generate
+      class WorkQueue < Thor::Group
+
+        argument :name, :desc => "The name of the queue to generate"#, :default => "default"
+
+        def resource_yml
+          puts "queue-name.yml #{name}"
+        end
+
+        def lambda_handler
+          puts "handler.rb"
+        end
+
+        def function_definition
+          puts "function_definition.yml"
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/funktor/cli/init.rb

@@ -0,0 +1,78 @@
+module Funktor
+  module CLI
+    class Init < Thor::Group
+      include Thor::Actions
+
+      class_option :framework, :aliases => "-f",
+        :type => :string, :desc => "The deployment/provisioning framework to use.",
+        :default => "serverless"
+      class_option :directory, :aliases => "-d",
+        :type => :string, :desc => "The directory to initialize",
+        :default => "funktor"
+
+      desc <<~DESC
+        Description:
+          Initialize a new funktor deployment directory.
+      DESC
+
+      def self.source_root
+        File.join File.dirname(__FILE__), 'templates'
+      end
+
+      def self.destination_root
+        options[:directory]
+      end
+
+      def serverless_yml
+        template "serverless.yml", File.join(options[:directory], "serverless.yml")
+      end
+
+      def funktor_config_yml
+        #template "funktor_config.yml", File.join(options[:directory], "funktor_config.yml")
+        template File.join("config", "funktor.yml"), File.join(options[:directory], "config", "funktor.yml")
+        template File.join("config", "ruby_layer.yml"), File.join(options[:directory], "config", "ruby_layer.yml")
+        template File.join("config", "package.yml"), File.join(options[:directory], "config", "package.yml")
+        template File.join("config", "environment.yml"), File.join(options[:directory], "config", "environment.yml")
+      end
+
+      def package_json
+        template "package.json", File.join(options[:directory], "package.json")
+      end
+
+      def gemfile
+        template "Gemfile", File.join(options[:directory], "Gemfile")
+      end
+
+      def gitignore
+        template "gitignore", File.join(options[:directory], ".gitignore")
+      end
+
+      def resources
+        template File.join("resources", "incoming_job_queue.yml"), File.join(options[:directory], "resources", "incoming_job_queue.yml")
+        template File.join("resources", "incoming_job_queue_user.yml"), File.join(options[:directory], "resources", "incoming_job_queue_user.yml")
+        template File.join("resources", "active_job_queue.yml"), File.join(options[:directory], "resources", "active_job_queue.yml")
+        template File.join("resources", "cloudwatch_dashboard.yml"), File.join(options[:directory], "resources", "cloudwatch_dashboard.yml")
+      end
+
+      def iam_permissions
+        template File.join("iam_permissions", "ssm.yml"), File.join(options[:directory], "iam_permissions", "ssm.yml")
+        template File.join("iam_permissions", "incoming_job_queue.yml"), File.join(options[:directory], "iam_permissions", "incoming_job_queue.yml")
+        template File.join("iam_permissions", "active_job_queue.yml"), File.join(options[:directory], "iam_permissions", "active_job_queue.yml")
+      end
+
+      def function_definitions
+        template File.join("function_definitions", "active_job_handler.yml"), File.join(options[:directory], "function_definitions", "active_job_handler.yml")
+        template File.join("function_definitions", "incoming_job_handler.yml"), File.join(options[:directory], "function_definitions", "incoming_job_handler.yml")
+      end
+
+      def lambda_handlers
+        template File.join("handlers", "active_job_handler.rb"), File.join(options[:directory], "handlers", "active_job_handler.rb")
+        template File.join("handlers", "incoming_job_handler.rb"), File.join(options[:directory], "handlers", "incoming_job_handler.rb")
+      end
+
+      def workers
+        template File.join("workers", "hello_worker.rb"), File.join(options[:directory], "workers", "hello_worker.rb")
+      end
+    end
+  end
+end

data/lib/funktor/cli/templates/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+ruby '~> 2.7'
+
+source "https://gem.fury.io/jagthedrummer/" do
+  gem 'funktor'
+end
+

data/lib/funktor/cli/templates/config/environment.yml

@@ -0,0 +1,4 @@
+FUNKTOR_INCOMING_JOB_QUEUE:
+  Ref: IncomingJobQueue
+FUNKTOR_ACTIVE_JOB_QUEUE:
+  Ref: ActiveJobQueue

data/lib/funktor/cli/templates/config/funktor.yml

@@ -0,0 +1,51 @@
+incomingJobHandler:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The incoming job handler usually will be pretty fast,
+  # but we default to a high number here to allow for the times when things go weird.
+  timeout: 900
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the incoming job handler you probably don't want to limit it because you
+  # want to get things onto work queues as quickly as possible.
+  reservedConcurrency: null
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the incoming jobs handler you probably don't need to set this
+  # unless your jobs are very bursty AND very time sensitive.
+  provisionedConcurrency: null
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the incoming jobs handler you probably don't need this to be too large,
+  # but if you're seeing large delays at this stage it might help to bump it up.
+  memorySize: 256
+
+activeJobHandler:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The active job handler may be slow if your jobs are
+  # doing a lot of work, so we default to the maximum here.
+  timeout: 900
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the active job handler you may want to limit it if you have resource limitations
+  # like database connections that you need to avoid exhausting.
+  reservedConcurrency: null
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the active job handler you probably don't need to set this
+  # unless your jobs are very bursty AND very time sensitive.
+  provisionedConcurrency: null
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the active jobs handler you'll want this to be at least as large as the memory
+  # required to actually do your jobs. You can choose an even higher number to increase
+  # the available CPU to make the jobs run faster.
+  memorySize: 256
+
+
+
+# You shouldn't need to mess with these under most circumstances. But you could if you want to change
+# the name of some of your resources in AWS.
+incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
+incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead-jobs
+activeJobQueueName: ${self:service}-${self:custom.stage}-active-jobs
+activityQueueName: ${self:service}-${self:custom.stage}-activity
+activityDeadQueueName: ${self:service}-${self:custom.stage}-activity-dead
+deadJobQueueName: ${self:service}-${self:custom.stage}-dead-jobs
+dashboardName: ${self:service}-${self:custom.stage}-dashboard

data/lib/funktor/cli/templates/config/package.yml

@@ -0,0 +1,9 @@
+include:
+  - Gemfile
+  - Gemfile.lock
+  - handlers/**
+  - workers/**
+# Evertyting is excluded by default with serverless-ruby-layer, but you could use
+# the lines below to exlude files that are inside an include path.
+#exclude:
+#  - workers/excluded_worker.rb

data/lib/funktor/cli/templates/config/ruby_layer.yml

@@ -0,0 +1,11 @@
+use_docker: false
+#docker_yums:
+  #- postgresql-devel
+#native_libs:
+  #- /usr/lib64/libpq.so.5
+  #- /usr/lib64/libldap_r-2.4.so.2
+  #- /usr/lib64/liblber-2.4.so.2
+  #- /usr/lib64/libsasl2.so.3
+  #- /usr/lib64/libssl3.so
+  #- /usr/lib64/libsmime3.so
+  #- /usr/lib64/libnss3.so

data/lib/funktor/cli/templates/function_definitions/active_job_handler.yml

@@ -0,0 +1,11 @@
+handler: handlers/active_job_handler.call
+timeout: ${self:custom.funktor.activeJobHandler.timeout, 900}
+reservedConcurrency: ${self:custom.funktor.activeJobHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.activeJobHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.activeJobHandler.memorySize, 256}
+events:
+  - sqs:
+      arn:
+        Fn::GetAtt:
+          - ActiveJobQueue
+          - Arn