funktor 0.2.1 → 0.2.6

data/lib/funktor/cli/templates/config/resources/work_queue.yml

@@ -0,0 +1,22 @@
+Resources:
+  <%= work_queue_name.camelize %>Queue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueName}
+      VisibilityTimeout: 300
+      RedrivePolicy:
+        deadLetterTargetArn:
+          "Fn::GetAtt": [ <%= work_queue_name.camelize %>DeadLetterQueue, Arn ]
+        maxReceiveCount: 5
+  <%= work_queue_name.camelize %>DeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.<%= work_queue_name.camelize %>DeadJobQueueName}
+
+Outputs:
+  <%= work_queue_name.camelize %>QueueUrl:
+    Value:
+      Ref: <%= work_queue_name.camelize %>Queue
+  <%= work_queue_name.camelize %>DeadLetterQueueUrl:
+    Value:
+      Ref: <%= work_queue_name.camelize %>DeadLetterQueue

data/lib/funktor/cli/templates/funktor_init.yml.tt

@@ -0,0 +1,69 @@
+# ℹ️  WARNING: This file doesn't control anything directly. It's used to allow you to set/maintain config options in a central place.
+# When you make changes to this file you should run `funktor init` again to propagate the changes to the 'active' configuration files.
+appName: <%= name %>
+runtime: ruby2.7
+
+# These values will be applied to any funktor handlers that don't specify custom values.
+# TODO - These still need to be wired up to do anything.
+handlerDefaults:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. We default to a high number here to allow for the
+  # times when things go weird.
+  timeout: 300
+
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # Usually you'll want to leave this as null so that handlers can scale infinitely
+  # (within your account limits). If you need to restrict concurrency to prevent
+  # resource exhaustion you should do it at the queue level.
+  reservedConcurrency: null
+
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. You probably want to set this on individual queues if you're going
+  # to use it.
+  provisionedConcurrency: null
+
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # We default to 256 as a reasonable trade off on saving money but getting
+  # reasonable performance.
+  memorySize: 256
+
+  # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
+  batchSize: 10
+
+  # How many seconds should AWS wait for a batch to fill up before executing lambda?
+  # For immediate execution set the batch size to 1.
+  maximumBatchingWindow : 1
+
+  # Visibility timeout should only come into play in the case of Funktor errors.
+  # Application level errors should be handled by Funktor retry mechanisms.
+  # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
+  visibilityTimeout: 900
+
+  # Set log rentention to save money
+  logRetentionInDays: 30
+
+# Incoming Job Handler
+incomingJobHandler:
+  # If your jobs are bursty AND time-sensitive you might want to have some lambdas pre-provisioned
+  # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
+  # provisionedConcurrency: 4
+
+
+queues:
+  - default:
+      # Set queue specific config options here
+      # memorySize: 512
+      # TODO - Is it advisable to use FIFO queuues with Funktor? Maybe this isn't really even supported by CloudFormation?
+      # fifo: false
+  - singleThread:
+      reservedConcurrency: 1
+
+# TODO - Maybe this shouldn't be surfaced this early?
+# TODO - This still needs to be wired up to do anything.
+package:
+  patterns:
+    - Gemfile
+    - Gemfile.lock
+    - app/**
+    - config/**

data/lib/funktor/cli/templates/{handlers → lambda_event_handlers}/incoming_job_handler.rb

@@ -1,4 +1,4 @@
-require 'funktor'
+require_relative '../config/boot'
 
 $handler = Funktor::IncomingJobHandler.new
 

data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb

@@ -0,0 +1,8 @@
+require_relative '../config/boot'
+
+$handler = Funktor::ActiveJobHandler.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/lib/funktor/cli/templates/package.json

@@ -1 +1,8 @@
-{"name":"serverless-sub","description":"","version":"0.1.0","devDependencies":{"serverless-ruby-layer":"^1.4.0"}}
+{
+  "name":"<%= name %>",
+  "description":"",
+  "version":"0.1.0",
+  "devDependencies":{
+    "serverless-ruby-layer":"^1.4.0"
+  }
+}

data/lib/funktor/cli/templates/serverless.yml

@@ -12,7 +12,7 @@
 # Happy Coding!
 
 # The name of your service. All your AWS resources will contain this name.
-service: yourapp-funktor
+service: <%= name %>
 
 # This causes serverless to throw an error early if the config is bad, instead of waiting for CloudFormation to try and fail to deploy it.
 configValidationMode: error
@@ -22,13 +22,14 @@ frameworkVersion: '2'
 
 provider:
   name: aws
-  runtime: ruby2.7
+  runtime: <%= runtime %>
   lambdaHashingVersion: 20201221
   environment: ${file(config/environment.yml)}
+  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   iamRoleStatements:
-    - ${file(iam_permissions/ssm.yml)}
-    - ${file(iam_permissions/active_job_queue.yml)}
-    - ${file(iam_permissions/incoming_job_queue.yml)}
+    <%- all_iam_permissions.each do |iam_permission| -%>
+    - ${file(<%= iam_permission %>)}
+    <%- end -%>
 
 
 custom:
@@ -41,14 +42,14 @@ custom:
 package: ${file(config/package.yml)}
 
 functions:
-  incomingJobHandler: ${file(function_definitions/incoming_job_handler.yml)}
-  activeJobHandler: ${file(function_definitions/active_job_handler.yml)}
+  <%- all_function_definitions.each do |function_definition| -%>
+  <%= File.basename(function_definition, ".yml").camelize %>: ${file(<%= function_definition %>)}
+  <%- end -%>
 
 resources:
-  - ${file(resources/active_job_queue.yml)}
-  - ${file(resources/incoming_job_queue.yml)}
-  - ${file(resources/incoming_job_queue_user.yml)}
-  - ${file(resources/cloudwatch_dashboard.yml)}
+  <%- all_resources.each do |resource| -%>
+  - ${file(<%= resource %>)}
+  <%- end -%>
 
 plugins:
-  - /Users/jgreen/projects/serverless-ruby-layer
+  - serverless-ruby-layer

data/lib/funktor/incoming_job_handler.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-sqs'
+require 'active_support/core_ext/string/inflections'
 
 module Funktor
   class IncomingJobHandler
@@ -26,10 +27,19 @@ module Funktor
       ENV['FUNKTOR_ACTIVE_JOB_QUEUE']
     end
 
+    def queue_for_job(job)
+      queue_name = job.queue || 'default'
+      queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
+      puts "queue_constant = #{queue_constant}"
+      puts "ENV value = #{ENV[queue_constant]}"
+      ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
+    end
+
     def push_to_active_job_queue(job)
+      puts "job = #{job.to_json}"
       sqs_client.send_message({
         # TODO : How to get this URL...
-        queue_url: active_job_queue,
+        queue_url: queue_for_job(job),
         message_body: job.to_json,
         delay_seconds: job.delay
       })

data/lib/funktor/job.rb

@@ -10,6 +10,10 @@ module Funktor
       @job_data ||= Funktor.parse_json(job_string)
     end
 
+    def queue
+      job_data["queue"]
+    end
+
     def worker_class_name
       job_data["worker"]
     end

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.1"
+  VERSION = "0.2.6"
 end

data/lib/funktor/worker.rb

@@ -28,6 +28,10 @@ module Funktor::Worker
       get_funktor_options[:queue_url]
     end
 
+    def custom_queue
+      get_funktor_options[:queue]
+    end
+
     def queue_url
       # TODO : Should this default to FUNKTOR_ACTIVE_JOB_QUEUE?
       # Depends how e handle this in pro...?
@@ -73,10 +77,15 @@ module Funktor::Worker
       @client ||= Aws::SQS::Client.new
     end
 
+    def work_queue
+      (self.custom_queue || 'default').to_s
+    end
+
     def build_job_payload(job_id, delay, *worker_params)
       {
         worker: self.name,
         worker_params: worker_params,
+        queue: self.work_queue,
         job_id: job_id,
         delay: delay,
         funktor_options: get_funktor_options

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.6
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-04 00:00:00.000000000 Z
+date: 2021-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -89,6 +89,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -114,29 +115,27 @@ files:
 - lib/funktor/cli/generate/work_queue.rb
 - lib/funktor/cli/init.rb
 - lib/funktor/cli/templates/Gemfile
+- lib/funktor/cli/templates/app/workers/hello_worker.rb
+- lib/funktor/cli/templates/config/boot.rb
 - lib/funktor/cli/templates/config/environment.yml
+- lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml
+- lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml
 - lib/funktor/cli/templates/config/funktor.yml
+- lib/funktor/cli/templates/config/iam_permissions/incoming_job_queue.yml
+- lib/funktor/cli/templates/config/iam_permissions/ssm.yml
+- lib/funktor/cli/templates/config/iam_permissions/work_queue.yml
 - lib/funktor/cli/templates/config/package.yml
+- lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml
+- lib/funktor/cli/templates/config/resources/incoming_job_queue.yml
+- lib/funktor/cli/templates/config/resources/incoming_job_queue_user.yml
+- lib/funktor/cli/templates/config/resources/work_queue.yml
 - lib/funktor/cli/templates/config/ruby_layer.yml
-- lib/funktor/cli/templates/function_definitions/active_job_handler.yml
-- lib/funktor/cli/templates/function_definitions/incoming_job_handler.yml
-- lib/funktor/cli/templates/funktor.yml.tt
+- lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
-- lib/funktor/cli/templates/handlers/active_job_handler.rb
-- lib/funktor/cli/templates/handlers/incoming_job_handler.rb
-- lib/funktor/cli/templates/iam_permissions/active_job_queue.yml
-- lib/funktor/cli/templates/iam_permissions/incoming_job_queue.yml
-- lib/funktor/cli/templates/iam_permissions/ssm.yml
+- lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
+- lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/package.json
-- lib/funktor/cli/templates/resources/active_job_queue.yml
-- lib/funktor/cli/templates/resources/cloudwatch_dashboard.yml
-- lib/funktor/cli/templates/resources/incoming_job_queue.yml
-- lib/funktor/cli/templates/resources/incoming_job_queue_user.yml
 - lib/funktor/cli/templates/serverless.yml
-- lib/funktor/cli/templates/workers/hello_worker.rb
-- lib/funktor/deploy/cli.rb
-- lib/funktor/deploy/serverless.rb
-- lib/funktor/deploy/serverless_templates/serverless.yml
 - lib/funktor/fake_job_queue.rb
 - lib/funktor/incoming_job_handler.rb
 - lib/funktor/job.rb

data/lib/funktor/cli/templates/function_definitions/active_job_handler.yml

@@ -1,11 +0,0 @@
-handler: handlers/active_job_handler.call
-timeout: ${self:custom.funktor.activeJobHandler.timeout, 900}
-reservedConcurrency: ${self:custom.funktor.activeJobHandler.reservedConcurrency, null}
-provisionedConcurrency: ${self:custom.funktor.activeJobHandler.provisionedConcurrency, null}
-memorySize: ${self:custom.funktor.activeJobHandler.memorySize, 256}
-events:
-  - sqs:
-      arn:
-        Fn::GetAtt:
-          - ActiveJobQueue
-          - Arn

data/lib/funktor/cli/templates/funktor.yml.tt

@@ -1,51 +0,0 @@
-# ℹ️  WARNING: This file doesn't control anything directly. It's used to allow you to set/maintain config options in a central place.
-# When you make changes to this file you should run `funktor init` again to propagate the changes to the 'active' configuration files.
-runtime: ruby2.7
-
-# Incoming Job Handler
-incoming_job_handler:
-  # Use memory_size to adjust the reousrces (both memory and CPU) available
-  # memory_size: 512
-  timeout_in_seconds: 20 # This handler might have to handle several incoming jobs at once
-  # WARNING : You probably don't want to limit concurrency on incoming jobs.
-  # concurrency: 100
-
-# Delayed Job Activator
-delayed_job_activator:
-  # Use memory_size to adjust the reousrces (both memory and CPU) available
-  # memory_size: 512
-  execution_schedule: rate(1 minute)
-  activation_window_in_seconds: 120 # Activate any jobs scheduled for the next two minutes
-  timeout_in_seconds: 300 # Allow an activation job to run for up to 5 minutes
-  concurrency: 1
-
-
-queues:
-  default:
-    # Use memory_size to adjust the reousrces (both memory and CPU) available
-    # memory_size: 512
-    # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
-    # batch_size: 10
-    # How many seconds should AWS wait for a batch to fill up before executing lambda?
-    # For immediate execution set the batch size to 1.
-    # maximumBatchingWindow : 1
-    # A single handler can receive up to batch_size jobs at a time. Make sure timeout is long enough.
-    timeout_in_seconds: 300
-    # You might want to limit concurrency of executing jobs to stay within resource limits (like DB connections).
-    # concurrency: 10
-    # Visibility timeout should only come into play in the case of Funktor errors.
-    # Application level errors should be handled by Funktor retry mechanisms.
-    # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
-    # visibility_timeout: 300
-    # TODO - Is it advisable to use FIFO queuues with Funktor?
-    # TODO - Maybe this isn't really even supported by CloudFormation?
-    # fifo: false
-
-# TODO - Maybe this is handled in the Dockerfile?
-package:
-  patterns:
-    - Gemfile
-    - Gemfile.lock
-    - ../app/**
-    - ../config/**
-    - ../lambda_handlers/**

data/lib/funktor/cli/templates/handlers/active_job_handler.rb

@@ -1,17 +0,0 @@
-require 'funktor'
-
-# Bundler is hard to make work because AWS includes some gems in the basic ruby runtime.
-# We're probably going to need to use containers...
-#require "rubygems"
-#require "bundler/setup"
-#Bundler.require(:default)
-
-# TODO : Ideally this wouldn't be needed
-require_relative '../workers/hello_worker'
-
-$handler = Funktor::ActiveJobHandler.new
-
-def call(event:, context:)
-  $handler.call(event: event, context: context)
-end
-

data/lib/funktor/cli/templates/resources/active_job_queue.yml

@@ -1,22 +0,0 @@
-Resources:
-  ActiveJobQueue:
-    Type: AWS::SQS::Queue
-    Properties:
-      QueueName: ${self:custom.funktor.activeJobQueueName}
-      VisibilityTimeout: 300
-      RedrivePolicy:
-        deadLetterTargetArn:
-          "Fn::GetAtt": [ ActiveJobDeadLetterQueue, Arn ]
-        maxReceiveCount: 5
-  ActiveJobDeadLetterQueue:
-    Type: AWS::SQS::Queue
-    Properties:
-      QueueName: ${self:custom.funktor.deadJobQueueName}
-
-Outputs:
-  ActiveJobQueueUrl:
-    Value:
-      Ref: ActiveJobQueue
-  ActiveJobDeadLetterQueueUrl:
-    Value:
-      Ref: ActiveJobDeadLetterQueue

data/lib/funktor/deploy/cli.rb

@@ -1,42 +0,0 @@
-require 'optparse'
-require 'funktor/deploy/serverless'
-
-module Funktor
-  module Deploy
-    class CLI
-      attr_reader :options
-      def initialize
-        @options = {
-          verbose: false,
-          file: 'funktor.yml',
-          tmp_dir_prefix: '.funktor',
-          stage: 'dev'
-        }
-      end
-
-      def parse(argv = ARGV)
-        OptionParser.new do |opts|
-          opts.on('-v', '--verbose', 'Display verbose output') do |verbose|
-            options[:verbose] = verbose
-          end
-          opts.on('-f', '--file=FILE', 'The path to the funktor.yml file to deploy') do |file|
-            options[:file] = file
-          end
-          opts.on('-t', '--tmp_dir_prefix=TMP_DIR_PREFIX', 'The prefix for the tmp dir. The stage will be appended.') do |tmp_dir_prefix|
-            options[:tmp_dir_prefix] = tmp_dir_prefix
-          end
-          opts.on('-s', '--stage=STAGE', 'The stage to deploy to. Defaults to "dev"') do |stage|
-            options[:stage] = stage
-          end
-          opts.on('-h') { puts opts; exit }
-          opts.parse!(argv)
-        end
-      end
-
-      def run
-        Funktor::Deploy::Serverless.new(**options).call
-      end
-    end
-  end
-end
-