funktor 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 154286564dee4cc895ef3515dcacfa515a91ecde743862e36c833d6db0389447
-  data.tar.gz: e1786a3bdda966cd7d1fdda07ae3faf12f3ec2ca27b51506257520b3d13b02dd
+  metadata.gz: 38d383dcdafdb71a93c3ab881ec7772e8694fe0a8ca913a11a27ab634b068349
+  data.tar.gz: 92e17fab7ca35700c1721dbd9f674d14a1dd8a033486ed1af9d65236cad305cc
 SHA512:
-  metadata.gz: c6b7de2400aceac0dcad12d2ad4093a07be595a120f88c58eddf7fa63f3a353e7637c47963cca8e79263b1ec29838ee5c26147a0e924cc7508092ac968e104b2
-  data.tar.gz: c5cf1bdea5c9ad86415d7da9350b28699900b623a05d4860ecfb11c77daa773fbdd4b1c33a166cfa7c7025a238c268d3748b8a9b9989043e0f29cae6a8446d99
+  metadata.gz: 1643f95d5f02f83272c2304e6c22cda5c91bb39f4f109cd4d363d9b667294679dbfa4634962162193dd9c2ea90da57f21048a41d2b5710a80f19c91631136f8d
+  data.tar.gz: 559c5693bf8a958b22e0b6b189f72ca43d758701498ccb85c52c3cf3a9e7c2b148d26d17efdea72089328e393c1ad671b3d03b6f8dcee1ac39498fc6cf6f7535

data/.github/workflows/ruby.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -10,6 +10,6 @@
 # rspec failure tracking
 .rspec_status
 
-# The funktor dir isn't a real part of the gem, it's where things
+# The funktor app dir isn't a real part of the gem, it's where things
 # get generated as I'm testing the gem locally. It doesn't need to be in git.
-/funktor/
+/myapp-funktor/

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.1)
+    funktor (0.2.2)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor
@@ -41,7 +41,7 @@ GEM
     minitest (5.14.4)
     public_suffix (4.0.6)
     rake (12.3.3)
-    rexml (3.2.4)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)

data/funktor.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'aws-sdk-sqs', '~> 1.37'
   spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
-  spec.add_dependency "thor" # TODO - Can we build our own verison of cattr_accessor to avoid this?
+  spec.add_dependency "thor" # Thor drives the CLI
 
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'webmock'

data/lib/funktor.rb

@@ -11,7 +11,6 @@ require 'json'
 
 module Funktor
   class Error < StandardError; end
-  # Your code goes here...
 
   def self.configure_job_pusher
     yield self

data/lib/funktor/cli/bootstrap.rb

@@ -3,16 +3,15 @@ module Funktor
     class Bootstrap < Thor::Group
       include Thor::Actions
 
+      argument :name, :type => :string, :desc => "The name of the app to initialize"
+
       class_option :file, :aliases => "-f",
         :type => :string, :desc => "The bootstrap file to generate.",
-        :default => "funktor.yml"
-      class_option :directory, :aliases => "-d",
-        :type => :string, :desc => "The directory to initialize",
-        :default => "funktor"
+        :default => "funktor_init.yml"
 
       desc <<~DESC
         Description:
-          Bootstrap a new funktor application by generating a funktor.yml file."
+          Bootstrap a new funktor application by generating a funktor_init.yml file."
       DESC
 
       def self.source_root
@@ -20,13 +19,14 @@ module Funktor
       end
 
       def funktor_yml
+        # TODO - Should we camelize the app name before writing it into the config? (CloudFormation names get weird with underscores and dashes.)
         puts funktor_file_target
-        template "templates/funktor.yml", funktor_file_target
+        template "templates/funktor_init.yml", funktor_file_target
       end
 
       private
       def funktor_file_target
-        File.join options[:directory], options[:file]
+        File.join name, options[:file]
       end
 
     end

data/lib/funktor/cli/init.rb

@@ -1,14 +1,21 @@
+require 'yaml'
+require 'active_support/core_ext/string/inflections'
+
 module Funktor
   module CLI
     class Init < Thor::Group
       include Thor::Actions
 
-      class_option :framework, :aliases => "-f",
+      attr_accessor :work_queue_name
+      attr_accessor :work_queue_config
+
+      class_option :deployment_framework, :aliases => "-d",
         :type => :string, :desc => "The deployment/provisioning framework to use.",
         :default => "serverless"
-      class_option :directory, :aliases => "-d",
-        :type => :string, :desc => "The directory to initialize",
-        :default => "funktor"
+
+      class_option :file, :aliases => "-f",
+        :type => :string, :desc => "The funktor init file.",
+        :default => "funktor.yml"
 
       desc <<~DESC
         Description:
@@ -20,59 +27,106 @@ module Funktor
       end
 
       def self.destination_root
-        options[:directory]
+        name
       end
 
       def serverless_yml
-        template "serverless.yml", File.join(options[:directory], "serverless.yml")
+        template "serverless.yml", File.join("serverless.yml")
       end
 
       def funktor_config_yml
-        #template "funktor_config.yml", File.join(options[:directory], "funktor_config.yml")
-        template File.join("config", "funktor.yml"), File.join(options[:directory], "config", "funktor.yml")
-        template File.join("config", "ruby_layer.yml"), File.join(options[:directory], "config", "ruby_layer.yml")
-        template File.join("config", "package.yml"), File.join(options[:directory], "config", "package.yml")
-        template File.join("config", "environment.yml"), File.join(options[:directory], "config", "environment.yml")
+        #template "funktor_config.yml", File.join("funktor_config.yml")
+        template File.join("config", "funktor.yml"), File.join("config", "funktor.yml")
+        template File.join("config", "ruby_layer.yml"), File.join("config", "ruby_layer.yml")
+        template File.join("config", "package.yml"), File.join("config", "package.yml")
+        template File.join("config", "environment.yml"), File.join("config", "environment.yml")
       end
 
       def package_json
-        template "package.json", File.join(options[:directory], "package.json")
+        template "package.json", File.join("package.json")
       end
 
       def gemfile
-        template "Gemfile", File.join(options[:directory], "Gemfile")
+        template "Gemfile", File.join("Gemfile")
       end
 
       def gitignore
-        template "gitignore", File.join(options[:directory], ".gitignore")
+        template "gitignore", File.join(".gitignore")
       end
 
       def resources
-        template File.join("resources", "incoming_job_queue.yml"), File.join(options[:directory], "resources", "incoming_job_queue.yml")
-        template File.join("resources", "incoming_job_queue_user.yml"), File.join(options[:directory], "resources", "incoming_job_queue_user.yml")
-        template File.join("resources", "active_job_queue.yml"), File.join(options[:directory], "resources", "active_job_queue.yml")
-        template File.join("resources", "cloudwatch_dashboard.yml"), File.join(options[:directory], "resources", "cloudwatch_dashboard.yml")
+        template File.join("config", "resources", "incoming_job_queue.yml"), File.join("config", "resources", "incoming_job_queue.yml")
+        template File.join("config", "resources", "incoming_job_queue_user.yml"), File.join("config", "resources", "incoming_job_queue_user.yml")
+        # TODO - Figure out how to make the dashboard aware of various queues...
+        template File.join("config", "resources", "cloudwatch_dashboard.yml"), File.join("config", "resources", "cloudwatch_dashboard.yml")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("config", "resources", "work_queue.yml"), File.join("config", "resources", "#{work_queue_name.underscore}_queue.yml")
+        end
       end
 
       def iam_permissions
-        template File.join("iam_permissions", "ssm.yml"), File.join(options[:directory], "iam_permissions", "ssm.yml")
-        template File.join("iam_permissions", "incoming_job_queue.yml"), File.join(options[:directory], "iam_permissions", "incoming_job_queue.yml")
-        template File.join("iam_permissions", "active_job_queue.yml"), File.join(options[:directory], "iam_permissions", "active_job_queue.yml")
+        template File.join("config", "iam_permissions", "ssm.yml"), File.join("config", "iam_permissions", "ssm.yml")
+        template File.join("config", "iam_permissions", "incoming_job_queue.yml"), File.join("config", "iam_permissions", "incoming_job_queue.yml")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("config", "iam_permissions", "work_queue.yml"), File.join("config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
+        end
       end
 
       def function_definitions
-        template File.join("function_definitions", "active_job_handler.yml"), File.join(options[:directory], "function_definitions", "active_job_handler.yml")
-        template File.join("function_definitions", "incoming_job_handler.yml"), File.join(options[:directory], "function_definitions", "incoming_job_handler.yml")
+        template File.join("config", "function_definitions", "incoming_job_handler.yml"), File.join("config", "function_definitions", "incoming_job_handler.yml")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("config", "function_definitions", "work_queue_handler.yml"), File.join("config", "function_definitions", "#{work_queue_name.underscore}_queue_handler.yml")
+        end
       end
 
       def lambda_handlers
-        template File.join("handlers", "active_job_handler.rb"), File.join(options[:directory], "handlers", "active_job_handler.rb")
-        template File.join("handlers", "incoming_job_handler.rb"), File.join(options[:directory], "handlers", "incoming_job_handler.rb")
+        template File.join("app", "handlers", "incoming_job_handler.rb"), File.join("app", "handlers", "incoming_job_handler.rb")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("app", "handlers", "work_queue_handler.rb"), File.join("app", "handlers", "#{work_queue_name.underscore}_queue_handler.rb")
+        end
       end
 
       def workers
-        template File.join("workers", "hello_worker.rb"), File.join(options[:directory], "workers", "hello_worker.rb")
+        template File.join("app", "workers", "hello_worker.rb"), File.join("app", "workers", "hello_worker.rb")
+      end
+
+      private
+      def funktor_config
+        @funktor_config ||= YAML.load_file options[:file]
+      end
+
+      def name
+        funktor_config["appName"]
+      end
+
+      def runtime
+        funktor_config["runtime"]
       end
+
+      def queues
+        funktor_config["queues"]
+      end
+
+      def queue_names
+        funktor_config["queues"].map{|queue_details| queue_details.keys.first }
+      end
+
+      def work_queue_name
+        @work_queue_name
+      end
+
+      def work_queue_config
+        @work_queue_config
+      end
+
     end
   end
 end

data/lib/funktor/cli/templates/config/environment.yml

@@ -1,4 +1,6 @@
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue
-FUNKTOR_ACTIVE_JOB_QUEUE:
-  Ref: ActiveJobQueue
+<%- queue_names.each do |queue_name| -%>
+FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
+  Ref: <%= queue_name.camelize %>Queue
+<%- end -%>

data/lib/funktor/cli/templates/{function_definitions → config/function_definitions}/incoming_job_handler.yml

@@ -1,4 +1,4 @@
-handler: handlers/incoming_job_handler.call
+handler: app/handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.incomingJobHandler.timeout, 30}
 reservedConcurrency: ${self:custom.funktor.incomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.incomingJobHandler.provisionedConcurrency, null}

data/lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml

@@ -0,0 +1,11 @@
+handler: app/handlers/<%= work_queue_name.underscore %>_queue_handler.call
+timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.timeout, 900}
+reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
+events:
+  - sqs:
+      arn:
+        Fn::GetAtt:
+          - <%= work_queue_name.camelize %>Queue
+          - Arn

data/lib/funktor/cli/templates/config/funktor.yml

@@ -3,7 +3,7 @@ incomingJobHandler:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
   # but we default to a high number here to allow for the times when things go weird.
-  timeout: 900
+  timeout: 300
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the incoming job handler you probably don't want to limit it because you
   # want to get things onto work queues as quickly as possible.
@@ -17,12 +17,13 @@ incomingJobHandler:
   # but if you're seeing large delays at this stage it might help to bump it up.
   memorySize: 256
 
-activeJobHandler:
+<%- queue_names.each do |queue_name| -%>
+<%= queue_name.camelize %>QueueHandler:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The active job handler may be slow if your jobs are
   # doing a lot of work, so we default to the maximum here.
-  timeout: 900
+  timeout: 300 # TODO pull this (and other values) from funktor_init.yml
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the active job handler you may want to limit it if you have resource limitations
   # like database connections that you need to avoid exhausting.
@@ -36,16 +37,20 @@ activeJobHandler:
   # required to actually do your jobs. You can choose an even higher number to increase
   # the available CPU to make the jobs run faster.
   memorySize: 256
+<%- end -%>
+
 
 
 
 # You shouldn't need to mess with these under most circumstances. But you could if you want to change
 # the name of some of your resources in AWS.
 incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
 incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
-incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead-jobs
-activeJobQueueName: ${self:service}-${self:custom.stage}-active-jobs
-activityQueueName: ${self:service}-${self:custom.stage}-activity
-activityDeadQueueName: ${self:service}-${self:custom.stage}-activity-dead
-deadJobQueueName: ${self:service}-${self:custom.stage}-dead-jobs
 dashboardName: ${self:service}-${self:custom.stage}-dashboard
+<%- queue_names.each do |queue_name| -%>
+<%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
+<%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
+<%- end -%>
+
+

data/lib/funktor/cli/templates/{iam_permissions/active_job_queue.yml → config/iam_permissions/work_queue.yml}

@@ -5,4 +5,4 @@ Action:
   - sqs:SendMessage
   - sqs:GetQueueAttributes
 Resource:
-  - "Fn::GetAtt": [ ActiveJobQueue, Arn ]
+  - "Fn::GetAtt": [ <%= work_queue_name.camelize %>Queue, Arn ]

data/lib/funktor/cli/templates/config/package.yml

@@ -1,8 +1,9 @@
+# TODO - Figure out how to allow individual packaging to work out of the box.
+individually: false
 include:
   - Gemfile
   - Gemfile.lock
-  - handlers/**
-  - workers/**
+  - app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/lib/funktor/cli/templates/{resources → config/resources}/cloudwatch_dashboard.yml

@@ -40,10 +40,10 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.activeJobQueueName}", { "label": "Received" } ],
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.DefaultQueueName}", { "label": "Received" } ],
                             [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
-                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-activeJobHandler", "Resource", "${self:service}-${self:provider.stage}-activeJobHandler", { "label": "Handler Invocations" } ],
-                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.activeJobQueueName}", { "label": "Pending?" } ],
+                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-DefaultQueueHandler", "Resource", "${self:service}-${self:provider.stage}-DefaultQueueHandler", { "label": "Handler Invocations" } ],
+                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.DefaultQueueName}", { "label": "Pending?" } ],
                             [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
                             [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
                             [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]

data/lib/funktor/cli/templates/config/resources/work_queue.yml

@@ -0,0 +1,22 @@
+Resources:
+  <%= work_queue_name.camelize %>Queue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueName}
+      VisibilityTimeout: 300
+      RedrivePolicy:
+        deadLetterTargetArn:
+          "Fn::GetAtt": [ <%= work_queue_name.camelize %>DeadLetterQueue, Arn ]
+        maxReceiveCount: 5
+  <%= work_queue_name.camelize %>DeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.<%= work_queue_name.camelize %>DeadJobQueueName}
+
+Outputs:
+  <%= work_queue_name.camelize %>QueueUrl:
+    Value:
+      Ref: <%= work_queue_name.camelize %>Queue
+  <%= work_queue_name.camelize %>DeadLetterQueueUrl:
+    Value:
+      Ref: <%= work_queue_name.camelize %>DeadLetterQueue

data/lib/funktor/cli/templates/funktor_init.yml.tt

@@ -0,0 +1,72 @@
+# ℹ️  WARNING: This file doesn't control anything directly. It's used to allow you to set/maintain config options in a central place.
+# When you make changes to this file you should run `funktor init` again to propagate the changes to the 'active' configuration files.
+appName: <%= name %>
+runtime: ruby2.7
+
+# These values will be applied to any funktor handlers that don't specify custom values.
+# TODO - These still need to be wired up to do anything.
+handler_defaults:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. We default to a high number here to allow for the
+  # times when things go weird.
+  timeout: 900
+
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # Usually you'll want to leave this as null so that handlers can scale infinitely
+  # (within your account limits). If you need to restrict concurrency to prevent
+  # resource exhaustion you should do it at the queue level.
+  reservedConcurrency: null
+
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. You probably want to set this on individual queues if you're going
+  # to use it.
+  provisionedConcurrency: null
+
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # We default to 256 as a reasonable trade off on saving money but getting
+  # reasonable performance.
+  memorySize: 256
+
+  # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
+  batchSize: 10
+
+  # How many seconds should AWS wait for a batch to fill up before executing lambda?
+  # For immediate execution set the batch size to 1.
+  maximumBatchingWindow : 1
+
+  # Visibility timeout should only come into play in the case of Funktor errors.
+  # Application level errors should be handled by Funktor retry mechanisms.
+  # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
+  visibilityTimeout: 900
+
+  # Set log rentention to save money
+  logRetentionInDays: 30
+
+# Incoming Job Handler
+# TODO - These still need to be wired up to do anything.
+incoming_job_handler:
+  # If your jobs are bursty AND time-sensitive you might want to have some lambdas pre-provisioned
+  # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
+  # provisionedConcurrency: 4
+
+
+# TODO - These still need to be wired up to do anything.
+queues:
+  - default:
+      # Set queue specific config options here
+      # memorySize: 512
+      # TODO - Is it advisable to use FIFO queuues with Funktor?
+      # TODO - Maybe this isn't really even supported by CloudFormation?
+      # fifo: false
+  - singleThread:
+      reservedConcurrency: 1
+
+# TODO - Maybe this shouldn't be surfaced this early?
+# TODO - This still needs to be wired up to do anything.
+package:
+  patterns:
+    - Gemfile
+    - Gemfile.lock
+    - app/**
+    - config/**

data/lib/funktor/cli/templates/package.json

@@ -1 +1,8 @@
-{"name":"serverless-sub","description":"","version":"0.1.0","devDependencies":{"serverless-ruby-layer":"^1.4.0"}}
+{
+  "name":"<%= name %>",
+  "description":"",
+  "version":"0.1.0",
+  "devDependencies":{
+    "serverless-ruby-layer":"^1.4.0"
+  }
+}

data/lib/funktor/cli/templates/serverless.yml

@@ -12,7 +12,7 @@
 # Happy Coding!
 
 # The name of your service. All your AWS resources will contain this name.
-service: yourapp-funktor
+service: <%= name %>
 
 # This causes serverless to throw an error early if the config is bad, instead of waiting for CloudFormation to try and fail to deploy it.
 configValidationMode: error
@@ -22,13 +22,16 @@ frameworkVersion: '2'
 
 provider:
   name: aws
-  runtime: ruby2.7
+  runtime: <%= runtime %>
   lambdaHashingVersion: 20201221
   environment: ${file(config/environment.yml)}
+  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   iamRoleStatements:
-    - ${file(iam_permissions/ssm.yml)}
-    - ${file(iam_permissions/active_job_queue.yml)}
-    - ${file(iam_permissions/incoming_job_queue.yml)}
+    - ${file(config/iam_permissions/ssm.yml)}
+    - ${file(config/iam_permissions/incoming_job_queue.yml)}
+    <%- queue_names.each do |queue_name| -%>
+    - ${file(config/iam_permissions/<%= queue_name.underscore %>_queue.yml)}
+    <%- end -%>
 
 
 custom:
@@ -41,14 +44,18 @@ custom:
 package: ${file(config/package.yml)}
 
 functions:
-  incomingJobHandler: ${file(function_definitions/incoming_job_handler.yml)}
-  activeJobHandler: ${file(function_definitions/active_job_handler.yml)}
+  incomingJobHandler: ${file(config/function_definitions/incoming_job_handler.yml)}
+  <%- queue_names.each do |queue_name| -%>
+  <%= queue_name.camelize %>QueueHandler: ${file(config/function_definitions/<%= queue_name.underscore %>_queue_handler.yml)}
+  <%- end -%>
 
 resources:
-  - ${file(resources/active_job_queue.yml)}
-  - ${file(resources/incoming_job_queue.yml)}
-  - ${file(resources/incoming_job_queue_user.yml)}
-  - ${file(resources/cloudwatch_dashboard.yml)}
+  - ${file(config/resources/incoming_job_queue.yml)}
+  - ${file(config/resources/incoming_job_queue_user.yml)}
+  - ${file(config/resources/cloudwatch_dashboard.yml)}
+  <%- queue_names.each do |queue_name| -%>
+  - ${file(config/resources/<%= queue_name.underscore %>_queue.yml)}
+  <%- end -%>
 
 plugins:
-  - /Users/jgreen/projects/serverless-ruby-layer
+  - serverless-ruby-layer

data/lib/funktor/incoming_job_handler.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-sqs'
+require 'active_support/core_ext/string/inflections'
 
 module Funktor
   class IncomingJobHandler
@@ -26,10 +27,19 @@ module Funktor
       ENV['FUNKTOR_ACTIVE_JOB_QUEUE']
     end
 
+    def queue_for_job(job)
+      queue_name = job.queue || 'default'
+      queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
+      puts "queue_constant = #{queue_constant}"
+      puts "ENV value = #{ENV[queue_constant]}"
+      ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
+    end
+
     def push_to_active_job_queue(job)
+      puts "job = #{job.to_json}"
       sqs_client.send_message({
         # TODO : How to get this URL...
-        queue_url: active_job_queue,
+        queue_url: queue_for_job(job),
         message_body: job.to_json,
         delay_seconds: job.delay
       })

data/lib/funktor/job.rb

@@ -10,6 +10,10 @@ module Funktor
       @job_data ||= Funktor.parse_json(job_string)
     end
 
+    def queue
+      job_data["queue"]
+    end
+
     def worker_class_name
       job_data["worker"]
     end

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/lib/funktor/worker.rb

@@ -28,6 +28,10 @@ module Funktor::Worker
       get_funktor_options[:queue_url]
     end
 
+    def custom_queue
+      get_funktor_options[:queue]
+    end
+
     def queue_url
       # TODO : Should this default to FUNKTOR_ACTIVE_JOB_QUEUE?
       # Depends how e handle this in pro...?
@@ -73,10 +77,15 @@ module Funktor::Worker
       @client ||= Aws::SQS::Client.new
     end
 
+    def work_queue
+      (self.custom_queue || 'default').to_s
+    end
+
     def build_job_payload(job_id, delay, *worker_params)
       {
         worker: self.name,
         worker_params: worker_params,
+        queue: self.work_queue,
         job_id: job_id,
         delay: delay,
         funktor_options: get_funktor_options

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-04 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -89,6 +89,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -114,29 +115,26 @@ files:
 - lib/funktor/cli/generate/work_queue.rb
 - lib/funktor/cli/init.rb
 - lib/funktor/cli/templates/Gemfile
+- lib/funktor/cli/templates/app/handlers/incoming_job_handler.rb
+- lib/funktor/cli/templates/app/handlers/work_queue_handler.rb
+- lib/funktor/cli/templates/app/workers/hello_worker.rb
 - lib/funktor/cli/templates/config/environment.yml
+- lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml
+- lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml
 - lib/funktor/cli/templates/config/funktor.yml
+- lib/funktor/cli/templates/config/iam_permissions/incoming_job_queue.yml
+- lib/funktor/cli/templates/config/iam_permissions/ssm.yml
+- lib/funktor/cli/templates/config/iam_permissions/work_queue.yml
 - lib/funktor/cli/templates/config/package.yml
+- lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml
+- lib/funktor/cli/templates/config/resources/incoming_job_queue.yml
+- lib/funktor/cli/templates/config/resources/incoming_job_queue_user.yml
+- lib/funktor/cli/templates/config/resources/work_queue.yml
 - lib/funktor/cli/templates/config/ruby_layer.yml
-- lib/funktor/cli/templates/function_definitions/active_job_handler.yml
-- lib/funktor/cli/templates/function_definitions/incoming_job_handler.yml
-- lib/funktor/cli/templates/funktor.yml.tt
+- lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
-- lib/funktor/cli/templates/handlers/active_job_handler.rb
-- lib/funktor/cli/templates/handlers/incoming_job_handler.rb
-- lib/funktor/cli/templates/iam_permissions/active_job_queue.yml
-- lib/funktor/cli/templates/iam_permissions/incoming_job_queue.yml
-- lib/funktor/cli/templates/iam_permissions/ssm.yml
 - lib/funktor/cli/templates/package.json
-- lib/funktor/cli/templates/resources/active_job_queue.yml
-- lib/funktor/cli/templates/resources/cloudwatch_dashboard.yml
-- lib/funktor/cli/templates/resources/incoming_job_queue.yml
-- lib/funktor/cli/templates/resources/incoming_job_queue_user.yml
 - lib/funktor/cli/templates/serverless.yml
-- lib/funktor/cli/templates/workers/hello_worker.rb
-- lib/funktor/deploy/cli.rb
-- lib/funktor/deploy/serverless.rb
-- lib/funktor/deploy/serverless_templates/serverless.yml
 - lib/funktor/fake_job_queue.rb
 - lib/funktor/incoming_job_handler.rb
 - lib/funktor/job.rb

data/lib/funktor/cli/templates/function_definitions/active_job_handler.yml

@@ -1,11 +0,0 @@
-handler: handlers/active_job_handler.call
-timeout: ${self:custom.funktor.activeJobHandler.timeout, 900}
-reservedConcurrency: ${self:custom.funktor.activeJobHandler.reservedConcurrency, null}
-provisionedConcurrency: ${self:custom.funktor.activeJobHandler.provisionedConcurrency, null}
-memorySize: ${self:custom.funktor.activeJobHandler.memorySize, 256}
-events:
-  - sqs:
-      arn:
-        Fn::GetAtt:
-          - ActiveJobQueue
-          - Arn

data/lib/funktor/cli/templates/funktor.yml.tt

@@ -1,51 +0,0 @@
-# ℹ️  WARNING: This file doesn't control anything directly. It's used to allow you to set/maintain config options in a central place.
-# When you make changes to this file you should run `funktor init` again to propagate the changes to the 'active' configuration files.
-runtime: ruby2.7
-
-# Incoming Job Handler
-incoming_job_handler:
-  # Use memory_size to adjust the reousrces (both memory and CPU) available
-  # memory_size: 512
-  timeout_in_seconds: 20 # This handler might have to handle several incoming jobs at once
-  # WARNING : You probably don't want to limit concurrency on incoming jobs.
-  # concurrency: 100
-
-# Delayed Job Activator
-delayed_job_activator:
-  # Use memory_size to adjust the reousrces (both memory and CPU) available
-  # memory_size: 512
-  execution_schedule: rate(1 minute)
-  activation_window_in_seconds: 120 # Activate any jobs scheduled for the next two minutes
-  timeout_in_seconds: 300 # Allow an activation job to run for up to 5 minutes
-  concurrency: 1
-
-
-queues:
-  default:
-    # Use memory_size to adjust the reousrces (both memory and CPU) available
-    # memory_size: 512
-    # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
-    # batch_size: 10
-    # How many seconds should AWS wait for a batch to fill up before executing lambda?
-    # For immediate execution set the batch size to 1.
-    # maximumBatchingWindow : 1
-    # A single handler can receive up to batch_size jobs at a time. Make sure timeout is long enough.
-    timeout_in_seconds: 300
-    # You might want to limit concurrency of executing jobs to stay within resource limits (like DB connections).
-    # concurrency: 10
-    # Visibility timeout should only come into play in the case of Funktor errors.
-    # Application level errors should be handled by Funktor retry mechanisms.
-    # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
-    # visibility_timeout: 300
-    # TODO - Is it advisable to use FIFO queuues with Funktor?
-    # TODO - Maybe this isn't really even supported by CloudFormation?
-    # fifo: false
-
-# TODO - Maybe this is handled in the Dockerfile?
-package:
-  patterns:
-    - Gemfile
-    - Gemfile.lock
-    - ../app/**
-    - ../config/**
-    - ../lambda_handlers/**

data/lib/funktor/cli/templates/resources/active_job_queue.yml

@@ -1,22 +0,0 @@
-Resources:
-  ActiveJobQueue:
-    Type: AWS::SQS::Queue
-    Properties:
-      QueueName: ${self:custom.funktor.activeJobQueueName}
-      VisibilityTimeout: 300
-      RedrivePolicy:
-        deadLetterTargetArn:
-          "Fn::GetAtt": [ ActiveJobDeadLetterQueue, Arn ]
-        maxReceiveCount: 5
-  ActiveJobDeadLetterQueue:
-    Type: AWS::SQS::Queue
-    Properties:
-      QueueName: ${self:custom.funktor.deadJobQueueName}
-
-Outputs:
-  ActiveJobQueueUrl:
-    Value:
-      Ref: ActiveJobQueue
-  ActiveJobDeadLetterQueueUrl:
-    Value:
-      Ref: ActiveJobDeadLetterQueue

data/lib/funktor/deploy/cli.rb

@@ -1,42 +0,0 @@
-require 'optparse'
-require 'funktor/deploy/serverless'
-
-module Funktor
-  module Deploy
-    class CLI
-      attr_reader :options
-      def initialize
-        @options = {
-          verbose: false,
-          file: 'funktor.yml',
-          tmp_dir_prefix: '.funktor',
-          stage: 'dev'
-        }
-      end
-
-      def parse(argv = ARGV)
-        OptionParser.new do |opts|
-          opts.on('-v', '--verbose', 'Display verbose output') do |verbose|
-            options[:verbose] = verbose
-          end
-          opts.on('-f', '--file=FILE', 'The path to the funktor.yml file to deploy') do |file|
-            options[:file] = file
-          end
-          opts.on('-t', '--tmp_dir_prefix=TMP_DIR_PREFIX', 'The prefix for the tmp dir. The stage will be appended.') do |tmp_dir_prefix|
-            options[:tmp_dir_prefix] = tmp_dir_prefix
-          end
-          opts.on('-s', '--stage=STAGE', 'The stage to deploy to. Defaults to "dev"') do |stage|
-            options[:stage] = stage
-          end
-          opts.on('-h') { puts opts; exit }
-          opts.parse!(argv)
-        end
-      end
-
-      def run
-        Funktor::Deploy::Serverless.new(**options).call
-      end
-    end
-  end
-end
-

data/lib/funktor/deploy/serverless.rb

@@ -1,60 +0,0 @@
-require 'yaml'
-
-module Funktor
-  module Deploy
-    class Serverless
-      attr_accessor :file, :tmp_dir_prefix, :verbose, :stage
-      def initialize(file:, tmp_dir_prefix:, verbose:, stage:)
-        @file = file
-        @tmp_dir_prefix = tmp_dir_prefix
-        @verbose = verbose
-        @stage = stage
-      end
-
-      def call
-        #puts "deploying file #{file} via tmp_dir_prefix #{tmp_dir_prefix} for stage #{stage}"
-        make_tmp_dir
-        create_serverless_file
-      end
-
-      def funktor_data
-        @funktor_data ||= squash_hash(YAML.load_file(file))
-      end
-
-      def squash_hash(hsh, stack=[])
-        hsh.reduce({}) do |res, (key, val)|
-          next_stack = [ *stack, key ]
-          if val.is_a?(Hash)
-            next res.merge(squash_hash(val, next_stack))
-          end
-          res.merge(next_stack.join(".").to_sym => val)
-        end
-      end
-
-      def make_tmp_dir
-        FileUtils.mkdir_p tmp_dir
-      end
-
-      def tmp_dir
-        "#{tmp_dir_prefix}_#{stage}"
-      end
-
-      def create_serverless_file
-        #puts "funktor_data = "
-        #puts funktor_data
-        template_source = File.open(serverless_file_source).read
-        file_content = template_source % funktor_data
-        File.open(serverless_file_destination, 'w') { |file| file.write(file_content) }
-      end
-
-      def serverless_file_source
-        File.expand_path("../serverless_templates/serverless.yml", __FILE__)
-      end
-
-      def serverless_file_destination
-        File.join tmp_dir, 'serverless.yml'
-      end
-    end
-  end
-end
-

data/lib/funktor/deploy/serverless_templates/serverless.yml

@@ -1,156 +0,0 @@
-# WARNING : You probably don't want to mess with this file directly.
-service: %{stack_name}
-# app and org for use with dashboard.serverless.com
-#app: your-app-name
-#org: your-org-name
-
-frameworkVersion: '2'
-
-provider:
-  name: aws
-  runtime: %{runtime}
-  lambdaHashingVersion: 20201221
-  ecr:
-    images:
-      funktorimage:
-        path: ./
-        # TODO : Expose buildArgs via funktor.yml?
-        buildArgs:
-          BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
-  # TODO : Expose environment to funktor.yml?
-  environment:
-    FUNKTOR_INCOMING_JOB_QUEUE:
-      Ref: IncomingJobQueue
-    FUNKTOR_ACTIVE_JOB_QUEUE:
-      Ref: ActiveJobQueue
-    FUNKTOR_DELAYED_JOB_TABLE:
-      Ref: DelayedJobTable
-    delayedJobTable:
-      Ref: DelayedJobTable
-  iamRoleStatements:
-    - Effect: Allow
-      Action:
-        - ssm:Get*
-      Resource:
-        - '*' # TODO : This should probably be more selective...
-    - Effect: Allow
-      Action:
-        - sqs:ReceiveMessage
-        - sqs:DeleteMessage
-        - sqs:SendMessage
-        - sqs:GetQueueAttributes
-      Resource:
-        - "Fn::GetAtt": [ ActiveJobQueue, Arn ]
-    - Effect: Allow
-      Action:
-        - sqs:ReceiveMessage
-        - sqs:DeleteMessage
-        - sqs:SendMessage
-        - sqs:GetQueueAttributes
-      Resource:
-        - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
-    - Effect: Allow
-      Action:
-        - sqs:ReceiveMessage
-        - sqs:DeleteMessage
-        - sqs:SendMessage
-        - sqs:GetQueueAttributes
-      Resource:
-        - "Fn::GetAtt": [ ActivityQueue, Arn ]
-    - Effect: Allow
-      Action:
-        - dynamodb:PutItem
-        - dynamodb:DeleteItem
-      Resource:
-        - "Fn::GetAtt": [ DelayedJobTable, Arn ]
-    - Effect: Allow
-      Action:
-        - dynamodb:*
-      Resource:
-        - "Fn::GetAtt": [ StatsTable, Arn ]
-    - Effect: Allow
-      Action:
-        - dynamodb:Query
-      Resource:
-        Fn::Join:
-          - ""
-          - - "Fn::GetAtt": [ DelayedJobTable, Arn ]
-            - "/index/performAtIndex"
-
-custom:
-  # Our stage is based on what is passed in when running serverless
-  # commands. Or fallsback to what we have set in the provider section.
-  stage: ${opt:stage, 'dev'}
-  incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
-  incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
-  incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead-jobs
-  activeJobQueueName: ${self:service}-${self:custom.stage}-active-jobs
-  activityQueueName: ${self:service}-${self:custom.stage}-activity
-  activityDeadQueueName: ${self:service}-${self:custom.stage}-activity-dead
-  deadJobQueueName: ${self:service}-${self:custom.stage}-dead-jobs
-  delayedJobTableName: ${self:service}-${self:custom.stage}-delayed-jobs
-  statsTableName: ${self:service}-${self:custom.stage}-stats
-  dashboardName: ${self:service}-${self:custom.stage}-dashboard
-
-# you can define service wide environment variables here
-#  environment:
-#    variable1: value1
-
-functions:
-  # TODO - How could other functions be passed in from funktor.yml?
-  #random_job_generator:
-    ##handler: lambda_handlers/random_job_generator.RandomJobGenerator.call
-    #image:
-      #name: funktorimage
-      #command:
-        #- lambda_handlers/random_job_generator.RandomJobGenerator.call
-    #timeout: 170
-    #reservedConcurrency: 0
-    #events:
-      #- schedule: rate(1 minute)
-
-  delayed_job_activator:
-    image:
-      name: funktorimage
-      command:
-        - lambda_handlers/delayed_job_activator.call
-    timeout: %{delayed_job_activator.timeout_in_seconds}
-    # TODO - handle memory and concurrency
-    #reservedConcurrency: 1
-    events:
-      - schedule: %{delayed_job_activator.execution_schedule}
-
-  incoming_job_handler:
-    image:
-      name: funktorimage
-      command:
-        - lambda_handlers/incoming_job_handler.call
-    timeout: %{incoming_job_handler.timeout_in_seconds}
-    events:
-      - sqs:
-          arn:
-            Fn::GetAtt:
-              - IncomingJobQueue
-              - Arn
-
-  # TODO - We need one of these per work queue
-  active_job_handler:
-    image:
-      name: funktorimage
-      command:
-        - lambda_handlers/active_job_handler.call
-    timeout: 300
-    events:
-      - sqs:
-          arn:
-            Fn::GetAtt:
-              - ActiveJobQueue
-              - Arn
-
-# you can add CloudFormation resource templates here
-resources:
-  - ${file(resources/sqs-queue.yml)}
-  - ${file(resources/sqs-incoming-user.yml)}
-  - ${file(resources/dynamodb-table.yml)}
-  - ${file(resources/cloudfront-dashboard.yml)}
-