funktor 0.2.1 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 154286564dee4cc895ef3515dcacfa515a91ecde743862e36c833d6db0389447
-  data.tar.gz: e1786a3bdda966cd7d1fdda07ae3faf12f3ec2ca27b51506257520b3d13b02dd
+  metadata.gz: 3a92d7870ba1cde9f91c5d872aab7e5dcc9d7a0d9ee8b4deb75fa1e679ea6ddc
+  data.tar.gz: 79d938d9d46f9763ae6bf7001bf54bb1ad48eefee0a0b5dcbb624a934edbba85
 SHA512:
-  metadata.gz: c6b7de2400aceac0dcad12d2ad4093a07be595a120f88c58eddf7fa63f3a353e7637c47963cca8e79263b1ec29838ee5c26147a0e924cc7508092ac968e104b2
-  data.tar.gz: c5cf1bdea5c9ad86415d7da9350b28699900b623a05d4860ecfb11c77daa773fbdd4b1c33a166cfa7c7025a238c268d3748b8a9b9989043e0f29cae6a8446d99
+  metadata.gz: b741a360102285bc392c6b846677b1859dbb02652b9e9412773fa989172c4a2368429f7825736f4438a543f4ac058098884fd78ce8a8f89be81b10f6b2e4db94
+  data.tar.gz: a450bec63e833671d3eeb7e4e1f00c38eec6c66c59de610ba431f8660a1273dac0ab4db85ff7b41ca2ff9e234c19f771087127d16ab2a21612b731ed61678609

data/.github/workflows/ruby.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -10,6 +10,6 @@
 # rspec failure tracking
 .rspec_status
 
-# The funktor dir isn't a real part of the gem, it's where things
+# The funktor app dir isn't a real part of the gem, it's where things
 # get generated as I'm testing the gem locally. It doesn't need to be in git.
-/funktor/
+/myapp-funktor/

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.1)
+    funktor (0.2.6)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor
@@ -9,7 +9,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3.2)
+    activesupport (6.1.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -18,8 +18,8 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.1)
-    aws-partitions (1.465.0)
-    aws-sdk-core (3.114.1)
+    aws-partitions (1.471.0)
+    aws-sdk-core (3.115.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
@@ -29,7 +29,7 @@ GEM
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.3)
       aws-eventstream (~> 1, >= 1.0.2)
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.1.9)
     crack (0.4.5)
       rexml
     diff-lcs (1.4.4)
@@ -41,7 +41,7 @@ GEM
     minitest (5.14.4)
     public_suffix (4.0.6)
     rake (12.3.3)
-    rexml (3.2.4)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)

data/funktor.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'aws-sdk-sqs', '~> 1.37'
   spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
-  spec.add_dependency "thor" # TODO - Can we build our own verison of cattr_accessor to avoid this?
+  spec.add_dependency "thor" # Thor drives the CLI
 
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'webmock'

data/lib/funktor.rb

@@ -11,7 +11,6 @@ require 'json'
 
 module Funktor
   class Error < StandardError; end
-  # Your code goes here...
 
   def self.configure_job_pusher
     yield self

data/lib/funktor/cli/bootstrap.rb

@@ -3,16 +3,15 @@ module Funktor
     class Bootstrap < Thor::Group
       include Thor::Actions
 
+      argument :name, :type => :string, :desc => "The name of the app to initialize"
+
       class_option :file, :aliases => "-f",
         :type => :string, :desc => "The bootstrap file to generate.",
-        :default => "funktor.yml"
-      class_option :directory, :aliases => "-d",
-        :type => :string, :desc => "The directory to initialize",
-        :default => "funktor"
+        :default => "funktor_init.yml"
 
       desc <<~DESC
         Description:
-          Bootstrap a new funktor application by generating a funktor.yml file."
+          Bootstrap a new funktor application by generating a funktor_init.yml file."
       DESC
 
       def self.source_root
@@ -20,13 +19,14 @@ module Funktor
       end
 
       def funktor_yml
+        # TODO - Should we camelize the app name before writing it into the config? (CloudFormation names get weird with underscores and dashes.)
         puts funktor_file_target
-        template "templates/funktor.yml", funktor_file_target
+        template "templates/funktor_init.yml", funktor_file_target
       end
 
       private
       def funktor_file_target
-        File.join options[:directory], options[:file]
+        File.join name, options[:file]
       end
 
     end

data/lib/funktor/cli/init.rb

@@ -1,14 +1,21 @@
+require 'yaml'
+require 'active_support/core_ext/string/inflections'
+
 module Funktor
   module CLI
     class Init < Thor::Group
       include Thor::Actions
 
-      class_option :framework, :aliases => "-f",
+      attr_accessor :work_queue_name
+      attr_accessor :work_queue_config
+
+      class_option :deployment_framework, :aliases => "-d",
         :type => :string, :desc => "The deployment/provisioning framework to use.",
         :default => "serverless"
-      class_option :directory, :aliases => "-d",
-        :type => :string, :desc => "The directory to initialize",
-        :default => "funktor"
+
+      class_option :file, :aliases => "-f",
+        :type => :string, :desc => "The funktor init file.",
+        :default => "funktor_init.yml"
 
       desc <<~DESC
         Description:
@@ -20,59 +27,157 @@ module Funktor
       end
 
       def self.destination_root
-        options[:directory]
-      end
-
-      def serverless_yml
-        template "serverless.yml", File.join(options[:directory], "serverless.yml")
+        name
       end
 
       def funktor_config_yml
-        #template "funktor_config.yml", File.join(options[:directory], "funktor_config.yml")
-        template File.join("config", "funktor.yml"), File.join(options[:directory], "config", "funktor.yml")
-        template File.join("config", "ruby_layer.yml"), File.join(options[:directory], "config", "ruby_layer.yml")
-        template File.join("config", "package.yml"), File.join(options[:directory], "config", "package.yml")
-        template File.join("config", "environment.yml"), File.join(options[:directory], "config", "environment.yml")
+        #template "funktor_config.yml", File.join("funktor_config.yml")
+        template File.join("config", "funktor.yml"), File.join("config", "funktor.yml")
+        template File.join("config", "ruby_layer.yml"), File.join("config", "ruby_layer.yml")
+        template File.join("config", "package.yml"), File.join("config", "package.yml")
+        template File.join("config", "environment.yml"), File.join("config", "environment.yml")
+        template File.join("config", "boot.rb"), File.join("config", "boot.rb")
       end
 
       def package_json
-        template "package.json", File.join(options[:directory], "package.json")
+        template "package.json", File.join("package.json")
       end
 
       def gemfile
-        template "Gemfile", File.join(options[:directory], "Gemfile")
+        template "Gemfile", File.join("Gemfile")
       end
 
       def gitignore
-        template "gitignore", File.join(options[:directory], ".gitignore")
+        template "gitignore", File.join(".gitignore")
+      end
+
+      def workers
+        template File.join("app", "workers", "hello_worker.rb"), File.join("app", "workers", "hello_worker.rb")
       end
 
       def resources
-        template File.join("resources", "incoming_job_queue.yml"), File.join(options[:directory], "resources", "incoming_job_queue.yml")
-        template File.join("resources", "incoming_job_queue_user.yml"), File.join(options[:directory], "resources", "incoming_job_queue_user.yml")
-        template File.join("resources", "active_job_queue.yml"), File.join(options[:directory], "resources", "active_job_queue.yml")
-        template File.join("resources", "cloudwatch_dashboard.yml"), File.join(options[:directory], "resources", "cloudwatch_dashboard.yml")
+        template File.join("config", "resources", "incoming_job_queue.yml"), File.join("config", "resources", "incoming_job_queue.yml")
+        template File.join("config", "resources", "incoming_job_queue_user.yml"), File.join("config", "resources", "incoming_job_queue_user.yml")
+        # TODO - Figure out how to make the dashboard aware of various queues...
+        template File.join("config", "resources", "cloudwatch_dashboard.yml"), File.join("config", "resources", "cloudwatch_dashboard.yml")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("config", "resources", "work_queue.yml"), File.join("config", "resources", "#{work_queue_name.underscore}_queue.yml")
+        end
       end
 
       def iam_permissions
-        template File.join("iam_permissions", "ssm.yml"), File.join(options[:directory], "iam_permissions", "ssm.yml")
-        template File.join("iam_permissions", "incoming_job_queue.yml"), File.join(options[:directory], "iam_permissions", "incoming_job_queue.yml")
-        template File.join("iam_permissions", "active_job_queue.yml"), File.join(options[:directory], "iam_permissions", "active_job_queue.yml")
+        template File.join("config", "iam_permissions", "ssm.yml"), File.join("config", "iam_permissions", "ssm.yml")
+        template File.join("config", "iam_permissions", "incoming_job_queue.yml"), File.join("config", "iam_permissions", "incoming_job_queue.yml")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("config", "iam_permissions", "work_queue.yml"), File.join("config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
+        end
       end
 
       def function_definitions
-        template File.join("function_definitions", "active_job_handler.yml"), File.join(options[:directory], "function_definitions", "active_job_handler.yml")
-        template File.join("function_definitions", "incoming_job_handler.yml"), File.join(options[:directory], "function_definitions", "incoming_job_handler.yml")
+        template File.join("config", "function_definitions", "incoming_job_handler.yml"), File.join("config", "function_definitions", "incoming_job_handler.yml")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("config", "function_definitions", "work_queue_handler.yml"), File.join("config", "function_definitions", "#{work_queue_name.underscore}_queue_handler.yml")
+        end
       end
 
       def lambda_handlers
-        template File.join("handlers", "active_job_handler.rb"), File.join(options[:directory], "handlers", "active_job_handler.rb")
-        template File.join("handlers", "incoming_job_handler.rb"), File.join(options[:directory], "handlers", "incoming_job_handler.rb")
+        template File.join("lambda_event_handlers", "incoming_job_handler.rb"), File.join("lambda_event_handlers", "incoming_job_handler.rb")
+        queues.each do |queue_details|
+          @work_queue_name = queue_details.keys.first
+          @work_queue_config = queue_details.values.first
+          template File.join("lambda_event_handlers", "work_queue_handler.rb"), File.join("lambda_event_handlers", "#{work_queue_name.underscore}_queue_handler.rb")
+        end
       end
 
-      def workers
-        template File.join("workers", "hello_worker.rb"), File.join(options[:directory], "workers", "hello_worker.rb")
+      def serverless_yml
+        template "serverless.yml", File.join("serverless.yml")
+      end
+
+      private
+
+      def app_worker_names
+        app_worker_files.map do |file|
+          File.basename(file, ".rb").camelize
+        end
+      end
+
+      def app_worker_files
+        Dir.glob(File.join('app', 'workers', '**.rb'))
+      end
+
+      def all_iam_permissions
+        Dir.glob(File.join('config', 'iam_permissions', '**.yml'))
+      end
+
+      def all_function_definitions
+        Dir.glob(File.join('config', 'function_definitions', '**.yml'))
+      end
+
+      def all_resources
+        Dir.glob(File.join('config', 'resources', '**.yml'))
+      end
+
+      def funktor_config
+        @funktor_config ||= YAML.load_file options[:file]
+      end
+
+      def name
+        funktor_config["appName"]
+      end
+
+      def app_name
+        funktor_config["appName"]
+      end
+
+      def runtime
+        funktor_config["runtime"]
+      end
+
+      def queues
+        funktor_config["queues"]
       end
+
+      def queue_names
+        funktor_config["queues"].map{|queue_details| queue_details.keys.first }
+      end
+
+      def work_queue_name
+        @work_queue_name
+      end
+
+      def work_queue_config
+        @work_queue_config
+      end
+
+      def queue_config(queue_name)
+        funktor_config["queues"].each do |queue_details|
+          if queue_details.keys.first == queue_name
+            return queue_details.values.first
+          end
+        end
+        return nil
+      end
+
+      def incoming_config_value(config_name)
+        funktor_config.dig("incomingJobHandler", config_name) ||
+          funktor_config.dig("handlerDefaults", config_name) ||
+          "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
+      end
+
+      def queue_config_value(queue_name, config_name)
+        puts funktor_config
+        queue_config(queue_name)&.dig(config_name) ||
+          funktor_config.dig("handlerDefaults", config_name) ||
+          "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template
+
+      end
+
     end
   end
 end

data/lib/funktor/cli/templates/Gemfile

@@ -3,7 +3,5 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
 ruby '~> 2.7'
 
-source "https://gem.fury.io/jagthedrummer/" do
-  gem 'funktor'
-end
+gem 'funktor'
 

data/lib/funktor/cli/templates/config/boot.rb

@@ -0,0 +1,16 @@
+# Point at our Gemfile
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+
+# load rubygems & bundler
+require "rubygems"
+require 'bundler/setup'
+
+# Set up gems listed in the Gemfile.
+Bundler.require(:default, :production)
+
+# Load all ruby files in the app directory
+Dir.glob( File.join('..', 'app', '**', '*.rb'), base: File.dirname(__FILE__) ).each do |ruby_file|
+  puts "require_relative #{ruby_file}"
+  require_relative ruby_file
+end
+

data/lib/funktor/cli/templates/config/environment.yml

@@ -1,4 +1,6 @@
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue
-FUNKTOR_ACTIVE_JOB_QUEUE:
-  Ref: ActiveJobQueue
+<%- queue_names.each do |queue_name| -%>
+FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
+  Ref: <%= queue_name.camelize %>Queue
+<%- end -%>

data/lib/funktor/cli/templates/{function_definitions → config/function_definitions}/incoming_job_handler.yml

@@ -1,4 +1,4 @@
-handler: handlers/incoming_job_handler.call
+handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.incomingJobHandler.timeout, 30}
 reservedConcurrency: ${self:custom.funktor.incomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.incomingJobHandler.provisionedConcurrency, null}

data/lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml

@@ -0,0 +1,11 @@
+handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
+timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.timeout, 900}
+reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
+events:
+  - sqs:
+      arn:
+        Fn::GetAtt:
+          - <%= work_queue_name.camelize %>Queue
+          - Arn

data/lib/funktor/cli/templates/config/funktor.yml

@@ -3,49 +3,62 @@ incomingJobHandler:
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
   # but we default to a high number here to allow for the times when things go weird.
-  timeout: 900
+  timeout: <%= incoming_config_value 'timeout' %>
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the incoming job handler you probably don't want to limit it because you
   # want to get things onto work queues as quickly as possible.
-  reservedConcurrency: null
+  reservedConcurrency: <%= incoming_config_value 'reservedConcurrency' %>
   # provisionedConcurrency represents the number of lambda functions that will always
   # be available. For the incoming jobs handler you probably don't need to set this
   # unless your jobs are very bursty AND very time sensitive.
-  provisionedConcurrency: null
+  provisionedConcurrency: <%= incoming_config_value 'provisionedConcurrency' %>
   # Use memory_size to adjust the reousrces (both memory and CPU) available.
   # For the incoming jobs handler you probably don't need this to be too large,
   # but if you're seeing large delays at this stage it might help to bump it up.
-  memorySize: 256
+  memorySize: <%= incoming_config_value 'memorySize' %>
+  batchSize: <%= incoming_config_value 'batchSize' %>
+  maximumBatchingWindow: <%= incoming_config_value 'maximumBatchingWindow' %>
+  visibilityTimeout: <%= incoming_config_value 'visibilityTimeout' %>
+  logRetentionInDays: <%= incoming_config_value 'logRetentionInDays' %>
 
-activeJobHandler:
+<%- queue_names.each do |queue_name| -%>
+<%= queue_name.camelize %>QueueHandler:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The active job handler may be slow if your jobs are
   # doing a lot of work, so we default to the maximum here.
-  timeout: 900
+  timeout: <%= queue_config_value queue_name, 'timeout' %>
   # reservedConcurrency represents the maximum number of concurrent executions.
   # For the active job handler you may want to limit it if you have resource limitations
   # like database connections that you need to avoid exhausting.
-  reservedConcurrency: null
+  reservedConcurrency: <%= queue_config_value queue_name, 'reservedConcurrency' %>
   # provisionedConcurrency represents the number of lambda functions that will always
   # be available. For the active job handler you probably don't need to set this
   # unless your jobs are very bursty AND very time sensitive.
-  provisionedConcurrency: null
+  provisionedConcurrency: <%= queue_config_value queue_name, 'provisionedConcurrency' %>
   # Use memory_size to adjust the reousrces (both memory and CPU) available.
   # For the active jobs handler you'll want this to be at least as large as the memory
   # required to actually do your jobs. You can choose an even higher number to increase
   # the available CPU to make the jobs run faster.
-  memorySize: 256
+  memorySize: <%= queue_config_value queue_name, 'memorySize' %>
+  batchSize: <%= queue_config_value queue_name, 'batchSize' %>
+  maximumBatchingWindow: <%= queue_config_value queue_name, 'maximumBatchingWindow' %>
+  visibilityTimeout: <%= queue_config_value queue_name, 'visibilityTimeout' %>
+  logRetentionInDays: <%= queue_config_value queue_name, 'logRetentionInDays' %>
 
+<%- end -%>
 
 
 # You shouldn't need to mess with these under most circumstances. But you could if you want to change
 # the name of some of your resources in AWS.
 incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
 incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
-incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead-jobs
-activeJobQueueName: ${self:service}-${self:custom.stage}-active-jobs
-activityQueueName: ${self:service}-${self:custom.stage}-activity
-activityDeadQueueName: ${self:service}-${self:custom.stage}-activity-dead
-deadJobQueueName: ${self:service}-${self:custom.stage}-dead-jobs
 dashboardName: ${self:service}-${self:custom.stage}-dashboard
+<%- queue_names.each do |queue_name| -%>
+<%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
+<%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead
+<%= queue_name.camelize %>QueueHandlerName: ${self:service}-${self:custom.stage}-<%= queue_name.camelize %>QueueHandler
+<%- end -%>
+
+