funktor 0.2.3 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 700b659a76af0a0e7e6a21c91a91553f4d5ff5899140ef18313bd181d97de4e1
-  data.tar.gz: f9d311402f6a0686e11c0d30131135fc5372bf58cc8e6c32c0f5a90f649861d7
+  metadata.gz: ed00bb1e70f943b1b4193e681fdde83c41fd0bd84bae056d4b171689f2dbe5c8
+  data.tar.gz: 57c02b8d42a470f0f52b90d60fff3c4b176523a7c606a7f7c562649dddbdfe4e
 SHA512:
-  metadata.gz: 47e9682f6d0f96e9ce5d393574642b8361049c91f07dc2b9154c4492c0b10257bc651e22ea5512a1f904cd2a895c9435d9722c559b756086618f1ea58983298a
-  data.tar.gz: a498a2854d47e3c2b0bfd9f1bb2d2feb31b5bd3b8ac1f34c79a5efd127cb81d77862147cd6d27528589710676b748bd744bdd3ace35ebfdb118d4983a3414ac0
+  metadata.gz: 3575e30e6ae1dea0d36f4b31dcc06ec06b2e92ac07342f79cafc5cc99da9f66d22df79091643e4b036a24f22dab01747adb485806e4e3bae42be3441250f1839
+  data.tar.gz: f2a1827d30175051202691e9ab062929cbf73ba6b32b2bf8081259696fcb29d386822e94cfd4706993864581f294ded37918f45d23ab0cf986d5f21ada0c7f2c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.3)
+    funktor (0.2.8)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor
@@ -9,7 +9,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3.2)
+    activesupport (6.1.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -18,8 +18,8 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.1)
-    aws-partitions (1.468.0)
-    aws-sdk-core (3.114.3)
+    aws-partitions (1.472.0)
+    aws-sdk-core (3.115.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)

data/README.md

@@ -34,41 +34,20 @@ npm install -g serverless
 
 Then you can initialize a new app by doing:
 
-
 ```bash
+funktor bootstrap my-funktor-app
+cd my-funktor-app
 funktor init
 ```
 
+Then you sh
+
 This will create a `funktor` directory that is ready to deploy to AWS. If you've already configured
 your aws tools via `~/.aws/credentials` you should be ready to deploy.
 
-`funktor/serverless.yml` is the main file that ties everything together.
-
-`funktor/config` contains a few files that you can use to configure your `funktor` application.
-
-`funktor/resources` contains a few files that provision some AWS resources that are used by `funktor`.
-* An SQS Queue for the "incoming jobs queue"
-* A Dynamo DB table to allow queueing of jobs more than 15 minutes in the future (Funktor Pro)
-* One or more SQS Queues for active jobs (currently there is only the default queue, support for additional queues is coming soon)
-* An IAM User with permission to push jobs to the incoming jobs queue
-* A CloudWatch dashboard to let you keep tabs on your application
-
-`funktor/lambda_handlers` contains some scripts that receive events from Lambda, then invoke `funktor` to
-do various things:
-* `active_job_handler.rb` executes your jobs
-* `delayed_job_scheduler.rb` (Funktor Pro) pulls delayed jobs out of DynamoDB and places them on the active job queue.
-* `incoming_job_handler.rb` receives incoming jobs and pushes them to DynamoDB for delayed execution or to the active job queue as appropriate.
-
-`funktor/function_definitions` contains details about hooking up the `lambda_handlers` to events.
-
-`funktor/iam_permissions` contains some details about giving your lambda functions the appropriate permissions
-to interact with SQS.
-
-`funktor/workers` is where your workers will live.
-
-`funktor/Gemfile` is the `Gemfile` that contains the gems that are needed for your workers to execute
-jobs. This should be the minimal set of gems you can get away with so that cold start times remain reasonable.
-This file will already contain `funktor`. (Don't remove it or `funktor` won't work!)
+See the [wiki](https://github.com/Octo-Labs/funktor/wiki)
+for more info, especially the
+[section about getting started in a stand alone project](https://github.com/Octo-Labs/funktor/wiki/Getting-started-in-a-stand-alone-project).
 
 ## Deploying
 

data/lib/funktor/cli/init.rb

@@ -30,16 +30,13 @@ module Funktor
         name
       end
 
-      def serverless_yml
-        template "serverless.yml", File.join("serverless.yml")
-      end
-
       def funktor_config_yml
         #template "funktor_config.yml", File.join("funktor_config.yml")
         template File.join("config", "funktor.yml"), File.join("config", "funktor.yml")
         template File.join("config", "ruby_layer.yml"), File.join("config", "ruby_layer.yml")
         template File.join("config", "package.yml"), File.join("config", "package.yml")
         template File.join("config", "environment.yml"), File.join("config", "environment.yml")
+        template File.join("config", "boot.rb"), File.join("config", "boot.rb")
       end
 
       def package_json
@@ -54,6 +51,10 @@ module Funktor
         template "gitignore", File.join(".gitignore")
       end
 
+      def workers
+        template File.join("app", "workers", "hello_worker.rb"), File.join("app", "workers", "hello_worker.rb")
+      end
+
       def resources
         template File.join("config", "resources", "incoming_job_queue.yml"), File.join("config", "resources", "incoming_job_queue.yml")
         template File.join("config", "resources", "incoming_job_queue_user.yml"), File.join("config", "resources", "incoming_job_queue_user.yml")
@@ -86,19 +87,42 @@ module Funktor
       end
 
       def lambda_handlers
-        template File.join("app", "handlers", "incoming_job_handler.rb"), File.join("app", "handlers", "incoming_job_handler.rb")
+        template File.join("lambda_event_handlers", "incoming_job_handler.rb"), File.join("lambda_event_handlers", "incoming_job_handler.rb")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
-          template File.join("app", "handlers", "work_queue_handler.rb"), File.join("app", "handlers", "#{work_queue_name.underscore}_queue_handler.rb")
+          template File.join("lambda_event_handlers", "work_queue_handler.rb"), File.join("lambda_event_handlers", "#{work_queue_name.underscore}_queue_handler.rb")
         end
       end
 
-      def workers
-        template File.join("app", "workers", "hello_worker.rb"), File.join("app", "workers", "hello_worker.rb")
+      def serverless_yml
+        template "serverless.yml", File.join("serverless.yml")
       end
 
       private
+
+      def app_worker_names
+        app_worker_files.map do |file|
+          File.basename(file, ".rb").camelize
+        end
+      end
+
+      def app_worker_files
+        Dir.glob(File.join('app', 'workers', '**.rb'))
+      end
+
+      def all_iam_permissions
+        Dir.glob(File.join('config', 'iam_permissions', '**.yml'))
+      end
+
+      def all_function_definitions
+        Dir.glob(File.join('config', 'function_definitions', '**.yml'))
+      end
+
+      def all_resources
+        Dir.glob(File.join('config', 'resources', '**.yml'))
+      end
+
       def funktor_config
         @funktor_config ||= YAML.load_file options[:file]
       end
@@ -107,6 +131,10 @@ module Funktor
         funktor_config["appName"]
       end
 
+      def app_name
+        funktor_config["appName"]
+      end
+
       def runtime
         funktor_config["runtime"]
       end
@@ -143,7 +171,6 @@ module Funktor
       end
 
       def queue_config_value(queue_name, config_name)
-        puts funktor_config
         queue_config(queue_name)&.dig(config_name) ||
           funktor_config.dig("handlerDefaults", config_name) ||
           "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template

data/lib/funktor/cli/templates/config/boot.rb

@@ -0,0 +1,16 @@
+# Point at our Gemfile
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+
+# load rubygems & bundler
+require "rubygems"
+require 'bundler/setup'
+
+# Set up gems listed in the Gemfile.
+Bundler.require(:default, :production)
+
+# Load all ruby files in the app directory
+Dir.glob( File.join('..', 'app', '**', '*.rb'), base: File.dirname(__FILE__) ).each do |ruby_file|
+  puts "require_relative #{ruby_file}"
+  require_relative ruby_file
+end
+

data/lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,8 @@
-handler: app/handlers/incoming_job_handler.call
-timeout: ${self:custom.funktor.incomingJobHandler.timeout, 30}
-reservedConcurrency: ${self:custom.funktor.incomingJobHandler.reservedConcurrency, null}
-provisionedConcurrency: ${self:custom.funktor.incomingJobHandler.provisionedConcurrency, null}
-memorySize: ${self:custom.funktor.incomingJobHandler.memorySize, 256}
+handler: lambda_event_handlers/incoming_job_handler.call
+timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
+reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
 events:
   - sqs:
       arn:

data/lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml

@@ -1,4 +1,4 @@
-handler: app/handlers/<%= work_queue_name.underscore %>_queue_handler.call
+handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.timeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}

data/lib/funktor/cli/templates/config/funktor.yml

@@ -1,4 +1,4 @@
-incomingJobHandler:
+IncomingJobHandler:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
@@ -51,10 +51,11 @@ incomingJobHandler:
 
 # You shouldn't need to mess with these under most circumstances. But you could if you want to change
 # the name of some of your resources in AWS.
-incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
-incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
-incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
-dashboardName: ${self:service}-${self:custom.stage}-dashboard
+IncomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
+IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
+IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
+DashboardName: ${self:service}-${self:custom.stage}-dashboard
 <%- queue_names.each do |queue_name| -%>
 <%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
 <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead

data/lib/funktor/cli/templates/config/package.yml

@@ -3,6 +3,7 @@ individually: false
 include:
   - Gemfile
   - Gemfile.lock
+  - config/boot.rb
   - app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.

data/lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml

@@ -2,7 +2,7 @@ Resources:
   FunktorDashboard:
     Type: AWS::CloudWatch::Dashboard
     Properties:
-      DashboardName: ${self:custom.funktor.dashboardName}
+      DashboardName: ${self:custom.funktor.DashboardName}
       DashboardBody: >
         {
             "widgets": [
@@ -25,7 +25,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Messages Per Minute" } ]
                         ],
                         "view": "singleValue",
                         "region": "us-east-1",
@@ -42,7 +42,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "p10" } ],
                             [ "...", { "label": "p50", "stat": "p50" } ],
                             [ "...", { "label": "p99", "stat": "p99" } ],
                             [ "...", { "label": "Average", "stat": "Average" } ]
@@ -63,10 +63,10 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Received" } ],
                             [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
-                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
-                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
+                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "Handler Invocations" } ],
+                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Pending?" } ],
                             [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
                             [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
                             [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
@@ -90,7 +90,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Minimum" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Minimum" } ],
                             [ "...", { "stat": "Average" } ],
                             [ "...", { "stat": "Maximum" } ]
                         ],
@@ -110,7 +110,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
+                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
                             [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
                             [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
                         ],
@@ -137,7 +137,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Maximum" } ]
+                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Maximum" } ]
                         ],
                         "region": "us-east-1",
                         "title": "Incoming Job Handler Concurrent Executions",
@@ -308,10 +308,10 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "rails-lambda-experiment", "Duration", "WorkerClassName", "HelloWorker" ],
-                            [ "...", { "stat": "p99" } ],
-                            [ "...", "HelloLaterWorker" ],
-                            [ "...", { "stat": "p99" } ]
+                            <%- app_worker_names.each do |worker_name| -%>
+                            [ "<%= app_name %>", "Duration", "WorkerClassName", "<%= worker_name %>" ],
+                            [ "...", { "stat": "p99" } ]<%= worker_name == app_worker_names.last ? "" : "," %>
+                            <%- end -%>
                         ],
                         "view": "timeSeries",
                         "stacked": false,

data/lib/funktor/cli/templates/config/resources/incoming_job_queue.yml

@@ -2,7 +2,7 @@ Resources:
   IncomingJobQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingJobQueueName}
       VisibilityTimeout: 300
       RedrivePolicy:
         deadLetterTargetArn:
@@ -11,7 +11,7 @@ Resources:
   IncomingJobDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingDeadJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingDeadJobQueueName}
 
 Outputs:
   IncomingJobQueueUrl:

data/lib/funktor/cli/templates/{app/handlers → lambda_event_handlers}/incoming_job_handler.rb

@@ -1,4 +1,4 @@
-require 'funktor'
+require_relative '../config/boot'
 
 $handler = Funktor::IncomingJobHandler.new
 

data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb

@@ -0,0 +1,8 @@
+require_relative '../config/boot'
+
+$handler = Funktor::ActiveJobHandler.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/lib/funktor/cli/templates/serverless.yml

@@ -27,10 +27,8 @@ provider:
   environment: ${file(config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   iamRoleStatements:
-    - ${file(config/iam_permissions/ssm.yml)}
-    - ${file(config/iam_permissions/incoming_job_queue.yml)}
-    <%- queue_names.each do |queue_name| -%>
-    - ${file(config/iam_permissions/<%= queue_name.underscore %>_queue.yml)}
+    <%- all_iam_permissions.each do |iam_permission| -%>
+    - ${file(<%= iam_permission %>)}
     <%- end -%>
 
 
@@ -44,17 +42,13 @@ custom:
 package: ${file(config/package.yml)}
 
 functions:
-  incomingJobHandler: ${file(config/function_definitions/incoming_job_handler.yml)}
-  <%- queue_names.each do |queue_name| -%>
-  <%= queue_name.camelize %>QueueHandler: ${file(config/function_definitions/<%= queue_name.underscore %>_queue_handler.yml)}
+  <%- all_function_definitions.each do |function_definition| -%>
+  <%= File.basename(function_definition, ".yml").camelize %>: ${file(<%= function_definition %>)}
   <%- end -%>
 
 resources:
-  - ${file(config/resources/incoming_job_queue.yml)}
-  - ${file(config/resources/incoming_job_queue_user.yml)}
-  - ${file(config/resources/cloudwatch_dashboard.yml)}
-  <%- queue_names.each do |queue_name| -%>
-  - ${file(config/resources/<%= queue_name.underscore %>_queue.yml)}
+  <%- all_resources.each do |resource| -%>
+  - ${file(<%= resource %>)}
   <%- end -%>
 
 plugins:

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.3"
+  VERSION = "0.2.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.8
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-21 00:00:00.000000000 Z
+date: 2021-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -115,9 +115,8 @@ files:
 - lib/funktor/cli/generate/work_queue.rb
 - lib/funktor/cli/init.rb
 - lib/funktor/cli/templates/Gemfile
-- lib/funktor/cli/templates/app/handlers/incoming_job_handler.rb
-- lib/funktor/cli/templates/app/handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/app/workers/hello_worker.rb
+- lib/funktor/cli/templates/config/boot.rb
 - lib/funktor/cli/templates/config/environment.yml
 - lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml
 - lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml
@@ -133,6 +132,8 @@ files:
 - lib/funktor/cli/templates/config/ruby_layer.yml
 - lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
+- lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
+- lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/package.json
 - lib/funktor/cli/templates/serverless.yml
 - lib/funktor/fake_job_queue.rb

data/lib/funktor/cli/templates/app/handlers/work_queue_handler.rb

@@ -1,17 +0,0 @@
-require 'funktor'
-
-# Bundler is hard to make work because AWS includes some gems in the basic ruby runtime.
-# We're probably going to need to use containers...
-#require "rubygems"
-#require "bundler/setup"
-#Bundler.require(:default)
-
-# TODO : Ideally this wouldn't be needed
-require_relative '../workers/hello_worker'
-
-$handler = Funktor::ActiveJobHandler.new
-
-def call(event:, context:)
-  $handler.call(event: event, context: context)
-end
-