funktor 0.2.4 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 666c2af44a969a861936755046fdb067cd118f5c1c197673d3eaf9848a88a3a0
-  data.tar.gz: d3a6282e200730a25192d6030f3c28876d10cc6d2b8895797dcf51033605e2c7
+  metadata.gz: 6dea8b34c6e2cfd3ff2d8d85aaaef9875b7e051e270203f5db2951ee7ec12694
+  data.tar.gz: 68cbf6f504addd239757d34ad30be2d8fda169243191be1be043289b317a8797
 SHA512:
-  metadata.gz: acfa560d423d17f40e94d6605e7915b4bb2c9f8f1e52530735cf8be887451fa4990f986f8f7a314c43ce38e9d03771c9e33c6d311270dc9a18201aa2ea1760f7
-  data.tar.gz: abfa953bfc26962c069839a152cb05616cabb5ac8035760210fd0b815bd2efed13f1ffb597f5f027eda71d95f4a4dcc82de3c618d5b2e059aacca1d8dcb433a7
+  metadata.gz: 1a5284c2ee798e99301a9319b72d17c94a8625fa47d0fc8ad6b219add728d252115f8a4af4a3a1144ddfdf09ea8857bfa0034546628bcb14d184729f6c366b33
+  data.tar.gz: 5894ce676884bc96e131dfa827c1e1e74d47f83d0597211254e1d9b59063c96766213e7cf3585e0eff68eed185e200d4c80555a89faf0c2c1f95d1421b5c929c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.4)
+    funktor (0.2.9)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor
@@ -9,7 +9,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3.2)
+    activesupport (6.1.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -18,8 +18,8 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.1)
-    aws-partitions (1.468.0)
-    aws-sdk-core (3.114.3)
+    aws-partitions (1.472.0)
+    aws-sdk-core (3.115.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)

data/README.md

@@ -34,41 +34,20 @@ npm install -g serverless
 
 Then you can initialize a new app by doing:
 
-
 ```bash
+funktor bootstrap my-funktor-app
+cd my-funktor-app
 funktor init
 ```
 
+Then you sh
+
 This will create a `funktor` directory that is ready to deploy to AWS. If you've already configured
 your aws tools via `~/.aws/credentials` you should be ready to deploy.
 
-`funktor/serverless.yml` is the main file that ties everything together.
-
-`funktor/config` contains a few files that you can use to configure your `funktor` application.
-
-`funktor/resources` contains a few files that provision some AWS resources that are used by `funktor`.
-* An SQS Queue for the "incoming jobs queue"
-* A Dynamo DB table to allow queueing of jobs more than 15 minutes in the future (Funktor Pro)
-* One or more SQS Queues for active jobs (currently there is only the default queue, support for additional queues is coming soon)
-* An IAM User with permission to push jobs to the incoming jobs queue
-* A CloudWatch dashboard to let you keep tabs on your application
-
-`funktor/lambda_handlers` contains some scripts that receive events from Lambda, then invoke `funktor` to
-do various things:
-* `active_job_handler.rb` executes your jobs
-* `delayed_job_scheduler.rb` (Funktor Pro) pulls delayed jobs out of DynamoDB and places them on the active job queue.
-* `incoming_job_handler.rb` receives incoming jobs and pushes them to DynamoDB for delayed execution or to the active job queue as appropriate.
-
-`funktor/function_definitions` contains details about hooking up the `lambda_handlers` to events.
-
-`funktor/iam_permissions` contains some details about giving your lambda functions the appropriate permissions
-to interact with SQS.
-
-`funktor/workers` is where your workers will live.
-
-`funktor/Gemfile` is the `Gemfile` that contains the gems that are needed for your workers to execute
-jobs. This should be the minimal set of gems you can get away with so that cold start times remain reasonable.
-This file will already contain `funktor`. (Don't remove it or `funktor` won't work!)
+See the [wiki](https://github.com/Octo-Labs/funktor/wiki)
+for more info, especially the
+[section about getting started in a stand alone project](https://github.com/Octo-Labs/funktor/wiki/Getting-started-in-a-stand-alone-project).
 
 ## Deploying
 

data/lib/funktor/cli/init.rb

@@ -30,14 +30,13 @@ module Funktor
         name
       end
 
-      
-
       def funktor_config_yml
         #template "funktor_config.yml", File.join("funktor_config.yml")
         template File.join("config", "funktor.yml"), File.join("config", "funktor.yml")
         template File.join("config", "ruby_layer.yml"), File.join("config", "ruby_layer.yml")
         template File.join("config", "package.yml"), File.join("config", "package.yml")
         template File.join("config", "environment.yml"), File.join("config", "environment.yml")
+        template File.join("config", "boot.rb"), File.join("config", "boot.rb")
       end
 
       def package_json
@@ -88,11 +87,11 @@ module Funktor
       end
 
       def lambda_handlers
-        template File.join("app", "handlers", "incoming_job_handler.rb"), File.join("app", "handlers", "incoming_job_handler.rb")
+        template File.join("lambda_event_handlers", "incoming_job_handler.rb"), File.join("lambda_event_handlers", "incoming_job_handler.rb")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
-          template File.join("app", "handlers", "work_queue_handler.rb"), File.join("app", "handlers", "#{work_queue_name.underscore}_queue_handler.rb")
+          template File.join("lambda_event_handlers", "work_queue_handler.rb"), File.join("lambda_event_handlers", "#{work_queue_name.underscore}_queue_handler.rb")
         end
       end
 
@@ -172,7 +171,6 @@ module Funktor
       end
 
       def queue_config_value(queue_name, config_name)
-        puts funktor_config
         queue_config(queue_name)&.dig(config_name) ||
           funktor_config.dig("handlerDefaults", config_name) ||
           "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template

data/lib/funktor/cli/templates/config/boot.rb

@@ -0,0 +1,16 @@
+# Point at our Gemfile
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+
+# load rubygems & bundler
+require "rubygems"
+require 'bundler/setup'
+
+# Set up gems listed in the Gemfile.
+Bundler.require(:default, :production)
+
+# Load all ruby files in the app directory
+Dir.glob( File.join('..', 'app', '**', '*.rb'), base: File.dirname(__FILE__) ).each do |ruby_file|
+  puts "require_relative #{ruby_file}"
+  require_relative ruby_file
+end
+

data/lib/funktor/cli/templates/config/environment.yml

@@ -1,3 +1,4 @@
+FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue
 <%- queue_names.each do |queue_name| -%>

data/lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,8 @@
-handler: app/handlers/incoming_job_handler.call
-timeout: ${self:custom.funktor.incomingJobHandler.timeout, 30}
-reservedConcurrency: ${self:custom.funktor.incomingJobHandler.reservedConcurrency, null}
-provisionedConcurrency: ${self:custom.funktor.incomingJobHandler.provisionedConcurrency, null}
-memorySize: ${self:custom.funktor.incomingJobHandler.memorySize, 256}
+handler: lambda_event_handlers/incoming_job_handler.call
+timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
+reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
 events:
   - sqs:
       arn:

data/lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml

@@ -1,4 +1,4 @@
-handler: app/handlers/<%= work_queue_name.underscore %>_queue_handler.call
+handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.timeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}

data/lib/funktor/cli/templates/config/funktor.yml

@@ -1,4 +1,4 @@
-incomingJobHandler:
+IncomingJobHandler:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
@@ -51,10 +51,11 @@ incomingJobHandler:
 
 # You shouldn't need to mess with these under most circumstances. But you could if you want to change
 # the name of some of your resources in AWS.
-incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
-incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
-incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
-dashboardName: ${self:service}-${self:custom.stage}-dashboard
+IncomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
+IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
+IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
+DashboardName: ${self:service}-${self:custom.stage}-dashboard
 <%- queue_names.each do |queue_name| -%>
 <%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
 <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead

data/lib/funktor/cli/templates/config/package.yml

@@ -3,6 +3,7 @@ individually: false
 include:
   - Gemfile
   - Gemfile.lock
+  - config/boot.rb
   - app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.

data/lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml

@@ -2,7 +2,7 @@ Resources:
   FunktorDashboard:
     Type: AWS::CloudWatch::Dashboard
     Properties:
-      DashboardName: ${self:custom.funktor.dashboardName}
+      DashboardName: ${self:custom.funktor.DashboardName}
       DashboardBody: >
         {
             "widgets": [
@@ -25,7 +25,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Messages Per Minute" } ]
                         ],
                         "view": "singleValue",
                         "region": "us-east-1",
@@ -42,7 +42,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "p10" } ],
                             [ "...", { "label": "p50", "stat": "p50" } ],
                             [ "...", { "label": "p99", "stat": "p99" } ],
                             [ "...", { "label": "Average", "stat": "Average" } ]
@@ -63,10 +63,10 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Received" } ],
                             [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
-                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
-                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
+                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "Handler Invocations" } ],
+                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Pending?" } ],
                             [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
                             [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
                             [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
@@ -90,7 +90,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Minimum" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Minimum" } ],
                             [ "...", { "stat": "Average" } ],
                             [ "...", { "stat": "Maximum" } ]
                         ],
@@ -110,7 +110,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
+                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
                             [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
                             [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
                         ],
@@ -137,7 +137,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Maximum" } ]
+                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Maximum" } ]
                         ],
                         "region": "us-east-1",
                         "title": "Incoming Job Handler Concurrent Executions",

data/lib/funktor/cli/templates/config/resources/incoming_job_queue.yml

@@ -2,7 +2,7 @@ Resources:
   IncomingJobQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingJobQueueName}
       VisibilityTimeout: 300
       RedrivePolicy:
         deadLetterTargetArn:
@@ -11,7 +11,7 @@ Resources:
   IncomingJobDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingDeadJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingDeadJobQueueName}
 
 Outputs:
   IncomingJobQueueUrl:

data/lib/funktor/cli/templates/{app/handlers → lambda_event_handlers}/incoming_job_handler.rb

@@ -1,4 +1,4 @@
-require 'funktor'
+require_relative '../config/boot'
 
 $handler = Funktor::IncomingJobHandler.new
 

data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb

@@ -0,0 +1,8 @@
+require_relative '../config/boot'
+
+$handler = Funktor::ActiveJobHandler.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/lib/funktor/middleware/metrics.rb

@@ -19,7 +19,7 @@ module Funktor
             "Timestamp": Time.now.strftime('%s%3N').to_i,
             "CloudWatchMetrics": [
               {
-                "Namespace": "rails-lambda-experiment", # TODO - We should get this from config or someting
+                "Namespace": ENV['FUNKTOR_APP_NAME'], # TODO - We should get this from config or someting
                 "Dimensions": [["WorkerClassName"]],
                 "Metrics": [ # CPU, Memory, Duration, etc...
                              {

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.4"
+  VERSION = "0.2.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.9
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-21 00:00:00.000000000 Z
+date: 2021-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -115,9 +115,8 @@ files:
 - lib/funktor/cli/generate/work_queue.rb
 - lib/funktor/cli/init.rb
 - lib/funktor/cli/templates/Gemfile
-- lib/funktor/cli/templates/app/handlers/incoming_job_handler.rb
-- lib/funktor/cli/templates/app/handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/app/workers/hello_worker.rb
+- lib/funktor/cli/templates/config/boot.rb
 - lib/funktor/cli/templates/config/environment.yml
 - lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml
 - lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml
@@ -133,6 +132,8 @@ files:
 - lib/funktor/cli/templates/config/ruby_layer.yml
 - lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
+- lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
+- lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/package.json
 - lib/funktor/cli/templates/serverless.yml
 - lib/funktor/fake_job_queue.rb

data/lib/funktor/cli/templates/app/handlers/work_queue_handler.rb

@@ -1,17 +0,0 @@
-require 'funktor'
-
-# Bundler is hard to make work because AWS includes some gems in the basic ruby runtime.
-# We're probably going to need to use containers...
-#require "rubygems"
-#require "bundler/setup"
-#Bundler.require(:default)
-
-# TODO : Ideally this wouldn't be needed
-require_relative '../workers/hello_worker'
-
-$handler = Funktor::ActiveJobHandler.new
-
-def call(event:, context:)
-  $handler.call(event: event, context: context)
-end
-