funktor 0.2.5 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ca7738fdb4ce7e628d489b4eafa1975036cc1e82ec162bc4115383ac2b29233
-  data.tar.gz: 0a328bcb1d0c5a007d560eda72d88832269f9d21def9bfea3720746600f828f0
+  metadata.gz: 16ad166a922c31ac7de1008d48fe34824203b0d325e7faae2f7ede140fc0f836
+  data.tar.gz: fafbd5ca0858ecb87c2c9e87c8efa52accc9262f922d6f3f1d4b311b3c8c470b
 SHA512:
-  metadata.gz: 274578bd0e322e8cb4d5bfbdb27c5aaa97d966e37bc3e38ef63cec5536947e132f9b5018b78ba52f6d492ed73ba8f46ac872614b750bcb4fab425613c5eccbfe
-  data.tar.gz: 454d9ffaf4970543f409be989ce455717c5f600e81c85f6ba8aea640867d232c4aaf2ec9b8f79e7956da53db197668a6bb37627de5c18ceac8a9348dfe527521
+  metadata.gz: b40e4deee38ed584b93c863150d2d6c45dfc3aea1ac3ed0d7aa4d249c5661b527f252cf1b700545f54e80f98bf6158345c01b218f175eef117ce31626c5f5b21
+  data.tar.gz: 240839bae2964df499664ec8c21a44d3c8c4cf46d3d6df5cb798af95ba3b454deee6a41d5b4b0a3d71fc8c04e36bdfdb0d787f6168e71ef1cb516370e1c0d369

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.5)
+    funktor (0.2.10)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor
@@ -18,7 +18,7 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.1)
-    aws-partitions (1.471.0)
+    aws-partitions (1.472.0)
     aws-sdk-core (3.115.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)

data/README.md

@@ -34,41 +34,20 @@ npm install -g serverless
 
 Then you can initialize a new app by doing:
 
-
 ```bash
+funktor bootstrap my-funktor-app
+cd my-funktor-app
 funktor init
 ```
 
+Then you sh
+
 This will create a `funktor` directory that is ready to deploy to AWS. If you've already configured
 your aws tools via `~/.aws/credentials` you should be ready to deploy.
 
-`funktor/serverless.yml` is the main file that ties everything together.
-
-`funktor/config` contains a few files that you can use to configure your `funktor` application.
-
-`funktor/resources` contains a few files that provision some AWS resources that are used by `funktor`.
-* An SQS Queue for the "incoming jobs queue"
-* A Dynamo DB table to allow queueing of jobs more than 15 minutes in the future (Funktor Pro)
-* One or more SQS Queues for active jobs (currently there is only the default queue, support for additional queues is coming soon)
-* An IAM User with permission to push jobs to the incoming jobs queue
-* A CloudWatch dashboard to let you keep tabs on your application
-
-`funktor/lambda_handlers` contains some scripts that receive events from Lambda, then invoke `funktor` to
-do various things:
-* `active_job_handler.rb` executes your jobs
-* `delayed_job_scheduler.rb` (Funktor Pro) pulls delayed jobs out of DynamoDB and places them on the active job queue.
-* `incoming_job_handler.rb` receives incoming jobs and pushes them to DynamoDB for delayed execution or to the active job queue as appropriate.
-
-`funktor/function_definitions` contains details about hooking up the `lambda_handlers` to events.
-
-`funktor/iam_permissions` contains some details about giving your lambda functions the appropriate permissions
-to interact with SQS.
-
-`funktor/workers` is where your workers will live.
-
-`funktor/Gemfile` is the `Gemfile` that contains the gems that are needed for your workers to execute
-jobs. This should be the minimal set of gems you can get away with so that cold start times remain reasonable.
-This file will already contain `funktor`. (Don't remove it or `funktor` won't work!)
+See the [wiki](https://github.com/Octo-Labs/funktor/wiki)
+for more info, especially the
+[section about getting started in a stand alone project](https://github.com/Octo-Labs/funktor/wiki/Getting-started-in-a-stand-alone-project).
 
 ## Deploying
 

data/lib/funktor/cli/init.rb

@@ -30,14 +30,13 @@ module Funktor
         name
       end
 
-      
-
       def funktor_config_yml
         #template "funktor_config.yml", File.join("funktor_config.yml")
         template File.join("config", "funktor.yml"), File.join("config", "funktor.yml")
         template File.join("config", "ruby_layer.yml"), File.join("config", "ruby_layer.yml")
         template File.join("config", "package.yml"), File.join("config", "package.yml")
         template File.join("config", "environment.yml"), File.join("config", "environment.yml")
+        template File.join("config", "boot.rb"), File.join("config", "boot.rb")
       end
 
       def package_json
@@ -172,7 +171,6 @@ module Funktor
       end
 
       def queue_config_value(queue_name, config_name)
-        puts funktor_config
         queue_config(queue_name)&.dig(config_name) ||
           funktor_config.dig("handlerDefaults", config_name) ||
           "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template

data/lib/funktor/cli/templates/config/boot.rb

@@ -0,0 +1,16 @@
+# Point at our Gemfile
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+
+# load rubygems & bundler
+require "rubygems"
+require 'bundler/setup'
+
+# Set up gems listed in the Gemfile.
+Bundler.require(:default, :production)
+
+# Load all ruby files in the app directory
+Dir.glob( File.join('..', 'app', '**', '*.rb'), base: File.dirname(__FILE__) ).each do |ruby_file|
+  puts "require_relative #{ruby_file}"
+  require_relative ruby_file
+end
+

data/lib/funktor/cli/templates/config/environment.yml

@@ -1,3 +1,4 @@
+FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue
 <%- queue_names.each do |queue_name| -%>

data/lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,8 @@
 handler: lambda_event_handlers/incoming_job_handler.call
-timeout: ${self:custom.funktor.incomingJobHandler.timeout, 30}
-reservedConcurrency: ${self:custom.funktor.incomingJobHandler.reservedConcurrency, null}
-provisionedConcurrency: ${self:custom.funktor.incomingJobHandler.provisionedConcurrency, null}
-memorySize: ${self:custom.funktor.incomingJobHandler.memorySize, 256}
+timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
+reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
 events:
   - sqs:
       arn:

data/lib/funktor/cli/templates/config/funktor.yml

@@ -1,4 +1,4 @@
-incomingJobHandler:
+IncomingJobHandler:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
@@ -51,10 +51,11 @@ incomingJobHandler:
 
 # You shouldn't need to mess with these under most circumstances. But you could if you want to change
 # the name of some of your resources in AWS.
-incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
-incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
-incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
-dashboardName: ${self:service}-${self:custom.stage}-dashboard
+IncomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
+IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
+IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
+DashboardName: ${self:service}-${self:custom.stage}-dashboard
 <%- queue_names.each do |queue_name| -%>
 <%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
 <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead

data/lib/funktor/cli/templates/config/package.yml

@@ -3,6 +3,7 @@ individually: false
 include:
   - Gemfile
   - Gemfile.lock
+  - config/boot.rb
   - app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.

data/lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml

@@ -2,7 +2,7 @@ Resources:
   FunktorDashboard:
     Type: AWS::CloudWatch::Dashboard
     Properties:
-      DashboardName: ${self:custom.funktor.dashboardName}
+      DashboardName: ${self:custom.funktor.DashboardName}
       DashboardBody: >
         {
             "widgets": [
@@ -25,7 +25,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Messages Per Minute" } ]
                         ],
                         "view": "singleValue",
                         "region": "us-east-1",
@@ -42,7 +42,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "p10" } ],
                             [ "...", { "label": "p50", "stat": "p50" } ],
                             [ "...", { "label": "p99", "stat": "p99" } ],
                             [ "...", { "label": "Average", "stat": "Average" } ]
@@ -63,10 +63,10 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Received" } ],
                             [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
-                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
-                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
+                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "Handler Invocations" } ],
+                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Pending?" } ],
                             [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
                             [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
                             [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
@@ -90,7 +90,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Minimum" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Minimum" } ],
                             [ "...", { "stat": "Average" } ],
                             [ "...", { "stat": "Maximum" } ]
                         ],
@@ -110,7 +110,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
+                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
                             [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
                             [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
                         ],
@@ -137,7 +137,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Maximum" } ]
+                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Maximum" } ]
                         ],
                         "region": "us-east-1",
                         "title": "Incoming Job Handler Concurrent Executions",

data/lib/funktor/cli/templates/config/resources/incoming_job_queue.yml

@@ -2,7 +2,7 @@ Resources:
   IncomingJobQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingJobQueueName}
       VisibilityTimeout: 300
       RedrivePolicy:
         deadLetterTargetArn:
@@ -11,7 +11,7 @@ Resources:
   IncomingJobDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingDeadJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingDeadJobQueueName}
 
 Outputs:
   IncomingJobQueueUrl:

data/lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb

@@ -1,3 +1,6 @@
+# For this handler we don't need to know about your app, or any of the other gems,
+# so instead of doing `require_relative '../config/boog'` we just manually require
+# the one gem that we do need.
 require 'funktor'
 
 $handler = Funktor::IncomingJobHandler.new

data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb

@@ -1,13 +1,4 @@
-require 'funktor'
-
-# Bundler is hard to make work because AWS includes some gems in the basic ruby runtime.
-# We're probably going to need to use containers...
-#require "rubygems"
-#require "bundler/setup"
-#Bundler.require(:default)
-
-# TODO : Ideally this wouldn't be needed
-require_relative '../workers/hello_worker'
+require_relative '../config/boot'
 
 $handler = Funktor::ActiveJobHandler.new
 

data/lib/funktor/job.rb

@@ -43,7 +43,7 @@ module Funktor
     end
 
     def execute
-      worker_class.new.perform(worker_params)
+      worker_class.new.perform(*worker_params)
     end
 
     def worker_class

data/lib/funktor/middleware/metrics.rb

@@ -19,7 +19,7 @@ module Funktor
             "Timestamp": Time.now.strftime('%s%3N').to_i,
             "CloudWatchMetrics": [
               {
-                "Namespace": "rails-lambda-experiment", # TODO - We should get this from config or someting
+                "Namespace": ENV['FUNKTOR_APP_NAME'],
                 "Dimensions": [["WorkerClassName"]],
                 "Metrics": [ # CPU, Memory, Duration, etc...
                              {

data/lib/funktor/testing.rb

@@ -57,7 +57,7 @@ module Funktor
 
   class InlineJobPusherMiddleware
     def call(worker, payload)
-      worker.new.perform(payload[:worker_params])
+      worker.new.perform(*payload[:worker_params])
     end
   end
 

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.5"
+  VERSION = "0.2.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.10
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-29 00:00:00.000000000 Z
+date: 2021-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -116,6 +116,7 @@ files:
 - lib/funktor/cli/init.rb
 - lib/funktor/cli/templates/Gemfile
 - lib/funktor/cli/templates/app/workers/hello_worker.rb
+- lib/funktor/cli/templates/config/boot.rb
 - lib/funktor/cli/templates/config/environment.yml
 - lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml
 - lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml