funktor 0.2.6 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a92d7870ba1cde9f91c5d872aab7e5dcc9d7a0d9ee8b4deb75fa1e679ea6ddc
-  data.tar.gz: 79d938d9d46f9763ae6bf7001bf54bb1ad48eefee0a0b5dcbb624a934edbba85
+  metadata.gz: 99102148cae3682c1c870882cd44b9651869e3d1ec3adb7e3b4145ef7a6becf5
+  data.tar.gz: c6adb462dbe095673a72e81570d293a59a7f75d18376178684cabe5c81342f3c
 SHA512:
-  metadata.gz: b741a360102285bc392c6b846677b1859dbb02652b9e9412773fa989172c4a2368429f7825736f4438a543f4ac058098884fd78ce8a8f89be81b10f6b2e4db94
-  data.tar.gz: a450bec63e833671d3eeb7e4e1f00c38eec6c66c59de610ba431f8660a1273dac0ab4db85ff7b41ca2ff9e234c19f771087127d16ab2a21612b731ed61678609
+  metadata.gz: 501e7dd2f92ff36d492da512861bf4dd6ab56f07a4fcbc6c362fa5249a52f0e2694b835d6eda1b454304046be2315c8a9a253c92a25e8548605a7dc4829008ac
+  data.tar.gz: fa98731784d2b0007b9bf831b2536f9ec93bbb5a8870e7f5b9e8810b414002d1f760000d18fa581b7e6f8016f48e9dbca1d905bf1d2821dbfaf85cdec80158dc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.6)
+    funktor (0.2.11)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor
@@ -18,7 +18,7 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.1)
-    aws-partitions (1.471.0)
+    aws-partitions (1.472.0)
     aws-sdk-core (3.115.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)

data/README.md

@@ -34,41 +34,20 @@ npm install -g serverless
 
 Then you can initialize a new app by doing:
 
-
 ```bash
+funktor bootstrap my-funktor-app
+cd my-funktor-app
 funktor init
 ```
 
+Then you sh
+
 This will create a `funktor` directory that is ready to deploy to AWS. If you've already configured
 your aws tools via `~/.aws/credentials` you should be ready to deploy.
 
-`funktor/serverless.yml` is the main file that ties everything together.
-
-`funktor/config` contains a few files that you can use to configure your `funktor` application.
-
-`funktor/resources` contains a few files that provision some AWS resources that are used by `funktor`.
-* An SQS Queue for the "incoming jobs queue"
-* A Dynamo DB table to allow queueing of jobs more than 15 minutes in the future (Funktor Pro)
-* One or more SQS Queues for active jobs (currently there is only the default queue, support for additional queues is coming soon)
-* An IAM User with permission to push jobs to the incoming jobs queue
-* A CloudWatch dashboard to let you keep tabs on your application
-
-`funktor/lambda_handlers` contains some scripts that receive events from Lambda, then invoke `funktor` to
-do various things:
-* `active_job_handler.rb` executes your jobs
-* `delayed_job_scheduler.rb` (Funktor Pro) pulls delayed jobs out of DynamoDB and places them on the active job queue.
-* `incoming_job_handler.rb` receives incoming jobs and pushes them to DynamoDB for delayed execution or to the active job queue as appropriate.
-
-`funktor/function_definitions` contains details about hooking up the `lambda_handlers` to events.
-
-`funktor/iam_permissions` contains some details about giving your lambda functions the appropriate permissions
-to interact with SQS.
-
-`funktor/workers` is where your workers will live.
-
-`funktor/Gemfile` is the `Gemfile` that contains the gems that are needed for your workers to execute
-jobs. This should be the minimal set of gems you can get away with so that cold start times remain reasonable.
-This file will already contain `funktor`. (Don't remove it or `funktor` won't work!)
+See the [wiki](https://github.com/Octo-Labs/funktor/wiki)
+for more info, especially the
+[section about getting started in a stand alone project](https://github.com/Octo-Labs/funktor/wiki/Getting-started-in-a-stand-alone-project).
 
 ## Deploying
 

data/lib/funktor.rb

@@ -1,6 +1,7 @@
 require "funktor/version"
 require 'funktor/aws/sqs/event'
 require 'funktor/aws/sqs/record'
+require 'funktor/counter'
 require 'funktor/job'
 require 'funktor/worker'
 require 'funktor/middleware_chain'

data/lib/funktor/active_job_handler.rb

@@ -3,6 +3,11 @@ require 'aws-sdk-sqs'
 module Funktor
   class ActiveJobHandler
 
+    def initialize
+      @failed_counter = Funktor::Counter.new('failed')
+      @processed_counter = Funktor::Counter.new('processed')
+    end
+
     def call(event:, context:)
       event = Funktor::Aws::Sqs::Event.new(event)
       puts "event.jobs.count = #{event.jobs.count}"
@@ -20,10 +25,12 @@ module Funktor
         Funktor.active_job_handler_middleware.invoke(job) do
           job.execute
         end
+        @processed_counter.incr(job)
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
         puts "Error during processing: #{$!}"
         puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
+        @failed_counter.incr(job)
         attempt_retry_or_bail(job)
       end
     end

data/lib/funktor/cli/init.rb

@@ -171,7 +171,6 @@ module Funktor
       end
 
       def queue_config_value(queue_name, config_name)
-        puts funktor_config
         queue_config(queue_name)&.dig(config_name) ||
           funktor_config.dig("handlerDefaults", config_name) ||
           "null" # When we parse yaml 'null' gets turned to nil, which comes out as an empty string in the template

data/lib/funktor/cli/templates/config/environment.yml

@@ -1,3 +1,4 @@
+FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue
 <%- queue_names.each do |queue_name| -%>

data/lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,8 @@
 handler: lambda_event_handlers/incoming_job_handler.call
-timeout: ${self:custom.funktor.incomingJobHandler.timeout, 30}
-reservedConcurrency: ${self:custom.funktor.incomingJobHandler.reservedConcurrency, null}
-provisionedConcurrency: ${self:custom.funktor.incomingJobHandler.provisionedConcurrency, null}
-memorySize: ${self:custom.funktor.incomingJobHandler.memorySize, 256}
+timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
+reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
 events:
   - sqs:
       arn:

data/lib/funktor/cli/templates/config/funktor.yml

@@ -1,4 +1,4 @@
-incomingJobHandler:
+IncomingJobHandler:
   # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
   # to a handler at one time, so you'll want this to be at least 10x the maximum time you
   # expect to spend for one message. The incoming job handler usually will be pretty fast,
@@ -51,10 +51,11 @@ incomingJobHandler:
 
 # You shouldn't need to mess with these under most circumstances. But you could if you want to change
 # the name of some of your resources in AWS.
-incomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
-incomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
-incomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
-dashboardName: ${self:service}-${self:custom.stage}-dashboard
+IncomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
+IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
+IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
+DashboardName: ${self:service}-${self:custom.stage}-dashboard
 <%- queue_names.each do |queue_name| -%>
 <%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
 <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead

data/lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml

@@ -2,7 +2,7 @@ Resources:
   FunktorDashboard:
     Type: AWS::CloudWatch::Dashboard
     Properties:
-      DashboardName: ${self:custom.funktor.dashboardName}
+      DashboardName: ${self:custom.funktor.DashboardName}
       DashboardBody: >
         {
             "widgets": [
@@ -25,7 +25,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Messages Per Minute" } ]
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Messages Per Minute" } ]
                         ],
                         "view": "singleValue",
                         "region": "us-east-1",
@@ -42,7 +42,7 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "p10" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "p10" } ],
                             [ "...", { "label": "p50", "stat": "p50" } ],
                             [ "...", { "label": "p99", "stat": "p99" } ],
                             [ "...", { "label": "Average", "stat": "Average" } ]
@@ -63,10 +63,10 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Received" } ],
+                            [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Received" } ],
                             [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
-                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", "Resource", "${self:service}-${self:provider.stage}-incoming_job_handler", { "label": "Handler Invocations" } ],
-                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.incomingJobQueueName}", { "label": "Pending?" } ],
+                            [ "AWS/Lambda", "Invocations", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", "Resource", "${self:custom.funktor.IncomingJobHandlerName}", { "label": "Handler Invocations" } ],
+                            [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName", "${self:custom.funktor.IncomingJobQueueName}", { "label": "Pending?" } ],
                             [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
                             [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
                             [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
@@ -90,7 +90,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Minimum" } ],
+                            [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Minimum" } ],
                             [ "...", { "stat": "Average" } ],
                             [ "...", { "stat": "Maximum" } ]
                         ],
@@ -110,7 +110,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
+                            [ "AWS/Lambda", "Errors", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
                             [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
                             [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
                         ],
@@ -137,7 +137,7 @@ Resources:
                     "properties": {
                         "period": 60,
                         "metrics": [
-                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:service}-${self:provider.stage}-incoming_job_handler", { "stat": "Maximum" } ]
+                            [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName", "${self:custom.funktor.IncomingJobHandlerName}", { "stat": "Maximum" } ]
                         ],
                         "region": "us-east-1",
                         "title": "Incoming Job Handler Concurrent Executions",

data/lib/funktor/cli/templates/config/resources/incoming_job_queue.yml

@@ -2,7 +2,7 @@ Resources:
   IncomingJobQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingJobQueueName}
       VisibilityTimeout: 300
       RedrivePolicy:
         deadLetterTargetArn:
@@ -11,7 +11,7 @@ Resources:
   IncomingJobDeadLetterQueue:
     Type: AWS::SQS::Queue
     Properties:
-      QueueName: ${self:custom.funktor.incomingDeadJobQueueName}
+      QueueName: ${self:custom.funktor.IncomingDeadJobQueueName}
 
 Outputs:
   IncomingJobQueueUrl:

data/lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb

@@ -1,4 +1,7 @@
-require_relative '../config/boot'
+# For this handler we don't need to know about your app, or any of the other gems,
+# so instead of doing `require_relative '../config/boog'` we just manually require
+# the one gem that we do need.
+require 'funktor'
 
 $handler = Funktor::IncomingJobHandler.new
 

data/lib/funktor/counter.rb

@@ -0,0 +1,42 @@
+module Funktor
+  class Counter
+    attr_accessor :dimension
+
+    def initialize(dimension)
+      @dimension = dimension
+    end
+
+    def incr(job)
+      put_metric_to_stdout(job)
+    end
+
+    def put_metric_to_stdout(job)
+      puts Funktor.dump_json(metric_hash(job))
+    end
+
+    def metric_hash(job)
+      {
+        "_aws": {
+          "Timestamp": Time.now.strftime('%s%3N').to_i,
+          "CloudWatchMetrics": [
+            {
+              "Namespace": ENV['FUNKTOR_APP_NAME'],
+              "Dimensions": [["WorkerClassName"]],
+              "Metrics": [ # CPU, Memory, Duration, etc...
+                           {
+                             "Name": dimension,
+                             "Unit": "Count"
+                           }
+              ]
+            }
+          ]
+        },
+        "WorkerClassName": job.worker_class_name,
+        "#{dimension}": 1
+        #"count": value,
+        #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
+      }
+      #data[key] = value
+    end
+  end
+end

data/lib/funktor/job.rb

@@ -43,7 +43,7 @@ module Funktor
     end
 
     def execute
-      worker_class.new.perform(worker_params)
+      worker_class.new.perform(*worker_params)
     end
 
     def worker_class

data/lib/funktor/middleware/metrics.rb

@@ -19,7 +19,7 @@ module Funktor
             "Timestamp": Time.now.strftime('%s%3N').to_i,
             "CloudWatchMetrics": [
               {
-                "Namespace": "rails-lambda-experiment", # TODO - We should get this from config or someting
+                "Namespace": ENV['FUNKTOR_APP_NAME'],
                 "Dimensions": [["WorkerClassName"]],
                 "Metrics": [ # CPU, Memory, Duration, etc...
                              {

data/lib/funktor/testing.rb

@@ -57,7 +57,7 @@ module Funktor
 
   class InlineJobPusherMiddleware
     def call(worker, payload)
-      worker.new.perform(payload[:worker_params])
+      worker.new.perform(*payload[:worker_params])
     end
   end
 

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.6"
+  VERSION = "0.2.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.11
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-29 00:00:00.000000000 Z
+date: 2021-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -136,6 +136,7 @@ files:
 - lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/package.json
 - lib/funktor/cli/templates/serverless.yml
+- lib/funktor/counter.rb
 - lib/funktor/fake_job_queue.rb
 - lib/funktor/incoming_job_handler.rb
 - lib/funktor/job.rb