funktor 0.7.3 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58967787594138d8c3e24feb2cb0cf5c742899d494099e972a505de9a49e8bc2
-  data.tar.gz: bae55701018e66b63cacbd5d7911cdd5fa1f063969716f6726be423ffaee896b
+  metadata.gz: 00c5852fd10c7e6ad87b576ff0a1ed13b62f69fc19391202d4c2d289376a3fb2
+  data.tar.gz: e780ae7466a2c4dfc7060abfacd230a9284105c89aa95db376e99b7fe14a2b89
 SHA512:
-  metadata.gz: 2251b6acccc727761a109cee8cf847000c4857726441cc01dc04cb6649f53b272eb0d0f335494d31a1089cfd335988073a872dd660295dc55e6e4c07e9bf813a
-  data.tar.gz: b785cc6ba1db1c2e4e3af54b0a1c40045bb0f1d9cf2456edd82be1c1b370fa23edd71bc36edc5f8dfd9cf08990dcf490b5ec20ebd855d2da5610e48d7737e174
+  metadata.gz: e7015cc189689fbffb456727899417e1edb7f823436536b6a3f316a96a83291bdaf0ee43b860d2925b53c239be15d685df268bf277c0843efc853b2b07abe0d6
+  data.tar.gz: a8d58b3d8d45b5bd2ecd70c1f18956685af350959cd6d3263e212d611e9e4d7d069b52681866e725cc432af09a434465e0b9bff80dc94d7a444f88d21a87a36d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.7.3)
+    funktor (0.7.5)
       activesupport
       aws-sdk-dynamodb (~> 1.62)
       aws-sdk-sqs (~> 1.37)
@@ -22,19 +22,19 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.2.0)
-    aws-partitions (1.494.0)
-    aws-sdk-core (3.120.0)
+    aws-partitions (1.584.0)
+    aws-sdk-core (3.130.2)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
+      aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-dynamodb (1.63.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+    aws-sdk-dynamodb (1.74.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.44.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+    aws-sdk-sqs (1.51.1)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.3.0)
+    aws-sigv4 (1.5.0)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     coderay (1.1.3)
@@ -48,7 +48,7 @@ GEM
     hashdiff (1.0.1)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    jmespath (1.4.0)
+    jmespath (1.6.1)
     method_source (1.0.0)
     minitest (5.14.4)
     mustermann (1.1.1)
@@ -90,7 +90,7 @@ GEM
       rack (~> 2.2)
       rack-protection (= 2.1.0)
       tilt (~> 2.0)
-    thor (1.1.0)
+    thor (1.2.1)
     tilt (2.0.10)
     timecop (0.9.4)
     tzinfo (2.0.4)

data/README.md

@@ -40,8 +40,6 @@ cd my-funktor-app
 funktor init
 ```
 
-Then you sh
-
 This will create a `funktor` directory that is ready to deploy to AWS. If you've already configured
 your aws tools via `~/.aws/credentials` you should be ready to deploy.
 

data/funktor-testapp/Dockerfile

@@ -0,0 +1,63 @@
+FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
+
+# Uncomment this as a cache buster
+#RUN echo date
+
+# If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
+#ARG BUNDLE_GEM__FURY__IO
+#ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
+
+# First we install some stuff that we need for gems that have to compile native extensions
+#RUN yum groupinstall "Development Tools" -y
+#RUN yum install -y amazon-linux-extras
+#RUN amazon-linux-extras enable postgresql11
+#RUN yum install -y postgresql-devel
+
+# Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
+COPY Gemfile Gemfile.lock .
+
+# Set a few bundle configuration options
+RUN bundle lock --add-platform x86_64-linux
+#RUN bundle config --local deployment true
+#RUN bundle config --local plugins false
+#RUN bundle config --local frozen true
+#RUN bundle config --local without 'development test'
+RUN bundle config --local path './vendor/bundle'
+
+# Now install our gems
+RUN bundle install --quiet --jobs 4
+
+# Now we start a second stage in the build that is a clean image without build tools
+FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
+
+#ENV RAILS_ENV=production
+#ENV RACK_ENV=production
+
+# Install node so that asset related gems have a JS runtime.
+# We ship the node executeable to production to make it easier to get an app deployed.
+# TODO: Document steps that could be taken to remove this dependency.
+#RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
+#RUN yum install -y nodejs
+
+# Then copy some postgres related files
+#COPY --from=build_image  /usr/lib64/libpq.so.5 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/liblber-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsasl2.so.3 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libssl3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsmime3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libnss3.so /usr/lib64/
+
+
+# Next we copy the app from our local directory and we copy
+# the bundled gems from the build image.
+# We do this after copying dependencies becuase the app will
+# change more frequently, and we can used caching up to here.
+
+COPY --from=build_image /var/task .
+COPY . .
+
+# And finally we have the CMD for the deployed container
+# You can overwrite command in `serverless.yml` template
+CMD [ "app.LambdaFunction::Handler.process" ]
+

data/funktor-testapp/app/workers/audit_worker.rb

@@ -21,13 +21,17 @@ class AuditWorker
     puts "So long from the #{self.class.name}, and thanks for all the fish!"
   end
 
+  def metric_namespace
+    [ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
+  end
+
   def metric_hash(time_diff)
     {
       "_aws": {
         "Timestamp": Time.now.strftime('%s%3N').to_i,
         "CloudWatchMetrics": [
           {
-            "Namespace": ENV['FUNKTOR_APP_NAME'],
+            "Namespace": metric_namespace,
             "Dimensions": [["WorkerClassName"]],
             "Metrics": [ # CPU, Memory, Duration, etc...
                          {

data/funktor-testapp/funktor_config/boot.rb

@@ -12,6 +12,9 @@ Bundler.require(:default, :production)
 $LOAD_PATH.unshift 'funktor/lib'
 require 'funktor'
 
+
+Funktor.enable_work_queue_visibility = false
+
 # Load all ruby files in the app directory
 require_rel File.join('..', 'app')
 

data/funktor-testapp/funktor_config/environment.yml

@@ -1,7 +1,16 @@
-BUNDLE_WITHOUT: development:test:assets
+BUNDLE_WITHOUT: development:test
 BUNDLE_PLUGINS: false
 BUNDLE_FROZEN: true
+BUNDLE_DEPLOYMENT: true
 SERVERLESS_STAGE: ${self:custom.stage}
+
+FUNKTOR_LOG_LEVEL: INFO
+
+RAILS_LOG_TO_STDOUT: true
+RAILS_ENV: production
+RACK_ENV: production
+RAILS_MAX_THREADS: 1
+
 FUNKTOR_APP_NAME: funktor-testapp
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/default_queue_handler.call
+#handler: lambda_event_handlers/default_queue_handler.call
 timeout: ${self:custom.funktor.DefaultQueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.DefaultQueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.DefaultQueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.DefaultQueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/default_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.DefaultQueueHandler.batchSize, 1}

data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/incoming_job_handler.call
+#handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/incoming_job_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}

data/funktor-testapp/funktor_config/function_definitions/job_activator.yml

@@ -1,7 +1,11 @@
-handler: lambda_event_handlers/job_activator.call
+#handler: lambda_event_handlers/job_activator.call
 timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/job_activator.call
 events:
   - schedule: rate(1 minute)

data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/low_concurrency_queue_handler.call
+#handler: lambda_event_handlers/low_concurrency_queue_handler.call
 timeout: ${self:custom.funktor.LowConcurrencyQueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.LowConcurrencyQueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/low_concurrency_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.LowConcurrencyQueueHandler.batchSize, 1}

data/funktor-testapp/funktor_config/funktor.yml

@@ -103,6 +103,7 @@ IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
 IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
 IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
 DashboardName: ${self:service}-${self:custom.stage}-dashboard
+DashboardNamespace: ${self:service}-${self:custom.stage}
 DefaultQueueName: ${self:service}-${self:custom.stage}-default
 DefaultDeadJobQueueName: ${self:service}-${self:custom.stage}-default-dead
 DefaultQueueHandlerName: ${self:service}-${self:custom.stage}-DefaultQueueHandler

data/funktor-testapp/funktor_config/package.yml

@@ -1,10 +1,10 @@
 # TODO - Figure out how to allow individual packaging to work out of the box.
-individually: false
-include:
-  - Gemfile
-  - Gemfile.lock
-  - funktor_config/boot.rb
-  - app/**
+#individually: false
+#include:
+  #- Gemfile
+  #- Gemfile.lock
+  #- funktor_config/boot.rb
+  #- app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml

@@ -29,13 +29,13 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "funktor-testapp", "Duration", "WorkerClassName", "AuditWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "AuditWorker" ],
                             [ "...", { "stat": "p99" } ],
-                            [ "funktor-testapp", "Duration", "WorkerClassName", "GreetingsWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "GreetingsWorker" ],
                             [ "...", { "stat": "p99" } ],
-                            [ "funktor-testapp", "Duration", "WorkerClassName", "HelloWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "HelloWorker" ],
                             [ "...", { "stat": "p99" } ],
-                            [ "funktor-testapp", "Duration", "WorkerClassName", "SingleThreadAuditWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "SingleThreadAuditWorker" ],
                             [ "...", { "stat": "p99" } ]
                         ],
                         "view": "timeSeries",
@@ -55,13 +55,13 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "funktor-testapp", "processed", "WorkerClassName", "AuditWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "AuditWorker" ],
                             [ ".", "failed", ".", "." ],
-                            [ "funktor-testapp", "processed", "WorkerClassName", "GreetingsWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "GreetingsWorker" ],
                             [ ".", "failed", ".", "." ],
-                            [ "funktor-testapp", "processed", "WorkerClassName", "HelloWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "HelloWorker" ],
                             [ ".", "failed", ".", "." ],
-                            [ "funktor-testapp", "processed", "WorkerClassName", "SingleThreadAuditWorker" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "SingleThreadAuditWorker" ],
                             [ ".", "failed", ".", "." ]
                         ],
                         "view": "timeSeries",
@@ -81,9 +81,9 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "funktor-testapp", "Duration", "Queue", "default" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "Queue", "default" ],
                             [ "...", { "stat": "p99" } ],
-                            [ "funktor-testapp", "Duration", "Queue", "low_concurrency" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "Queue", "low_concurrency" ],
                             [ "...", { "stat": "p99" } ]
                         ],
                         "view": "timeSeries",
@@ -102,9 +102,9 @@ Resources:
                     "type": "metric",
                     "properties": {
                         "metrics": [
-                            [ "funktor-testapp", "processed", "Queue", "default" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "Queue", "default" ],
                             [ ".", "failed", ".", "." ],
-                            [ "funktor-testapp", "processed", "Queue", "low_concurrency" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "Queue", "low_concurrency" ],
                             [ ".", "failed", ".", "." ]
                         ],
                         "view": "timeSeries",
@@ -117,17 +117,17 @@ Resources:
                 },
 
 
+                
                 { 
                     "height": 3,
                     "width": 24,
-                    "y": 9,
+                    "y": 12,
                     "x": 0,
                     "type": "text",
                     "properties": {
                         "markdown": "\n# Behind the scenes\n\n The stats below give some insight into the inner workings of the Funktor apparatus."
                     }
                 },
-                
 
 
                 

data/funktor-testapp/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/funktor-testapp/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: ruby2.7
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -42,9 +52,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
@@ -62,4 +72,4 @@ resources:
   - ${file(funktor_config/resources/jobs_table.yml)}
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/activity_tracker.rb

@@ -77,13 +77,17 @@ module Funktor
       @dynamodb_client ||= ::Aws::DynamoDB::Client.new
     end
 
+    def metric_namespace
+      [ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
+    end
+
     def put_metric_to_stdout(key, value)
       data = {
         "_aws": {
           "Timestamp": Time.now.strftime('%s%3N').to_i,
           "CloudWatchMetrics": [
             {
-              "Namespace": "rails-lambda-experiment",
+              "Namespace": metric_namespace,
               "Dimensions": [["functionVersion"]],
               "Metrics": [ # CPU, Memory, Duration, etc...
                            {

data/lib/funktor/cli/init.rb

@@ -46,8 +46,10 @@ module Funktor
             say "serverless-ruby-layer is already installed in package.json"
           else
             if File.exist?("package-lock.json")
-              run "npm install serverless-ruby-layer@1.4.0"
+              run "npm install serverless-ruby-layer@1.4.0 --save-dev"
               # TODO - Add handers for yarn and what not
+            elsif File.exist?("yarn.lock")
+              run "yarn add --dev serverless-ruby-layer@1.4.0"
             else
               say "You should install serverless-ruby-layer version 1.4.0 using yor package manager of choice."
             end
@@ -133,6 +135,10 @@ module Funktor
         template "serverless.yml", File.join("serverless.yml")
       end
 
+      def dockerfile
+        template "Dockerfile", File.join("Dockerfile")
+      end
+
       private
 
       def app_worker_names

data/lib/funktor/cli/templates/Dockerfile

@@ -0,0 +1,63 @@
+FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
+
+# Uncomment this as a cache buster
+#RUN echo date
+
+# If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
+#ARG BUNDLE_GEM__FURY__IO
+#ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
+
+# First we install some stuff that we need for gems that have to compile native extensions
+#RUN yum groupinstall "Development Tools" -y
+#RUN yum install -y amazon-linux-extras
+#RUN amazon-linux-extras enable postgresql11
+#RUN yum install -y postgresql-devel
+
+# Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
+COPY Gemfile Gemfile.lock .
+
+# Set a few bundle configuration options
+RUN bundle lock --add-platform x86_64-linux
+#RUN bundle config --local deployment true
+#RUN bundle config --local plugins false
+#RUN bundle config --local frozen true
+#RUN bundle config --local without 'development test'
+RUN bundle config --local path './vendor/bundle'
+
+# Now install our gems
+RUN bundle install --quiet --jobs 4
+
+# Now we start a second stage in the build that is a clean image without build tools
+FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
+
+#ENV RAILS_ENV=production
+#ENV RACK_ENV=production
+
+# Install node so that asset related gems have a JS runtime.
+# We ship the node executeable to production to make it easier to get an app deployed.
+# TODO: Document steps that could be taken to remove this dependency.
+#RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
+#RUN yum install -y nodejs
+
+# Then copy some postgres related files
+#COPY --from=build_image  /usr/lib64/libpq.so.5 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/liblber-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsasl2.so.3 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libssl3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsmime3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libnss3.so /usr/lib64/
+
+
+# Next we copy the app from our local directory and we copy
+# the bundled gems from the build image.
+# We do this after copying dependencies becuase the app will
+# change more frequently, and we can used caching up to here.
+
+COPY --from=build_image /var/task .
+COPY . .
+
+# And finally we have the CMD for the deployed container
+# You can overwrite command in `serverless.yml` template
+CMD [ "app.LambdaFunction::Handler.process" ]
+

data/lib/funktor/cli/templates/funktor_config/environment.yml

@@ -1,7 +1,16 @@
-BUNDLE_WITHOUT: development:test:assets
+BUNDLE_WITHOUT: development:test
 BUNDLE_PLUGINS: false
 BUNDLE_FROZEN: true
+BUNDLE_DEPLOYMENT: true
 SERVERLESS_STAGE: ${self:custom.stage}
+
+FUNKTOR_LOG_LEVEL: INFO
+
+RAILS_LOG_TO_STDOUT: true
+RAILS_ENV: production
+RACK_ENV: production
+RAILS_MAX_THREADS: 1
+
 FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/incoming_job_handler.call
+#handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/incoming_job_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml

@@ -1,7 +1,11 @@
-handler: lambda_event_handlers/job_activator.call
+#handler: lambda_event_handlers/job_activator.call
 timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/job_activator.call
 events:
   - schedule: rate(1 minute)

data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
+#handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/funktor.yml

@@ -81,6 +81,7 @@ IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
 IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
 IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
 DashboardName: ${self:service}-${self:custom.stage}-dashboard
+DashboardNamespace: ${self:service}-${self:custom.stage}
 <%- queue_names.each do |queue_name| -%>
 <%= queue_name.camelize %>QueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>
 <%= queue_name.camelize %>DeadJobQueueName: ${self:service}-${self:custom.stage}-<%= queue_name.underscore.dasherize %>-dead

data/lib/funktor/cli/templates/funktor_config/package.yml

@@ -1,10 +1,10 @@
 # TODO - Figure out how to allow individual packaging to work out of the box.
-individually: false
-include:
-  - Gemfile
-  - Gemfile.lock
-  - funktor_config/boot.rb
-  - app/**
+#individually: false
+#include:
+  #- Gemfile
+  #- Gemfile.lock
+  #- funktor_config/boot.rb
+  #- app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml

@@ -30,7 +30,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- app_worker_names.each do |worker_name| -%>
-                            [ "<%= app_name %>", "Duration", "WorkerClassName", "<%= worker_name %>" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ],
                             [ "...", { "stat": "p99" } ]<%= worker_name == app_worker_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -52,7 +52,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- app_worker_names.each do |worker_name| -%>
-                            [ "<%= app_name %>", "processed", "WorkerClassName", "<%= worker_name %>" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ],
                             [ ".", "failed", ".", "." ]<%= worker_name == app_worker_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -74,7 +74,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- queue_names.each do |queue_name| -%>
-                            [ "<%= app_name %>", "Duration", "Queue", "<%= queue_name.underscore %>" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "Queue", "<%= queue_name.underscore %>" ],
                             [ "...", { "stat": "p99" } ]<%= queue_name == queue_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -95,7 +95,7 @@ Resources:
                     "properties": {
                         "metrics": [
                             <%- queue_names.each do |queue_name| -%>
-                            [ "<%= app_name %>", "processed", "Queue", "<%= queue_name.underscore %>" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "Queue", "<%= queue_name.underscore %>" ],
                             [ ".", "failed", ".", "." ]<%= queue_name == queue_names.last ? "" : "," %>
                             <%- end -%>
                         ],
@@ -109,6 +109,7 @@ Resources:
                 },
 
 
+                <% current_y += 3 %>
                 { <% "Funktor Behind the Scenes Banner" %>
                     "height": 3,
                     "width": 24,
@@ -119,7 +120,6 @@ Resources:
                         "markdown": "\n# Behind the scenes\n\n The stats below give some insight into the inner workings of the Funktor apparatus."
                     }
                 },
-                <% current_y += 3 %>
 
 
                 <% current_y += 6 %>

data/lib/funktor/cli/templates/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/lib/funktor/cli/templates/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: <%= runtime %>
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -39,9 +49,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   <%- all_function_definitions.each do |function_definition| -%>
@@ -54,4 +64,4 @@ resources:
   <%- end -%>
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/counter.rb

@@ -17,13 +17,17 @@ module Funktor
       Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(job))
     end
 
+    def metric_namespace
+      [ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
+    end
+
     def metric_hash(job)
       {
         "_aws": {
           "Timestamp": Time.now.strftime('%s%3N').to_i,
           "CloudWatchMetrics": [
             {
-              "Namespace": ENV['FUNKTOR_APP_NAME'],
+              "Namespace": metric_namespace,
               "Dimensions": [["WorkerClassName"], ["Queue"]],
               "Metrics": [ # CPU, Memory, Duration, etc...
                            {

data/lib/funktor/incoming_job_handler.rb

@@ -30,11 +30,13 @@ module Funktor
         # TODO : This number should be configurable via ENV var
         if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
           Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
-          # We push to the jobs table first becauase the work queue handler will expect to be able
-          # to update the stats of a record that's already in the table.
-          # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
-          # work queue handler to be ok with a job that's not yet in the table?
-          push_to_jobs_table(job, "queued")
+          if Funktor.enable_work_queue_visibility
+            # We push to the jobs table first becauase the work queue handler will expect to be able
+            # to update the stats of a record that's already in the table.
+            # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
+            # work queue handler to be ok with a job that's not yet in the table?
+            push_to_jobs_table(job, "queued")
+          end
           push_to_work_queue(job)
           if job.is_retry?
             @tracker.track(:retryActivated, job)

data/lib/funktor/job_activator.rb

@@ -61,23 +61,40 @@ module Funktor
     end
 
     def activate_job(job_shard, job_id, current_category, queue_immediately = false)
+      
+      # TODO: WorkQueueVisibilityMiddleware to alter what happens here? Maybe we delete by default and then the middleware puts it back in the table?
       # First we conditionally update the item in  Dynamo to be sure that another scheduler hasn't gotten
       # to it, and if that works then send to SQS. This is basically how Sidekiq scheduler works.
-      response = dynamodb_client.update_item({
-        key: {
-          "jobShard" => job_shard,
-          "jobId" => job_id
-        },
-        update_expression: "SET category = :category, queueable = :queueable",
-        condition_expression: "category = :current_category",
-        expression_attribute_values: {
-          ":current_category" => current_category,
-          ":queueable" => "false",
-          ":category" => "queued"
-        },
-        table_name: delayed_job_table,
-        return_values: "ALL_OLD"
-      })
+      response = if Funktor.enable_work_queue_visibility
+                   dynamodb_client.update_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     update_expression: "SET category = :category, queueable = :queueable",
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category,
+                       ":queueable" => "false",
+                       ":category" => "queued"
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 else
+                   dynamodb_client.delete_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 end
       if response.attributes # this means the record was still there in the state we expected
         Funktor.logger.debug "response.attributes ====== "
         Funktor.logger.debug response.attributes

data/lib/funktor/middleware/metrics.rb

@@ -18,13 +18,17 @@ module Funktor
         Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(time_diff, job))
       end
 
+      def metric_namespace
+        [ENV['FUNKTOR_APP_NAME'], ENV['SERVERLESS_STAGE']].join('-')
+      end
+
       def metric_hash(time_diff_in_seconds, job)
         {
           "_aws": {
             "Timestamp": Time.now.strftime('%s%3N').to_i,
             "CloudWatchMetrics": [
               {
-                "Namespace": ENV['FUNKTOR_APP_NAME'],
+                "Namespace": metric_namespace,
                 "Dimensions": [["WorkerClassName"], ["Queue"]],
                 "Metrics": [ # CPU, Memory, Duration, etc...
                              {

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.7.3"
+  VERSION = "0.7.6"
 end

data/lib/funktor/work_queue_handler.rb

@@ -30,13 +30,18 @@ module Funktor
     def dispatch(job)
       begin
         @tracker.track(:processingStarted, job)
-        update_job_category(job, "processing")
+        if Funktor.enable_work_queue_visibility
+          update_job_category(job, "processing")
+        end
         Funktor.work_queue_handler_middleware.invoke(job) do
           job.execute
         end
         @processed_counter.incr(job)
         @tracker.track(:processingComplete, job)
-        delete_job_from_dynamodb(job)
+
+        if Funktor.enable_work_queue_visibility
+          delete_job_from_dynamodb(job)
+        end
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
         handle_error(e, job)
@@ -44,11 +49,17 @@ module Funktor
         job.error = e
         if job.can_retry
           @tracker.track(:retrying, job)
-          update_job_category(job, "retry")
+
+          if Funktor.enable_work_queue_visibility
+            update_job_category(job, "retry")
+          end
           trigger_retry(job)
         else
           @tracker.track(:bailingOut, job)
-          update_job_category(job, "dead")
+
+          if Funktor.enable_work_queue_visibility
+            update_job_category(job, "dead")
+          end
           Funktor.logger.error "We retried max times. We're bailing on this one."
           Funktor.logger.error job.to_json
         end
@@ -71,6 +82,7 @@ module Funktor
     end
 
     def update_job_category(job, category)
+      puts "starting update_job_category #{category}"
       dynamodb_client.update_item({
         key: {
           "jobShard" => job.shard,
@@ -84,9 +96,11 @@ module Funktor
         },
         return_values: "ALL_OLD"
       })
+      puts "ending update_job_category #{category}"
     end
 
     def delete_job_from_dynamodb(job)
+      puts "starting delete_job_from_dynamodb"
       dynamodb_client.delete_item({
         key: {
           "jobShard" => job.shard,
@@ -95,6 +109,7 @@ module Funktor
         table_name: delayed_job_table,
         return_values: "ALL_OLD"
       })
+      puts "ending delete_job_from_dynamodb"
     end
 
   end

data/lib/funktor.rb

@@ -18,7 +18,8 @@ module Funktor
 
   DEFAULT_OPTIONS = {
     error_handlers: [],
-    log_level: Logger::DEBUG # Set a high log level during early, active development
+    log_level: Logger::DEBUG, # Set a high log level during early, active development
+    enable_work_queue_visibility: true # Enable this by default during early, active development
   }
 
   def self.configure_job_pusher
@@ -110,6 +111,14 @@ module Funktor
     @logger = logger
   end
 
+  def self.enable_work_queue_visibility
+    options[:enable_work_queue_visibility]
+  end
+
+  def self.enable_work_queue_visibility= enabled
+    options[:enable_work_queue_visibility] = enabled
+  end
+
   # We have a raw_logger that doesn't add timestamps and what not. This is used to publish
   # CloudWatch metrics that can be used in dashboards.
   def self.raw_logger

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.6
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2022-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -176,6 +176,7 @@ files:
 - exe/funktor-deploy
 - funktor-testapp/.envrc
 - funktor-testapp/.gitignore
+- funktor-testapp/Dockerfile
 - funktor-testapp/Gemfile
 - funktor-testapp/Gemfile.lock
 - funktor-testapp/app/services/job_flood.rb
@@ -227,6 +228,7 @@ files:
 - lib/funktor/cli/generate/base.rb
 - lib/funktor/cli/generate/work_queue.rb
 - lib/funktor/cli/init.rb
+- lib/funktor/cli/templates/Dockerfile
 - lib/funktor/cli/templates/Gemfile
 - lib/funktor/cli/templates/app/workers/hello_worker.rb
 - lib/funktor/cli/templates/funktor_config/boot.rb