funktor 0.7.5 → 0.7.8

data/funktor-testapp/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/funktor-testapp/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: ruby2.7
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -42,9 +52,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
@@ -62,4 +72,4 @@ resources:
   - ${file(funktor_config/resources/jobs_table.yml)}
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/activity_tracker.rb

@@ -1,5 +1,4 @@
 require 'json'
-require 'aws-sdk-dynamodb'
 
 module Funktor
   class ActivityTracker
@@ -74,7 +73,7 @@ module Funktor
     end
 
     def dynamodb_client
-      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      Funktor.dynamodb_client
     end
 
     def metric_namespace

data/lib/funktor/cli/init.rb

@@ -135,6 +135,10 @@ module Funktor
         template "serverless.yml", File.join("serverless.yml")
       end
 
+      def dockerfile
+        template "Dockerfile", File.join("Dockerfile")
+      end
+
       private
 
       def app_worker_names

data/lib/funktor/cli/templates/Dockerfile

@@ -0,0 +1,63 @@
+FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
+
+# Uncomment this as a cache buster
+#RUN echo date
+
+# If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
+#ARG BUNDLE_GEM__FURY__IO
+#ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
+
+# First we install some stuff that we need for gems that have to compile native extensions
+#RUN yum groupinstall "Development Tools" -y
+#RUN yum install -y amazon-linux-extras
+#RUN amazon-linux-extras enable postgresql11
+#RUN yum install -y postgresql-devel
+
+# Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
+COPY Gemfile Gemfile.lock .
+
+# Set a few bundle configuration options
+RUN bundle lock --add-platform x86_64-linux
+#RUN bundle config --local deployment true
+#RUN bundle config --local plugins false
+#RUN bundle config --local frozen true
+#RUN bundle config --local without 'development test'
+RUN bundle config --local path './vendor/bundle'
+
+# Now install our gems
+RUN bundle install --quiet --jobs 4
+
+# Now we start a second stage in the build that is a clean image without build tools
+FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
+
+#ENV RAILS_ENV=production
+#ENV RACK_ENV=production
+
+# Install node so that asset related gems have a JS runtime.
+# We ship the node executeable to production to make it easier to get an app deployed.
+# TODO: Document steps that could be taken to remove this dependency.
+#RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
+#RUN yum install -y nodejs
+
+# Then copy some postgres related files
+#COPY --from=build_image  /usr/lib64/libpq.so.5 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/liblber-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsasl2.so.3 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libssl3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsmime3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libnss3.so /usr/lib64/
+
+
+# Next we copy the app from our local directory and we copy
+# the bundled gems from the build image.
+# We do this after copying dependencies becuase the app will
+# change more frequently, and we can used caching up to here.
+
+COPY --from=build_image /var/task .
+COPY . .
+
+# And finally we have the CMD for the deployed container
+# You can overwrite command in `serverless.yml` template
+CMD [ "app.LambdaFunction::Handler.process" ]
+

data/lib/funktor/cli/templates/funktor_config/environment.yml

@@ -1,7 +1,16 @@
-BUNDLE_WITHOUT: development:test:assets
+BUNDLE_WITHOUT: development:test
 BUNDLE_PLUGINS: false
 BUNDLE_FROZEN: true
+BUNDLE_DEPLOYMENT: true
 SERVERLESS_STAGE: ${self:custom.stage}
+
+FUNKTOR_LOG_LEVEL: INFO
+
+RAILS_LOG_TO_STDOUT: true
+RAILS_ENV: production
+RACK_ENV: production
+RAILS_MAX_THREADS: 1
+
 FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/incoming_job_handler.call
+#handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/incoming_job_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml

@@ -1,7 +1,11 @@
-handler: lambda_event_handlers/job_activator.call
+#handler: lambda_event_handlers/job_activator.call
 timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/job_activator.call
 events:
   - schedule: rate(1 minute)

data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
+#handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/package.yml

@@ -1,10 +1,10 @@
 # TODO - Figure out how to allow individual packaging to work out of the box.
-individually: false
-include:
-  - Gemfile
-  - Gemfile.lock
-  - funktor_config/boot.rb
-  - app/**
+#individually: false
+#include:
+  #- Gemfile
+  #- Gemfile.lock
+  #- funktor_config/boot.rb
+  #- app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml

@@ -21,6 +21,160 @@ Resources:
                 },
                 <% current_y += 3 %>
 
+
+              <%- app_worker_names.each do |worker_name| -%>
+                { <% "Worker" %>
+                    "height": 3,
+                    "width": 6,
+                    "y": <%= current_y %>,
+                    "x": 0,
+                    "type": "text",
+                    "properties": {
+                        "markdown": "\n# <%= worker_name %>\n"
+                    }
+                },
+                { <% "Worker Jobs per minute" %>
+                    "height": 3,
+                    "width": 6,
+                    "y": <%= current_y %>,
+                    "x": 6,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                            [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ],
+                            [ "${self:custom.funktor.DashboardNamespace}", "failed", "WorkerClassName", "<%= worker_name %>", { "color": "#d62728" } ]
+                        ],
+                        "view": "singleValue",
+                        "region": "us-east-1",
+                        "stat": "Sum",
+                        "period": 60,
+                        "sparkline": true,
+                        "title": "<%= worker_name %> Jobs Per Minute"
+                    }
+                },
+                { <% "Worker Durations" %>
+                    "height": 3,
+                    "width": 12,
+                    "y": <%= current_y %>,
+                    "x": 12,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "p10" } ],
+                            [ "...", { "label": "p50", "stat": "p50" } ],
+                            [ "...", { "label": "p99", "stat": "p99" } ],
+                            [ "...", { "label": "Average", "stat": "Average" } ]
+                        ],
+                        "view": "singleValue",
+                        "region": "us-east-1",
+                        "stat": "p10",
+                        "period": 60,
+                        "sparkline": true,
+                        "title": "<%= worker_name %> Duration"
+                    }
+                },
+                <% current_y += 3 %>
+
+
+
+
+                { <% "Processed Jobs By Worker" %>
+                    "height": 3,
+                    "width": 6,
+                    "y": <%= current_y %>,
+                    "x": 0,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                          [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ]
+                        ],
+                        "view": "timeSeries",
+                        "stacked": false,
+                        "region": "us-east-1",
+                        "title": "<%= worker_name %> Processed Jobs",
+                        "period": 60,
+                        "stat": "Sum"
+                  }
+                },
+                { <% "Failed Jobs By Worker" %>
+                    "height": 3,
+                    "width": 6,
+                    "y": <%= current_y + 3 %>,
+                    "x": 0,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                          [ "${self:custom.funktor.DashboardNamespace}", "failed", "WorkerClassName", "<%= worker_name %>", { "color": "#d62728" }]
+                        ],
+                        "view": "timeSeries",
+                        "stacked": false,
+                        "region": "us-east-1",
+                        "title": "<%= worker_name %> Failed Jobs",
+                        "period": 60,
+                        "stat": "Sum"
+                  }
+                },
+                { <% "Job Duration By Worker" %>
+                    "height": 6,
+                    "width": 12,
+                    "y": <%= current_y %>,
+                    "x": 6,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ],
+                            [ "...", { "stat": "p80" } ],
+                            [ "...", { "stat": "p70" } ],
+                            [ "...", { "stat": "p60" } ]
+                        ],
+                        "view": "timeSeries",
+                        "stacked": false,
+                        "region": "us-east-1",
+                        "stat": "p90",
+                        "period": 60,
+                        "title": "<%= worker_name %> Duration Percentiles"
+                    }
+                },
+                { <% "Maximum By Worker" %>
+                    "height": 3,
+                    "width": 6,
+                    "y": <%= current_y %>,
+                    "x": 18,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ]
+                        ],
+                        "view": "timeSeries",
+                        "stacked": false,
+                        "region": "us-east-1",
+                        "stat": "Maximum",
+                        "period": 60,
+                        "title": "<%= worker_name %> Max"
+                    }
+                },
+                { <% "Minimum By Worker" %>
+                    "height": 3,
+                    "width": 6,
+                    "y": <%= current_y %>,
+                    "x": 18,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                            [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ]
+                        ],
+                        "view": "timeSeries",
+                        "stacked": false,
+                        "region": "us-east-1",
+                        "stat": "Minimum",
+                        "period": 60,
+                        "title": "<%= worker_name %> Min"
+                    }
+                },
+                <% current_y += 6 %>
+              <%- end -%>
+
+
               { <% "Job Duration By Worker" %>
                     "height": 6,
                     "width": 12,
@@ -65,6 +219,7 @@ Resources:
                   }
                 },
                 <% current_y += 6 %>
+
                 { <% "Job Duration By Queue" %>
                     "height": 6,
                     "width": 12,
@@ -147,6 +302,7 @@ Resources:
                         "region": "us-east-1",
                         "stat": "Sum",
                         "period": 60,
+                        "sparkline": true,
                         "title": "Messages Per Minute"
                     }
                 },
@@ -167,6 +323,7 @@ Resources:
                         "region": "us-east-1",
                         "stat": "p10",
                         "period": 60,
+                        "sparkline": true,
                         "title": "Handler Duration"
                     }
                 },
@@ -294,6 +451,7 @@ Resources:
                         "region": "us-east-1",
                         "stat": "Sum",
                         "period": 60,
+                        "sparkline": true,
                         "title": "Messages Per Minute"
                     }
                 },
@@ -316,6 +474,7 @@ Resources:
                         "region": "us-east-1",
                         "stat": "p10",
                         "period": 60,
+                        "sparkline": true,
                         "title": "Handler Duration"
                     }
                 },
@@ -444,6 +603,7 @@ Resources:
                         "region": "us-east-1",
                         "stat": "Average",
                         "period": 60,
+                        "sparkline": true,
                         "title": "Messages To Be Scheduled"
                     }
                 },
@@ -464,6 +624,7 @@ Resources:
                         "region": "us-east-1",
                         "stat": "p10",
                         "period": 60,
+                        "sparkline": true,
                         "title": "Handler Duration"
                     }
                 },

data/lib/funktor/cli/templates/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/lib/funktor/cli/templates/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: <%= runtime %>
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -39,9 +49,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   <%- all_function_definitions.each do |function_definition| -%>
@@ -54,4 +64,4 @@ resources:
   <%- end -%>
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/incoming_job_handler.rb

@@ -18,11 +18,11 @@ module Funktor
     end
 
     def sqs_client
-      @sqs_client ||= ::Aws::SQS::Client.new
+      Funktor.sqs_client
     end
 
     def dynamodb_client
-      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      Funktor.dynamodb_client
     end
 
     def dispatch(job)
@@ -30,11 +30,13 @@ module Funktor
         # TODO : This number should be configurable via ENV var
         if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
           Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
-          # We push to the jobs table first becauase the work queue handler will expect to be able
-          # to update the stats of a record that's already in the table.
-          # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
-          # work queue handler to be ok with a job that's not yet in the table?
-          push_to_jobs_table(job, "queued")
+          if Funktor.enable_work_queue_visibility
+            # We push to the jobs table first becauase the work queue handler will expect to be able
+            # to update the stats of a record that's already in the table.
+            # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
+            # work queue handler to be ok with a job that's not yet in the table?
+            push_to_jobs_table(job, "queued")
+          end
           push_to_work_queue(job)
           if job.is_retry?
             @tracker.track(:retryActivated, job)

data/lib/funktor/job_activator.rb

@@ -9,11 +9,11 @@ module Funktor
     end
 
     def dynamodb_client
-      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      Funktor.dynamodb_client
     end
 
     def sqs_client
-      @sqs_client ||= ::Aws::SQS::Client.new
+      Funktor.sqs_client
     end
 
     def delayed_job_table
@@ -61,23 +61,39 @@ module Funktor
     end
 
     def activate_job(job_shard, job_id, current_category, queue_immediately = false)
+      # TODO: WorkQueueVisibilityMiddleware to alter what happens here? Maybe we delete by default and then the middleware puts it back in the table?
       # First we conditionally update the item in  Dynamo to be sure that another scheduler hasn't gotten
       # to it, and if that works then send to SQS. This is basically how Sidekiq scheduler works.
-      response = dynamodb_client.update_item({
-        key: {
-          "jobShard" => job_shard,
-          "jobId" => job_id
-        },
-        update_expression: "SET category = :category, queueable = :queueable",
-        condition_expression: "category = :current_category",
-        expression_attribute_values: {
-          ":current_category" => current_category,
-          ":queueable" => "false",
-          ":category" => "queued"
-        },
-        table_name: delayed_job_table,
-        return_values: "ALL_OLD"
-      })
+      response = if Funktor.enable_work_queue_visibility
+                   dynamodb_client.update_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     update_expression: "SET category = :category, queueable = :queueable",
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category,
+                       ":queueable" => "false",
+                       ":category" => "queued"
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 else
+                   dynamodb_client.delete_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 end
       if response.attributes # this means the record was still there in the state we expected
         Funktor.logger.debug "response.attributes ====== "
         Funktor.logger.debug response.attributes

data/lib/funktor/job_pusher.rb

@@ -19,7 +19,7 @@ module Funktor
     private
 
     def client
-      @client ||= ::Aws::SQS::Client.new
+      Funktor.sqs_client
     end
 
     def queue_url(payload)

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.7.5"
+  VERSION = "0.7.8"
 end

data/lib/funktor/web/application.rb

@@ -128,7 +128,7 @@ module Funktor
       end
 
       def dynamodb_client
-        @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+        Funktor.dynamodb_client
       end
 
       # start the server if ruby file executed directly