funktor 0.7.24 → 0.7.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fdcd3e939904ca9cd77b31c29842e3a43c5cbf53113180c8fbb32fc3fe7c5e2
-  data.tar.gz: 3ed291a899ad41382275e2f0a1cb4da28d688a5b262831a4eed96e5487024200
+  metadata.gz: ef7feda36c3d7a6609b907b94632602d744b98a2f685853eb4686f77aa62330b
+  data.tar.gz: 6ceaf132ea54f0e97760a05660208e262d7ed9c1d0473522100b5bf5c3d39161
 SHA512:
-  metadata.gz: 6c2c23526960146d1d7d0e895484dba8c1403bc1dafbbe88d1fa4309765c8176294c126447b5ca4806f59f74d03ae504a710121d97301a1759234b133e6a15ce
-  data.tar.gz: 15ad50995710277c1bf8105d53b38c6f3746ae1139613dd2f4e4630c5539204452d8675f815d50bdb53046ccae13452e939909ff081d481f435eac67318b97f5
+  metadata.gz: 5ee21aa7210894ad2a05554d73eaa0958100ac87956e8e70e85146e0e04e419e402afc905f9eb20ba488537be053882739deb8595842cb50953594617e5f2dfc
+  data.tar.gz: 9dbbed69db716f99cf8bd16db47938a45036b16d88bd5c576f862c3b2dd9457a589b15697829f95ed669d801d5e58bac856a4406c980d94157fd8a627bd88c2e

data/Gemfile

@@ -5,3 +5,5 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+
+gem 'newrelic_rpm'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.7.24)
+    funktor (0.7.27)
       activesupport
       aws-sdk-dynamodb (~> 1.62)
       aws-sdk-sqs (~> 1.37)
@@ -22,7 +22,7 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.2.0)
-    aws-partitions (1.595.0)
+    aws-partitions (1.597.0)
     aws-sdk-core (3.131.1)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.525.0)
@@ -53,6 +53,7 @@ GEM
     minitest (5.14.4)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
+    newrelic_rpm (8.8.0)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -107,6 +108,7 @@ PLATFORMS
 DEPENDENCIES
   activejob (>= 5.1.5)
   funktor!
+  newrelic_rpm
   pry-byebug
   rake (~> 12.0)
   rspec (~> 3.0)

data/funktor-testapp/Dockerfile

@@ -8,7 +8,7 @@ FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
 #ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
 
 # First we install some stuff that we need for gems that have to compile native extensions
-#RUN yum groupinstall "Development Tools" -y
+RUN yum groupinstall "Development Tools" -y
 #RUN yum install -y amazon-linux-extras
 #RUN amazon-linux-extras enable postgresql11
 #RUN yum install -y postgresql-devel
@@ -57,6 +57,12 @@ FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
 COPY --from=build_image /var/task .
 COPY . .
 
+# Now we prime the bootsnap cache to minimize cold start times.
+
+ENV BOOTSNAP_CACHE_DIR=/var/task/tmp/cache
+RUN bundle exec bootsnap precompile --gemfile . \
+ && SKIP_ENV=true bundle exec ruby lambda_event_handlers/incoming_job_handler.rb
+
 # And finally we have the CMD for the deployed container
 # You can overwrite command in `serverless.yml` template
 CMD [ "app.LambdaFunction::Handler.process" ]

data/funktor-testapp/Gemfile

@@ -23,3 +23,5 @@ gem 'aws-sdk-dynamodb', '~> 1.62'
 gem "activesupport"
 gem "thor" # Thor drives the CLI TODO - Maybe we don't need it here?
 
+# This helps make cold starts faster
+gem 'bootsnap', require: false

data/funktor-testapp/Gemfile.lock

@@ -22,11 +22,14 @@ GEM
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.4)
       aws-eventstream (~> 1, >= 1.0.2)
+    bootsnap (1.12.0)
+      msgpack (~> 1.2)
     concurrent-ruby (1.1.9)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
     minitest (5.14.4)
+    msgpack (1.5.2)
     require_all (1.5.0)
     thor (1.1.0)
     tzinfo (2.0.4)
@@ -41,6 +44,7 @@ DEPENDENCIES
   activesupport
   aws-sdk-dynamodb (~> 1.62)
   aws-sdk-sqs (~> 1.37)
+  bootsnap
   require_all (~> 1.0)
   thor
 

data/funktor-testapp/funktor_config/boot.rb

@@ -5,6 +5,9 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 require "rubygems"
 require 'bundler/setup'
 
+# load bootsnap for faster cold starts
+require 'bootsnap/setup'
+
 # Set up gems listed in the Gemfile.
 Bundler.require(:default, :production)
 

data/funktor-testapp/serverless.yml

@@ -52,9 +52,6 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
-
-#package: ${file(funktor_config/package.yml)}
 
 functions:
   IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
@@ -72,4 +69,3 @@ resources:
   - ${file(funktor_config/resources/jobs_table.yml)}
 
 plugins:
-  #- serverless-ruby-layer

data/lib/funktor/cli/init.rb

@@ -33,27 +33,13 @@ module Funktor
       def funktor_config_yml
         #template "funktor_config.yml", File.join("funktor_config.yml")
         template File.join("funktor_config", "funktor.yml"), File.join("funktor_config", "funktor.yml")
-        template File.join("funktor_config", "ruby_layer.yml"), File.join("funktor_config", "ruby_layer.yml")
-        template File.join("funktor_config", "package.yml"), File.join("funktor_config", "package.yml")
         template File.join("funktor_config", "environment.yml"), File.join("funktor_config", "environment.yml")
         template File.join("funktor_config", "boot.rb"), File.join("funktor_config", "boot.rb")
       end
 
       def package_json
         if File.exist?("package.json")
-          package_data = File.open("package.json").read
-          if package_data =~ /serverless-ruby-layer/
-            say "serverless-ruby-layer is already installed in package.json"
-          else
-            if File.exist?("package-lock.json")
-              run "npm install serverless-ruby-layer@1.4.0 --save-dev"
-              # TODO - Add handers for yarn and what not
-            elsif File.exist?("yarn.lock")
-              run "yarn add --dev serverless-ruby-layer@1.4.0"
-            else
-              say "You should install serverless-ruby-layer version 1.4.0 using yor package manager of choice."
-            end
-          end
+          say "package.json already exists"
         else
           template "package.json", File.join("package.json")
         end

data/lib/funktor/cli/templates/Dockerfile

@@ -8,7 +8,7 @@ FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
 #ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
 
 # First we install some stuff that we need for gems that have to compile native extensions
-#RUN yum groupinstall "Development Tools" -y
+RUN yum groupinstall "Development Tools" -y
 #RUN yum install -y amazon-linux-extras
 #RUN amazon-linux-extras enable postgresql11
 #RUN yum install -y postgresql-devel
@@ -57,6 +57,12 @@ FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
 COPY --from=build_image /var/task .
 COPY . .
 
+# Now we prime the bootsnap cache to minimize cold start times.
+
+ENV BOOTSNAP_CACHE_DIR=/var/task/tmp/cache
+RUN bundle exec bootsnap precompile --gemfile . \
+ && SKIP_ENV=true bundle exec ruby lambda_event_handlers/incoming_job_handler.rb
+
 # And finally we have the CMD for the deployed container
 # You can overwrite command in `serverless.yml` template
 CMD [ "app.LambdaFunction::Handler.process" ]

data/lib/funktor/cli/templates/Gemfile

@@ -4,4 +4,4 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby '~> 2.7'
 
 gem 'funktor'
-
+gem 'bootsnap', require: false

data/lib/funktor/cli/templates/funktor_config/boot.rb

@@ -5,6 +5,9 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 require "rubygems"
 require 'bundler/setup'
 
+# load bootsnap for faster cold starts
+require 'bootsnap/setup'
+
 # Set up gems listed in the Gemfile.
 Bundler.require(:default, :production)
 

data/lib/funktor/cli/templates/funktor_init.yml.tt

@@ -38,12 +38,12 @@ handlerDefaults:
   # Visibility timeout should only come into play in the case of Funktor errors.
   # Application level errors should be handled by Funktor retry mechanisms.
   # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
-  visibilityTimeout: 1800
+  visibilityTimeout: 300
 
   # Max recieve count affects how many times a job will retry that has been throttled at the SQS -> Lambda boundary.
   # Amazon recommend this be at least 5, but I've found that higher numbers are better to avoid legit jobs ending
   # up in the dead letter queue
-  maxReceiveCount: 20
+  maxReceiveCount: 120
 
 # Incoming Job Handler
 incomingJobHandler:

data/lib/funktor/cli/templates/package.json

@@ -3,6 +3,5 @@
   "description": "",
   "version": "0.1.0",
   "devDependencies": {
-    "serverless-ruby-layer": "^1.4.0"
   }
 }

data/lib/funktor/cli/templates/serverless.yml

@@ -49,9 +49,6 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
-
-#package: ${file(funktor_config/package.yml)}
 
 functions:
   <%- all_function_definitions.each do |function_definition| -%>
@@ -64,4 +61,3 @@ resources:
   <%- end -%>
 
 plugins:
-  #- serverless-ruby-layer

data/lib/funktor/job.rb

@@ -114,8 +114,16 @@ module Funktor
       self.delay = seconds_to_delay(retries)
     end
 
-    # delayed_job and sidekiq use the same basic formula
     def seconds_to_delay(count)
+      if worker_class&.funktor_retry_in_block
+        worker_class.funktor_retry_in_block.call(count)
+      else
+        default_seconds_to_delay(count)
+      end
+    end
+
+    # delayed_job and sidekiq use the same basic formula
+    def default_seconds_to_delay(count)
       (count**4) + 15 + (rand(30) * (count + 1))
     end
 
@@ -124,7 +132,7 @@ module Funktor
     end
 
     def retry_limit
-      25
+      worker_class&.custom_retry_limit || 25
     end
 
     def can_retry

data/lib/funktor/middleware/new_relic.rb

@@ -0,0 +1,34 @@
+require 'newrelic_rpm'
+module Funktor
+  module Middleware
+    class NewRelic
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+      def call(job)
+        trace_args = {
+          :name => 'perform',
+          :class_name => job.worker_class_name_for_metrics,
+          :category => 'OtherTransaction/Funktor'
+        }
+        perform_action_with_newrelic_trace(trace_args) do
+          ::NewRelic::Agent::Transaction.merge_untrusted_agent_attributes(job.worker_params,
+                                                                        :'worker.funktor.params',
+                                                                        ::NewRelic::Agent::AttributeFilter::DST_NONE)
+          yield
+        end
+      end
+    end
+  end
+
+  def new_relic!
+    Funktor.configure_work_queue_handler do |config|
+      config.work_queue_handler_middleware do |chain|
+        chain.add Funktor::Middleware::NewRelic
+      end
+    end
+  end
+end
+
+
+
+

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.7.24"
+  VERSION = "0.7.27"
 end

data/lib/funktor/worker/funktor_options.rb

@@ -7,6 +7,7 @@ module Funktor
         base.extend ClassMethods
         base.class_eval do
           class_attribute :funktor_options_hash
+          class_attribute :funktor_retry_in_block
         end
       end
       module ClassMethods
@@ -18,6 +19,10 @@ module Funktor
           self.funktor_options_hash || {}
         end
 
+        def funktor_retry_in(&block)
+          self.funktor_retry_in_block = block
+        end
+
         def custom_queue_url
           get_funktor_options[:queue_url]
         end
@@ -26,6 +31,16 @@ module Funktor
           get_funktor_options[:queue]
         end
 
+        def custom_retry_limit
+          retry_limit = get_funktor_options[:retry]
+          if retry_limit.nil?
+            retry_limit = 25
+          elsif !retry_limit # if someone did "retry: false"
+            retry_limit = 0
+          end
+          return retry_limit
+        end
+
         def queue_url
           custom_queue_url
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.7.24
+  version: 0.7.27
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-06-03 00:00:00.000000000 Z
+date: 2022-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -199,14 +199,12 @@ files:
 - funktor-testapp/funktor_config/iam_permissions/jobs_table_secondary_index.yml
 - funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
 - funktor-testapp/funktor_config/iam_permissions/ssm.yml
-- funktor-testapp/funktor_config/package.yml
 - funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml
 - funktor-testapp/funktor_config/resources/default_queue.yml
 - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
 - funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml
 - funktor-testapp/funktor_config/resources/jobs_table.yml
 - funktor-testapp/funktor_config/resources/low_concurrency_queue.yml
-- funktor-testapp/funktor_config/ruby_layer.yml
 - funktor-testapp/funktor_init.yml
 - funktor-testapp/lambda_event_handlers/default_queue_handler.rb
 - funktor-testapp/lambda_event_handlers/incoming_job_handler.rb
@@ -242,13 +240,11 @@ files:
 - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
-- lib/funktor/cli/templates/funktor_config/package.yml
 - lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
 - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
 - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
 - lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml
 - lib/funktor/cli/templates/funktor_config/resources/work_queue.yml
-- lib/funktor/cli/templates/funktor_config/ruby_layer.yml
 - lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
 - lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
@@ -265,6 +261,7 @@ files:
 - lib/funktor/job_pusher.rb
 - lib/funktor/logger.rb
 - lib/funktor/middleware/metrics.rb
+- lib/funktor/middleware/new_relic.rb
 - lib/funktor/middleware_chain.rb
 - lib/funktor/rails.rb
 - lib/funktor/shard_utils.rb

data/funktor-testapp/funktor_config/package.yml

@@ -1,11 +0,0 @@
-# TODO - Figure out how to allow individual packaging to work out of the box.
-#individually: false
-#include:
-  #- Gemfile
-  #- Gemfile.lock
-  #- funktor_config/boot.rb
-  #- app/**
-# Evertyting is excluded by default with serverless-ruby-layer, but you could use
-# the lines below to exlude files that are inside an include path.
-#exclude:
-#  - workers/excluded_worker.rb

data/funktor-testapp/funktor_config/ruby_layer.yml

@@ -1,11 +0,0 @@
-#use_docker: false
-#docker_yums:
-  #- postgresql-devel
-#native_libs:
-  #- /usr/lib64/libpq.so.5
-  #- /usr/lib64/libldap_r-2.4.so.2
-  #- /usr/lib64/liblber-2.4.so.2
-  #- /usr/lib64/libsasl2.so.3
-  #- /usr/lib64/libssl3.so
-  #- /usr/lib64/libsmime3.so
-  #- /usr/lib64/libnss3.so

data/lib/funktor/cli/templates/funktor_config/package.yml

@@ -1,11 +0,0 @@
-# TODO - Figure out how to allow individual packaging to work out of the box.
-#individually: false
-#include:
-  #- Gemfile
-  #- Gemfile.lock
-  #- funktor_config/boot.rb
-  #- app/**
-# Evertyting is excluded by default with serverless-ruby-layer, but you could use
-# the lines below to exlude files that are inside an include path.
-#exclude:
-#  - workers/excluded_worker.rb

data/lib/funktor/cli/templates/funktor_config/ruby_layer.yml

@@ -1,11 +0,0 @@
-#use_docker: false
-#docker_yums:
-  #- postgresql-devel
-#native_libs:
-  #- /usr/lib64/libpq.so.5
-  #- /usr/lib64/libldap_r-2.4.so.2
-  #- /usr/lib64/liblber-2.4.so.2
-  #- /usr/lib64/libsasl2.so.3
-  #- /usr/lib64/libssl3.so
-  #- /usr/lib64/libsmime3.so
-  #- /usr/lib64/libnss3.so