funktor 0.7.5 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b78b0dfad4344af7ceaeef40b2871f68e1305deaa0218f87737b05b4d2e36578
-  data.tar.gz: 3405468ffbcf7c826ff25dce2a4a991df9c6ebe1076cfbcb357ce1b94174355d
+  metadata.gz: 00c5852fd10c7e6ad87b576ff0a1ed13b62f69fc19391202d4c2d289376a3fb2
+  data.tar.gz: e780ae7466a2c4dfc7060abfacd230a9284105c89aa95db376e99b7fe14a2b89
 SHA512:
-  metadata.gz: 556517fc7fe7ee77c5cb38b4a97c64333cc929aa7524cd4263ae5f9087e847ba5958def975a4b7401f95aa77bd9b276cc700440e591db769da49e41c22bb9e3f
-  data.tar.gz: 524a7ce3720d0f6d3f368ebc1bec6035568942e2eaaec44c001e3b4acfc7397797b08c6a8c528cd7730ea42007e9b7e1c0b2e55a6c83460a52d75a4a991faf53
+  metadata.gz: e7015cc189689fbffb456727899417e1edb7f823436536b6a3f316a96a83291bdaf0ee43b860d2925b53c239be15d685df268bf277c0843efc853b2b07abe0d6
+  data.tar.gz: a8d58b3d8d45b5bd2ecd70c1f18956685af350959cd6d3263e212d611e9e4d7d069b52681866e725cc432af09a434465e0b9bff80dc94d7a444f88d21a87a36d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.7.4)
+    funktor (0.7.5)
       activesupport
       aws-sdk-dynamodb (~> 1.62)
       aws-sdk-sqs (~> 1.37)

data/funktor-testapp/Dockerfile

@@ -0,0 +1,63 @@
+FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
+
+# Uncomment this as a cache buster
+#RUN echo date
+
+# If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
+#ARG BUNDLE_GEM__FURY__IO
+#ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
+
+# First we install some stuff that we need for gems that have to compile native extensions
+#RUN yum groupinstall "Development Tools" -y
+#RUN yum install -y amazon-linux-extras
+#RUN amazon-linux-extras enable postgresql11
+#RUN yum install -y postgresql-devel
+
+# Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
+COPY Gemfile Gemfile.lock .
+
+# Set a few bundle configuration options
+RUN bundle lock --add-platform x86_64-linux
+#RUN bundle config --local deployment true
+#RUN bundle config --local plugins false
+#RUN bundle config --local frozen true
+#RUN bundle config --local without 'development test'
+RUN bundle config --local path './vendor/bundle'
+
+# Now install our gems
+RUN bundle install --quiet --jobs 4
+
+# Now we start a second stage in the build that is a clean image without build tools
+FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
+
+#ENV RAILS_ENV=production
+#ENV RACK_ENV=production
+
+# Install node so that asset related gems have a JS runtime.
+# We ship the node executeable to production to make it easier to get an app deployed.
+# TODO: Document steps that could be taken to remove this dependency.
+#RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
+#RUN yum install -y nodejs
+
+# Then copy some postgres related files
+#COPY --from=build_image  /usr/lib64/libpq.so.5 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/liblber-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsasl2.so.3 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libssl3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsmime3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libnss3.so /usr/lib64/
+
+
+# Next we copy the app from our local directory and we copy
+# the bundled gems from the build image.
+# We do this after copying dependencies becuase the app will
+# change more frequently, and we can used caching up to here.
+
+COPY --from=build_image /var/task .
+COPY . .
+
+# And finally we have the CMD for the deployed container
+# You can overwrite command in `serverless.yml` template
+CMD [ "app.LambdaFunction::Handler.process" ]
+

data/funktor-testapp/funktor_config/boot.rb

@@ -12,6 +12,9 @@ Bundler.require(:default, :production)
 $LOAD_PATH.unshift 'funktor/lib'
 require 'funktor'
 
+
+Funktor.enable_work_queue_visibility = false
+
 # Load all ruby files in the app directory
 require_rel File.join('..', 'app')
 

data/funktor-testapp/funktor_config/environment.yml

@@ -1,7 +1,16 @@
-BUNDLE_WITHOUT: development:test:assets
+BUNDLE_WITHOUT: development:test
 BUNDLE_PLUGINS: false
 BUNDLE_FROZEN: true
+BUNDLE_DEPLOYMENT: true
 SERVERLESS_STAGE: ${self:custom.stage}
+
+FUNKTOR_LOG_LEVEL: INFO
+
+RAILS_LOG_TO_STDOUT: true
+RAILS_ENV: production
+RACK_ENV: production
+RAILS_MAX_THREADS: 1
+
 FUNKTOR_APP_NAME: funktor-testapp
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/default_queue_handler.call
+#handler: lambda_event_handlers/default_queue_handler.call
 timeout: ${self:custom.funktor.DefaultQueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.DefaultQueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.DefaultQueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.DefaultQueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/default_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.DefaultQueueHandler.batchSize, 1}

data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/incoming_job_handler.call
+#handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/incoming_job_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}

data/funktor-testapp/funktor_config/function_definitions/job_activator.yml

@@ -1,7 +1,11 @@
-handler: lambda_event_handlers/job_activator.call
+#handler: lambda_event_handlers/job_activator.call
 timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/job_activator.call
 events:
   - schedule: rate(1 minute)

data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/low_concurrency_queue_handler.call
+#handler: lambda_event_handlers/low_concurrency_queue_handler.call
 timeout: ${self:custom.funktor.LowConcurrencyQueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.LowConcurrencyQueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/low_concurrency_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.LowConcurrencyQueueHandler.batchSize, 1}

data/funktor-testapp/funktor_config/package.yml

@@ -1,10 +1,10 @@
 # TODO - Figure out how to allow individual packaging to work out of the box.
-individually: false
-include:
-  - Gemfile
-  - Gemfile.lock
-  - funktor_config/boot.rb
-  - app/**
+#individually: false
+#include:
+  #- Gemfile
+  #- Gemfile.lock
+  #- funktor_config/boot.rb
+  #- app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/funktor-testapp/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/funktor-testapp/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: ruby2.7
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -42,9 +52,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
@@ -62,4 +72,4 @@ resources:
   - ${file(funktor_config/resources/jobs_table.yml)}
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/cli/init.rb

@@ -135,6 +135,10 @@ module Funktor
         template "serverless.yml", File.join("serverless.yml")
       end
 
+      def dockerfile
+        template "Dockerfile", File.join("Dockerfile")
+      end
+
       private
 
       def app_worker_names

data/lib/funktor/cli/templates/Dockerfile

@@ -0,0 +1,63 @@
+FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
+
+# Uncomment this as a cache buster
+#RUN echo date
+
+# If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
+#ARG BUNDLE_GEM__FURY__IO
+#ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
+
+# First we install some stuff that we need for gems that have to compile native extensions
+#RUN yum groupinstall "Development Tools" -y
+#RUN yum install -y amazon-linux-extras
+#RUN amazon-linux-extras enable postgresql11
+#RUN yum install -y postgresql-devel
+
+# Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
+COPY Gemfile Gemfile.lock .
+
+# Set a few bundle configuration options
+RUN bundle lock --add-platform x86_64-linux
+#RUN bundle config --local deployment true
+#RUN bundle config --local plugins false
+#RUN bundle config --local frozen true
+#RUN bundle config --local without 'development test'
+RUN bundle config --local path './vendor/bundle'
+
+# Now install our gems
+RUN bundle install --quiet --jobs 4
+
+# Now we start a second stage in the build that is a clean image without build tools
+FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
+
+#ENV RAILS_ENV=production
+#ENV RACK_ENV=production
+
+# Install node so that asset related gems have a JS runtime.
+# We ship the node executeable to production to make it easier to get an app deployed.
+# TODO: Document steps that could be taken to remove this dependency.
+#RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
+#RUN yum install -y nodejs
+
+# Then copy some postgres related files
+#COPY --from=build_image  /usr/lib64/libpq.so.5 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/liblber-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsasl2.so.3 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libssl3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsmime3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libnss3.so /usr/lib64/
+
+
+# Next we copy the app from our local directory and we copy
+# the bundled gems from the build image.
+# We do this after copying dependencies becuase the app will
+# change more frequently, and we can used caching up to here.
+
+COPY --from=build_image /var/task .
+COPY . .
+
+# And finally we have the CMD for the deployed container
+# You can overwrite command in `serverless.yml` template
+CMD [ "app.LambdaFunction::Handler.process" ]
+

data/lib/funktor/cli/templates/funktor_config/environment.yml

@@ -1,7 +1,16 @@
-BUNDLE_WITHOUT: development:test:assets
+BUNDLE_WITHOUT: development:test
 BUNDLE_PLUGINS: false
 BUNDLE_FROZEN: true
+BUNDLE_DEPLOYMENT: true
 SERVERLESS_STAGE: ${self:custom.stage}
+
+FUNKTOR_LOG_LEVEL: INFO
+
+RAILS_LOG_TO_STDOUT: true
+RAILS_ENV: production
+RACK_ENV: production
+RAILS_MAX_THREADS: 1
+
 FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/incoming_job_handler.call
+#handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/incoming_job_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml

@@ -1,7 +1,11 @@
-handler: lambda_event_handlers/job_activator.call
+#handler: lambda_event_handlers/job_activator.call
 timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/job_activator.call
 events:
   - schedule: rate(1 minute)

data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
+#handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/package.yml

@@ -1,10 +1,10 @@
 # TODO - Figure out how to allow individual packaging to work out of the box.
-individually: false
-include:
-  - Gemfile
-  - Gemfile.lock
-  - funktor_config/boot.rb
-  - app/**
+#individually: false
+#include:
+  #- Gemfile
+  #- Gemfile.lock
+  #- funktor_config/boot.rb
+  #- app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/lib/funktor/cli/templates/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/lib/funktor/cli/templates/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: <%= runtime %>
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -39,9 +49,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   <%- all_function_definitions.each do |function_definition| -%>
@@ -54,4 +64,4 @@ resources:
   <%- end -%>
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/incoming_job_handler.rb

@@ -30,11 +30,13 @@ module Funktor
         # TODO : This number should be configurable via ENV var
         if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
           Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
-          # We push to the jobs table first becauase the work queue handler will expect to be able
-          # to update the stats of a record that's already in the table.
-          # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
-          # work queue handler to be ok with a job that's not yet in the table?
-          push_to_jobs_table(job, "queued")
+          if Funktor.enable_work_queue_visibility
+            # We push to the jobs table first becauase the work queue handler will expect to be able
+            # to update the stats of a record that's already in the table.
+            # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
+            # work queue handler to be ok with a job that's not yet in the table?
+            push_to_jobs_table(job, "queued")
+          end
           push_to_work_queue(job)
           if job.is_retry?
             @tracker.track(:retryActivated, job)

data/lib/funktor/job_activator.rb

@@ -61,23 +61,40 @@ module Funktor
     end
 
     def activate_job(job_shard, job_id, current_category, queue_immediately = false)
+      
+      # TODO: WorkQueueVisibilityMiddleware to alter what happens here? Maybe we delete by default and then the middleware puts it back in the table?
       # First we conditionally update the item in  Dynamo to be sure that another scheduler hasn't gotten
       # to it, and if that works then send to SQS. This is basically how Sidekiq scheduler works.
-      response = dynamodb_client.update_item({
-        key: {
-          "jobShard" => job_shard,
-          "jobId" => job_id
-        },
-        update_expression: "SET category = :category, queueable = :queueable",
-        condition_expression: "category = :current_category",
-        expression_attribute_values: {
-          ":current_category" => current_category,
-          ":queueable" => "false",
-          ":category" => "queued"
-        },
-        table_name: delayed_job_table,
-        return_values: "ALL_OLD"
-      })
+      response = if Funktor.enable_work_queue_visibility
+                   dynamodb_client.update_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     update_expression: "SET category = :category, queueable = :queueable",
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category,
+                       ":queueable" => "false",
+                       ":category" => "queued"
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 else
+                   dynamodb_client.delete_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 end
       if response.attributes # this means the record was still there in the state we expected
         Funktor.logger.debug "response.attributes ====== "
         Funktor.logger.debug response.attributes

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.7.5"
+  VERSION = "0.7.6"
 end

data/lib/funktor/work_queue_handler.rb

@@ -30,13 +30,18 @@ module Funktor
     def dispatch(job)
       begin
         @tracker.track(:processingStarted, job)
-        update_job_category(job, "processing")
+        if Funktor.enable_work_queue_visibility
+          update_job_category(job, "processing")
+        end
         Funktor.work_queue_handler_middleware.invoke(job) do
           job.execute
         end
         @processed_counter.incr(job)
         @tracker.track(:processingComplete, job)
-        delete_job_from_dynamodb(job)
+
+        if Funktor.enable_work_queue_visibility
+          delete_job_from_dynamodb(job)
+        end
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
         handle_error(e, job)
@@ -44,11 +49,17 @@ module Funktor
         job.error = e
         if job.can_retry
           @tracker.track(:retrying, job)
-          update_job_category(job, "retry")
+
+          if Funktor.enable_work_queue_visibility
+            update_job_category(job, "retry")
+          end
           trigger_retry(job)
         else
           @tracker.track(:bailingOut, job)
-          update_job_category(job, "dead")
+
+          if Funktor.enable_work_queue_visibility
+            update_job_category(job, "dead")
+          end
           Funktor.logger.error "We retried max times. We're bailing on this one."
           Funktor.logger.error job.to_json
         end
@@ -71,6 +82,7 @@ module Funktor
     end
 
     def update_job_category(job, category)
+      puts "starting update_job_category #{category}"
       dynamodb_client.update_item({
         key: {
           "jobShard" => job.shard,
@@ -84,9 +96,11 @@ module Funktor
         },
         return_values: "ALL_OLD"
       })
+      puts "ending update_job_category #{category}"
     end
 
     def delete_job_from_dynamodb(job)
+      puts "starting delete_job_from_dynamodb"
       dynamodb_client.delete_item({
         key: {
           "jobShard" => job.shard,
@@ -95,6 +109,7 @@ module Funktor
         table_name: delayed_job_table,
         return_values: "ALL_OLD"
       })
+      puts "ending delete_job_from_dynamodb"
     end
 
   end

data/lib/funktor.rb

@@ -18,7 +18,8 @@ module Funktor
 
   DEFAULT_OPTIONS = {
     error_handlers: [],
-    log_level: Logger::DEBUG # Set a high log level during early, active development
+    log_level: Logger::DEBUG, # Set a high log level during early, active development
+    enable_work_queue_visibility: true # Enable this by default during early, active development
   }
 
   def self.configure_job_pusher
@@ -110,6 +111,14 @@ module Funktor
     @logger = logger
   end
 
+  def self.enable_work_queue_visibility
+    options[:enable_work_queue_visibility]
+  end
+
+  def self.enable_work_queue_visibility= enabled
+    options[:enable_work_queue_visibility] = enabled
+  end
+
   # We have a raw_logger that doesn't add timestamps and what not. This is used to publish
   # CloudWatch metrics that can be used in dashboards.
   def self.raw_logger

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.6
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-04 00:00:00.000000000 Z
+date: 2022-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -176,6 +176,7 @@ files:
 - exe/funktor-deploy
 - funktor-testapp/.envrc
 - funktor-testapp/.gitignore
+- funktor-testapp/Dockerfile
 - funktor-testapp/Gemfile
 - funktor-testapp/Gemfile.lock
 - funktor-testapp/app/services/job_flood.rb
@@ -227,6 +228,7 @@ files:
 - lib/funktor/cli/generate/base.rb
 - lib/funktor/cli/generate/work_queue.rb
 - lib/funktor/cli/init.rb
+- lib/funktor/cli/templates/Dockerfile
 - lib/funktor/cli/templates/Gemfile
 - lib/funktor/cli/templates/app/workers/hello_worker.rb
 - lib/funktor/cli/templates/funktor_config/boot.rb