funktor 0.7.4 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e7890531c38acfc5bc6f24550ee96f1df4ce78bea2f329300dcd70f4bb3e357
-  data.tar.gz: a46dbd83e9ab5521868053b737f2e5b047bef2a4012041a12072e1d5b19cf034
+  metadata.gz: db4cbaf6480b6d6284a6370fbe9a87826647c72485ad75728ec7e28eae541836
+  data.tar.gz: 9b87f5a0ca280e84c4e321a8a86fa9926d32c420382737eb37ee59ca5f0cc73e
 SHA512:
-  metadata.gz: 3f108fea2135fd1675f62f4b211baa330207ca7be525851e141fe069d2ef042d17afeb88b3cabd4708917cf39cb9178df0634b4b7e05d36ed21ea6198118839c
-  data.tar.gz: a077b5c7d2429cfeb5a8fe2ff27fac22afbf129c29ce9ea35db091da0992a914e2d5b381c59ab6dd5d7919a2875996d594f4063b721aa6811777df7227d26807
+  metadata.gz: 1bd98cffe465815f6aa168fc9ec81eee8c7bcda99e7d65e01116204ee55c415877b4125dc119c3ea6e8cc3d17126d123baabe06e03f7bca92112a64eb108a2fd
+  data.tar.gz: 86c0c79b8d031b88a623489160a04cfc3e2d115c5bf4d460011b8a6a4a3539810fb931a607f8ed27a3e9a174914671d367b3238d33aa0bf4e5da10029011f588

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.7.3)
+    funktor (0.7.7)
       activesupport
       aws-sdk-dynamodb (~> 1.62)
       aws-sdk-sqs (~> 1.37)
@@ -22,19 +22,19 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.2.0)
-    aws-partitions (1.494.0)
-    aws-sdk-core (3.120.0)
+    aws-partitions (1.587.0)
+    aws-sdk-core (3.130.2)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
+      aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-dynamodb (1.63.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+    aws-sdk-dynamodb (1.74.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.44.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+    aws-sdk-sqs (1.51.1)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.3.0)
+    aws-sigv4 (1.5.0)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     coderay (1.1.3)
@@ -48,7 +48,7 @@ GEM
     hashdiff (1.0.1)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    jmespath (1.4.0)
+    jmespath (1.6.1)
     method_source (1.0.0)
     minitest (5.14.4)
     mustermann (1.1.1)
@@ -90,7 +90,7 @@ GEM
       rack (~> 2.2)
       rack-protection (= 2.1.0)
       tilt (~> 2.0)
-    thor (1.1.0)
+    thor (1.2.1)
     tilt (2.0.10)
     timecop (0.9.4)
     tzinfo (2.0.4)

data/funktor-testapp/Dockerfile

@@ -0,0 +1,63 @@
+FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
+
+# Uncomment this as a cache buster
+#RUN echo date
+
+# If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
+#ARG BUNDLE_GEM__FURY__IO
+#ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
+
+# First we install some stuff that we need for gems that have to compile native extensions
+#RUN yum groupinstall "Development Tools" -y
+#RUN yum install -y amazon-linux-extras
+#RUN amazon-linux-extras enable postgresql11
+#RUN yum install -y postgresql-devel
+
+# Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
+COPY Gemfile Gemfile.lock .
+
+# Set a few bundle configuration options
+RUN bundle lock --add-platform x86_64-linux
+#RUN bundle config --local deployment true
+#RUN bundle config --local plugins false
+#RUN bundle config --local frozen true
+#RUN bundle config --local without 'development test'
+RUN bundle config --local path './vendor/bundle'
+
+# Now install our gems
+RUN bundle install --quiet --jobs 4
+
+# Now we start a second stage in the build that is a clean image without build tools
+FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
+
+#ENV RAILS_ENV=production
+#ENV RACK_ENV=production
+
+# Install node so that asset related gems have a JS runtime.
+# We ship the node executeable to production to make it easier to get an app deployed.
+# TODO: Document steps that could be taken to remove this dependency.
+#RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
+#RUN yum install -y nodejs
+
+# Then copy some postgres related files
+#COPY --from=build_image  /usr/lib64/libpq.so.5 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/liblber-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsasl2.so.3 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libssl3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsmime3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libnss3.so /usr/lib64/
+
+
+# Next we copy the app from our local directory and we copy
+# the bundled gems from the build image.
+# We do this after copying dependencies becuase the app will
+# change more frequently, and we can used caching up to here.
+
+COPY --from=build_image /var/task .
+COPY . .
+
+# And finally we have the CMD for the deployed container
+# You can overwrite command in `serverless.yml` template
+CMD [ "app.LambdaFunction::Handler.process" ]
+

data/funktor-testapp/funktor_config/boot.rb

@@ -12,6 +12,9 @@ Bundler.require(:default, :production)
 $LOAD_PATH.unshift 'funktor/lib'
 require 'funktor'
 
+
+Funktor.enable_work_queue_visibility = false
+
 # Load all ruby files in the app directory
 require_rel File.join('..', 'app')
 

data/funktor-testapp/funktor_config/environment.yml

@@ -1,7 +1,16 @@
-BUNDLE_WITHOUT: development:test:assets
+BUNDLE_WITHOUT: development:test
 BUNDLE_PLUGINS: false
 BUNDLE_FROZEN: true
+BUNDLE_DEPLOYMENT: true
 SERVERLESS_STAGE: ${self:custom.stage}
+
+FUNKTOR_LOG_LEVEL: INFO
+
+RAILS_LOG_TO_STDOUT: true
+RAILS_ENV: production
+RACK_ENV: production
+RAILS_MAX_THREADS: 1
+
 FUNKTOR_APP_NAME: funktor-testapp
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/default_queue_handler.call
+#handler: lambda_event_handlers/default_queue_handler.call
 timeout: ${self:custom.funktor.DefaultQueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.DefaultQueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.DefaultQueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.DefaultQueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/default_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.DefaultQueueHandler.batchSize, 1}

data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/incoming_job_handler.call
+#handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/incoming_job_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}

data/funktor-testapp/funktor_config/function_definitions/job_activator.yml

@@ -1,7 +1,11 @@
-handler: lambda_event_handlers/job_activator.call
+#handler: lambda_event_handlers/job_activator.call
 timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/job_activator.call
 events:
   - schedule: rate(1 minute)

data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/low_concurrency_queue_handler.call
+#handler: lambda_event_handlers/low_concurrency_queue_handler.call
 timeout: ${self:custom.funktor.LowConcurrencyQueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.LowConcurrencyQueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/low_concurrency_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.LowConcurrencyQueueHandler.batchSize, 1}

data/funktor-testapp/funktor_config/package.yml

@@ -1,10 +1,10 @@
 # TODO - Figure out how to allow individual packaging to work out of the box.
-individually: false
-include:
-  - Gemfile
-  - Gemfile.lock
-  - funktor_config/boot.rb
-  - app/**
+#individually: false
+#include:
+  #- Gemfile
+  #- Gemfile.lock
+  #- funktor_config/boot.rb
+  #- app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml

@@ -117,17 +117,17 @@ Resources:
                 },
 
 
+                
                 { 
                     "height": 3,
                     "width": 24,
-                    "y": 9,
+                    "y": 12,
                     "x": 0,
                     "type": "text",
                     "properties": {
                         "markdown": "\n# Behind the scenes\n\n The stats below give some insight into the inner workings of the Funktor apparatus."
                     }
                 },
-                
 
 
                 

data/funktor-testapp/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/funktor-testapp/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: ruby2.7
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -42,9 +52,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
@@ -62,4 +72,4 @@ resources:
   - ${file(funktor_config/resources/jobs_table.yml)}
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/activity_tracker.rb

@@ -1,5 +1,4 @@
 require 'json'
-require 'aws-sdk-dynamodb'
 
 module Funktor
   class ActivityTracker
@@ -74,7 +73,7 @@ module Funktor
     end
 
     def dynamodb_client
-      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      Funktor.dynamodb_client
     end
 
     def metric_namespace

data/lib/funktor/cli/init.rb

@@ -135,6 +135,10 @@ module Funktor
         template "serverless.yml", File.join("serverless.yml")
       end
 
+      def dockerfile
+        template "Dockerfile", File.join("Dockerfile")
+      end
+
       private
 
       def app_worker_names

data/lib/funktor/cli/templates/Dockerfile

@@ -0,0 +1,63 @@
+FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
+
+# Uncomment this as a cache buster
+#RUN echo date
+
+# If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
+#ARG BUNDLE_GEM__FURY__IO
+#ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
+
+# First we install some stuff that we need for gems that have to compile native extensions
+#RUN yum groupinstall "Development Tools" -y
+#RUN yum install -y amazon-linux-extras
+#RUN amazon-linux-extras enable postgresql11
+#RUN yum install -y postgresql-devel
+
+# Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
+COPY Gemfile Gemfile.lock .
+
+# Set a few bundle configuration options
+RUN bundle lock --add-platform x86_64-linux
+#RUN bundle config --local deployment true
+#RUN bundle config --local plugins false
+#RUN bundle config --local frozen true
+#RUN bundle config --local without 'development test'
+RUN bundle config --local path './vendor/bundle'
+
+# Now install our gems
+RUN bundle install --quiet --jobs 4
+
+# Now we start a second stage in the build that is a clean image without build tools
+FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
+
+#ENV RAILS_ENV=production
+#ENV RACK_ENV=production
+
+# Install node so that asset related gems have a JS runtime.
+# We ship the node executeable to production to make it easier to get an app deployed.
+# TODO: Document steps that could be taken to remove this dependency.
+#RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
+#RUN yum install -y nodejs
+
+# Then copy some postgres related files
+#COPY --from=build_image  /usr/lib64/libpq.so.5 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/liblber-2.4.so.2 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsasl2.so.3 /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libssl3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libsmime3.so /usr/lib64/
+#COPY --from=build_image  /usr/lib64/libnss3.so /usr/lib64/
+
+
+# Next we copy the app from our local directory and we copy
+# the bundled gems from the build image.
+# We do this after copying dependencies becuase the app will
+# change more frequently, and we can used caching up to here.
+
+COPY --from=build_image /var/task .
+COPY . .
+
+# And finally we have the CMD for the deployed container
+# You can overwrite command in `serverless.yml` template
+CMD [ "app.LambdaFunction::Handler.process" ]
+

data/lib/funktor/cli/templates/funktor_config/environment.yml

@@ -1,7 +1,16 @@
-BUNDLE_WITHOUT: development:test:assets
+BUNDLE_WITHOUT: development:test
 BUNDLE_PLUGINS: false
 BUNDLE_FROZEN: true
+BUNDLE_DEPLOYMENT: true
 SERVERLESS_STAGE: ${self:custom.stage}
+
+FUNKTOR_LOG_LEVEL: INFO
+
+RAILS_LOG_TO_STDOUT: true
+RAILS_ENV: production
+RACK_ENV: production
+RAILS_MAX_THREADS: 1
+
 FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/incoming_job_handler.call
+#handler: lambda_event_handlers/incoming_job_handler.call
 timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/incoming_job_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml

@@ -1,7 +1,11 @@
-handler: lambda_event_handlers/job_activator.call
+#handler: lambda_event_handlers/job_activator.call
 timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
 reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/job_activator.call
 events:
   - schedule: rate(1 minute)

data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml

@@ -1,8 +1,12 @@
-handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
+#handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
 reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
 provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
 memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
+image:
+  name: funktorimage
+  command:
+    - lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
 events:
   - sqs:
       batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}

data/lib/funktor/cli/templates/funktor_config/package.yml

@@ -1,10 +1,10 @@
 # TODO - Figure out how to allow individual packaging to work out of the box.
-individually: false
-include:
-  - Gemfile
-  - Gemfile.lock
-  - funktor_config/boot.rb
-  - app/**
+#individually: false
+#include:
+  #- Gemfile
+  #- Gemfile.lock
+  #- funktor_config/boot.rb
+  #- app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.
 #exclude:

data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml

@@ -109,6 +109,7 @@ Resources:
                 },
 
 
+                <% current_y += 3 %>
                 { <% "Funktor Behind the Scenes Banner" %>
                     "height": 3,
                     "width": 24,
@@ -119,7 +120,6 @@ Resources:
                         "markdown": "\n# Behind the scenes\n\n The stats below give some insight into the inner workings of the Funktor apparatus."
                     }
                 },
-                <% current_y += 3 %>
 
 
                 <% current_y += 6 %>

data/lib/funktor/cli/templates/funktor_config/ruby_layer.yml

@@ -1,4 +1,4 @@
-use_docker: false
+#use_docker: false
 #docker_yums:
   #- postgresql-devel
 #native_libs:

data/lib/funktor/cli/templates/serverless.yml

@@ -25,6 +25,16 @@ provider:
   runtime: <%= runtime %>
   stage: ${opt:stage, 'dev'}
   lambdaHashingVersion: 20201221
+  ecr:
+    scanOnPush: true
+    images:
+      funktorimage:
+        path: ./
+        file: Dockerfile
+        #platform: linux/amd64 # May be needed if you're on an M1.
+        buildArgs:
+          # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
+          #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   logRetentionInDays: 7
@@ -39,9 +49,9 @@ custom:
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
   funktor: ${file(funktor_config/funktor.yml)}
-  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
+  #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(funktor_config/package.yml)}
+#package: ${file(funktor_config/package.yml)}
 
 functions:
   <%- all_function_definitions.each do |function_definition| -%>
@@ -54,4 +64,4 @@ resources:
   <%- end -%>
 
 plugins:
-  - serverless-ruby-layer
+  #- serverless-ruby-layer

data/lib/funktor/incoming_job_handler.rb

@@ -18,11 +18,11 @@ module Funktor
     end
 
     def sqs_client
-      @sqs_client ||= ::Aws::SQS::Client.new
+      Funktor.sqs_client
     end
 
     def dynamodb_client
-      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      Funktor.dynamodb_client
     end
 
     def dispatch(job)
@@ -30,11 +30,13 @@ module Funktor
         # TODO : This number should be configurable via ENV var
         if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
           Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
-          # We push to the jobs table first becauase the work queue handler will expect to be able
-          # to update the stats of a record that's already in the table.
-          # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
-          # work queue handler to be ok with a job that's not yet in the table?
-          push_to_jobs_table(job, "queued")
+          if Funktor.enable_work_queue_visibility
+            # We push to the jobs table first becauase the work queue handler will expect to be able
+            # to update the stats of a record that's already in the table.
+            # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
+            # work queue handler to be ok with a job that's not yet in the table?
+            push_to_jobs_table(job, "queued")
+          end
           push_to_work_queue(job)
           if job.is_retry?
             @tracker.track(:retryActivated, job)

data/lib/funktor/job_activator.rb

@@ -9,11 +9,11 @@ module Funktor
     end
 
     def dynamodb_client
-      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      Funktor.dynamodb_client
     end
 
     def sqs_client
-      @sqs_client ||= ::Aws::SQS::Client.new
+      Funktor.sqs_client
     end
 
     def delayed_job_table
@@ -61,23 +61,39 @@ module Funktor
     end
 
     def activate_job(job_shard, job_id, current_category, queue_immediately = false)
+      # TODO: WorkQueueVisibilityMiddleware to alter what happens here? Maybe we delete by default and then the middleware puts it back in the table?
       # First we conditionally update the item in  Dynamo to be sure that another scheduler hasn't gotten
       # to it, and if that works then send to SQS. This is basically how Sidekiq scheduler works.
-      response = dynamodb_client.update_item({
-        key: {
-          "jobShard" => job_shard,
-          "jobId" => job_id
-        },
-        update_expression: "SET category = :category, queueable = :queueable",
-        condition_expression: "category = :current_category",
-        expression_attribute_values: {
-          ":current_category" => current_category,
-          ":queueable" => "false",
-          ":category" => "queued"
-        },
-        table_name: delayed_job_table,
-        return_values: "ALL_OLD"
-      })
+      response = if Funktor.enable_work_queue_visibility
+                   dynamodb_client.update_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     update_expression: "SET category = :category, queueable = :queueable",
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category,
+                       ":queueable" => "false",
+                       ":category" => "queued"
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 else
+                   dynamodb_client.delete_item({
+                     key: {
+                       "jobShard" => job_shard,
+                       "jobId" => job_id
+                     },
+                     condition_expression: "category = :current_category",
+                     expression_attribute_values: {
+                       ":current_category" => current_category
+                     },
+                     table_name: delayed_job_table,
+                     return_values: "ALL_OLD"
+                   })
+                 end
       if response.attributes # this means the record was still there in the state we expected
         Funktor.logger.debug "response.attributes ====== "
         Funktor.logger.debug response.attributes

data/lib/funktor/job_pusher.rb

@@ -19,7 +19,7 @@ module Funktor
     private
 
     def client
-      @client ||= ::Aws::SQS::Client.new
+      Funktor.sqs_client
     end
 
     def queue_url(payload)

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.7.4"
+  VERSION = "0.7.7"
 end

data/lib/funktor/web/application.rb

@@ -128,7 +128,7 @@ module Funktor
       end
 
       def dynamodb_client
-        @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+        Funktor.dynamodb_client
       end
 
       # start the server if ruby file executed directly

data/lib/funktor/work_queue_handler.rb

@@ -20,23 +20,28 @@ module Funktor
     end
 
     def dynamodb_client
-      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      Funktor.dynamodb_client
     end
 
     def sqs_client
-      @sqs_client ||= ::Aws::SQS::Client.new
+      Funktor.sqs_client
     end
 
     def dispatch(job)
       begin
         @tracker.track(:processingStarted, job)
-        update_job_category(job, "processing")
+        if Funktor.enable_work_queue_visibility
+          update_job_category(job, "processing")
+        end
         Funktor.work_queue_handler_middleware.invoke(job) do
           job.execute
         end
         @processed_counter.incr(job)
         @tracker.track(:processingComplete, job)
-        delete_job_from_dynamodb(job)
+
+        if Funktor.enable_work_queue_visibility
+          delete_job_from_dynamodb(job)
+        end
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
         handle_error(e, job)
@@ -44,11 +49,17 @@ module Funktor
         job.error = e
         if job.can_retry
           @tracker.track(:retrying, job)
-          update_job_category(job, "retry")
+
+          if Funktor.enable_work_queue_visibility
+            update_job_category(job, "retry")
+          end
           trigger_retry(job)
         else
           @tracker.track(:bailingOut, job)
-          update_job_category(job, "dead")
+
+          if Funktor.enable_work_queue_visibility
+            update_job_category(job, "dead")
+          end
           Funktor.logger.error "We retried max times. We're bailing on this one."
           Funktor.logger.error job.to_json
         end
@@ -71,6 +82,7 @@ module Funktor
     end
 
     def update_job_category(job, category)
+      puts "starting update_job_category #{category}"
       dynamodb_client.update_item({
         key: {
           "jobShard" => job.shard,
@@ -84,9 +96,11 @@ module Funktor
         },
         return_values: "ALL_OLD"
       })
+      puts "ending update_job_category #{category}"
     end
 
     def delete_job_from_dynamodb(job)
+      puts "starting delete_job_from_dynamodb"
       dynamodb_client.delete_item({
         key: {
           "jobShard" => job.shard,
@@ -95,6 +109,7 @@ module Funktor
         table_name: delayed_job_table,
         return_values: "ALL_OLD"
       })
+      puts "ending delete_job_from_dynamodb"
     end
 
   end

data/lib/funktor.rb

@@ -12,13 +12,16 @@ require_relative 'funktor/job_activator'
 require_relative 'funktor/activity_tracker'
 
 require 'json'
+require 'aws-sdk-dynamodb'
+require 'aws-sdk-sqs'
 
 module Funktor
   class Error < StandardError; end
 
   DEFAULT_OPTIONS = {
     error_handlers: [],
-    log_level: Logger::DEBUG # Set a high log level during early, active development
+    log_level: Logger::DEBUG, # Set a high log level during early, active development
+    enable_work_queue_visibility: true # Enable this by default during early, active development
   }
 
   def self.configure_job_pusher
@@ -110,6 +113,14 @@ module Funktor
     @logger = logger
   end
 
+  def self.enable_work_queue_visibility
+    options[:enable_work_queue_visibility]
+  end
+
+  def self.enable_work_queue_visibility= enabled
+    options[:enable_work_queue_visibility] = enabled
+  end
+
   # We have a raw_logger that doesn't add timestamps and what not. This is used to publish
   # CloudWatch metrics that can be used in dashboards.
   def self.raw_logger
@@ -126,6 +137,14 @@ module Funktor
 
     @raw_logger = raw_logger
   end
+
+  def self.dynamodb_client
+    @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+  end
+
+  def self.sqs_client
+    @sqs_client ||= ::Aws::SQS::Client.new
+  end
 end
 
 # TODO - Is it a code smell that we need to include these at the bottom, after

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.7
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-04 00:00:00.000000000 Z
+date: 2022-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -176,6 +176,7 @@ files:
 - exe/funktor-deploy
 - funktor-testapp/.envrc
 - funktor-testapp/.gitignore
+- funktor-testapp/Dockerfile
 - funktor-testapp/Gemfile
 - funktor-testapp/Gemfile.lock
 - funktor-testapp/app/services/job_flood.rb
@@ -227,6 +228,7 @@ files:
 - lib/funktor/cli/generate/base.rb
 - lib/funktor/cli/generate/work_queue.rb
 - lib/funktor/cli/init.rb
+- lib/funktor/cli/templates/Dockerfile
 - lib/funktor/cli/templates/Gemfile
 - lib/funktor/cli/templates/app/workers/hello_worker.rb
 - lib/funktor/cli/templates/funktor_config/boot.rb