funktor 0.4.4 → 0.5.0

data/lib/funktor/job_pusher.rb

@@ -4,8 +4,6 @@ module Funktor
   class JobPusher
 
     def push(payload)
-      puts "payload ============"
-      pp payload
       job_id = SecureRandom.uuid
       payload[:job_id] = job_id
 

data/lib/funktor/middleware/metrics.rb

@@ -10,7 +10,12 @@ module Funktor
       end
 
       def put_metric_to_stdout(time_diff, job)
-        puts Funktor.dump_json(metric_hash(time_diff, job))
+        # NOTE : We use raw_logger here instead of Funktor.loggert o avoid getting extra
+        # timestamps or log level information in the log line. We need this specific format to
+        # be the only thing in the line so that CloudWatch can parse the logs and use the data.
+        # 'unknown' is a log level that will always be logged, no matter what is set in the
+        # runtime environment as far as log level.
+        Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(time_diff, job))
       end
 
       def metric_hash(time_diff_in_seconds, job)
@@ -43,8 +48,8 @@ module Funktor
   end
 end
 
-Funktor.configure_active_job_handler do |config|
-  config.active_job_handler_middleware do |chain|
+Funktor.configure_work_queue_handler do |config|
+  config.work_queue_handler_middleware do |chain|
     chain.add Funktor::Middleware::Metrics
   end
 end

data/lib/funktor/testing.rb

@@ -2,6 +2,7 @@ require 'funktor/worker'
 require 'funktor/fake_job_queue'
 
 module Funktor
+
   module Worker
     def self.clear_all
       Funktor::FakeJobQueue.clear_all
@@ -25,47 +26,68 @@ module Funktor
       end
     end
   end
+
   class Testing
+    class << self
+      attr_accessor :mode
 
-    def self.inline!(&block)
-      Funktor.configure_job_pusher do |config|
-        config.job_pusher_middleware do |chain|
-          chain.add Funktor::InlineJobPusherMiddleware
-        end
+      def inline?
+        mode == :inline
       end
-      yield
-      Funktor.configure_job_pusher do |config|
-        config.job_pusher_middleware do |chain|
-          chain.remove Funktor::InlineJobPusherMiddleware
-        end
+
+      def fake?
+        mode == :fake
       end
-    end
-    def self.fake!(&block)
-      Funktor.configure_job_pusher do |config|
-        config.job_pusher_middleware do |chain|
-          chain.add Funktor::FakeJobPusherMiddleware
+
+      def inline!(&block)
+        unless block_given?
+          raise "Funktor inline testing mode can only be called in block form."
         end
+        set_mode(:inline, &block)
+      end
+
+      def fake!(&block)
+        set_mode(:fake, &block)
       end
-      yield
-      Funktor.configure_job_pusher do |config|
-        config.job_pusher_middleware do |chain|
-          chain.remove Funktor::FakeJobPusherMiddleware
+
+      def disable!
+        set_mode(:disabled)
+      end
+
+      def set_mode(new_mode, &block)
+        if block_given?
+          original_mode = mode
+          self.mode = new_mode
+          begin
+            yield
+          ensure
+            self.mode = original_mode
+          end
+        else
+          self.mode = new_mode
         end
       end
     end
   end
 
-  class InlineJobPusherMiddleware
-    def call(payload)
-      payload = payload.with_indifferent_access
-      worker = Object.const_get payload["worker"]
-      worker.new.perform(*payload["worker_params"])
+  module TestingPusher
+    def push(payload)
+      if Funktor::Testing.inline?
+        Funktor.job_pusher_middleware.invoke(payload) do
+          payload = payload.with_indifferent_access
+          worker = Object.const_get payload["worker"]
+          worker.new.perform(*payload["worker_params"])
+        end
+      elsif Funktor::Testing.fake?
+        Funktor.job_pusher_middleware.invoke(payload) do
+          Funktor::FakeJobQueue.push(payload)
+        end
+      else
+        super
+      end
     end
   end
 
-  class FakeJobPusherMiddleware
-    def call(payload)
-      Funktor::FakeJobQueue.push(payload)
-    end
-  end
+  Funktor::JobPusher.prepend TestingPusher
+
 end

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.4.4"
+  VERSION = "0.5.0"
 end

data/lib/funktor/{active_job_handler.rb → work_queue_handler.rb}

@@ -1,17 +1,18 @@
 require 'aws-sdk-sqs'
 
 module Funktor
-  class ActiveJobHandler
+  class WorkQueueHandler
     include Funktor::ErrorHandler
 
     def initialize
       @failed_counter = Funktor::Counter.new('failed')
       @processed_counter = Funktor::Counter.new('processed')
+      @tracker = Funktor::ActivityTracker.new
     end
 
     def call(event:, context:)
       event = Funktor::Aws::Sqs::Event.new(event)
-      puts "event.jobs.count = #{event.jobs.count}"
+      Funktor.logger.debug "event.jobs.count = #{event.jobs.count}"
       event.jobs.each do |job|
         dispatch(job)
       end
@@ -23,31 +24,32 @@ module Funktor
 
     def dispatch(job)
       begin
-        Funktor.active_job_handler_middleware.invoke(job) do
+        @tracker.track(:processingStarted, job)
+        Funktor.work_queue_handler_middleware.invoke(job) do
           job.execute
         end
         @processed_counter.incr(job)
+        @tracker.track(:processingComplete, job)
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
         handle_error(e, job)
         @failed_counter.incr(job)
-        attempt_retry_or_bail(job)
-      end
-    end
-
-    def attempt_retry_or_bail(job)
-      if job.can_retry
-        trigger_retry(job)
-      else
-        puts "We retried max times. We're bailing on this one."
-        puts job.to_json
+        if job.can_retry
+          @tracker.track(:retrying, job)
+          trigger_retry(job)
+        else
+          @tracker.track(:bailingOut, job)
+          Funktor.logger.error "We retried max times. We're bailing on this one."
+          Funktor.logger.error job.to_json
+        end
+        @tracker.track(:processingFailed, job)
       end
     end
 
     def trigger_retry(job)
       job.increment_retries
-      puts "scheduling retry # #{job.retries} with delay of #{job.delay}"
-      puts job.to_json
+      Funktor.logger.error "scheduling retry # #{job.retries} with delay of #{job.delay}"
+      Funktor.logger.error job.to_json
       sqs_client.send_message({
         queue_url: job.retry_queue_url,
         message_body: job.to_json

data/lib/funktor/worker.rb

@@ -27,9 +27,6 @@ module Funktor::Worker
     end
 
     def perform_in(delay, *worker_params)
-      if delay > max_delay
-        raise Funktor::DelayTooLongError.new("The delay can't be longer than #{max_delay} seconds. This is a limitation of SQS. Funktor Pro has mechanisms to work around this limitation.")
-      end
       self.push(delay, *worker_params)
     end
 
@@ -38,10 +35,6 @@ module Funktor::Worker
       Funktor.job_pusher.push(payload)
     end
 
-    def max_delay
-      900
-    end
-
     def build_job_payload(delay, *worker_params)
       {
         worker: self.name,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-02 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.37'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-dynamodb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.62'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.62'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -120,6 +134,7 @@ files:
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".tool-versions"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -131,10 +146,52 @@ files:
 - bin/setup
 - exe/funktor
 - exe/funktor-deploy
+- funktor-testapp/.envrc
+- funktor-testapp/.gitignore
+- funktor-testapp/Gemfile
+- funktor-testapp/Gemfile.lock
+- funktor-testapp/app/services/job_flood.rb
+- funktor-testapp/app/workers/audit_worker.rb
+- funktor-testapp/app/workers/greetings_worker.rb
+- funktor-testapp/app/workers/hello_worker.rb
+- funktor-testapp/deploy-dev.sh
+- funktor-testapp/funktor_config/boot.rb
+- funktor-testapp/funktor_config/environment.yml
+- funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml
+- funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml
+- funktor-testapp/funktor_config/function_definitions/job_activator.yml
+- funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
+- funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml
+- funktor-testapp/funktor_config/funktor.yml
+- funktor-testapp/funktor_config/iam_permissions/activity_table.yml
+- funktor-testapp/funktor_config/iam_permissions/default_queue.yml
+- funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
+- funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
+- funktor-testapp/funktor_config/iam_permissions/jobs_table_secondary_index.yml
+- funktor-testapp/funktor_config/iam_permissions/single_thread_queue.yml
+- funktor-testapp/funktor_config/iam_permissions/ssm.yml
+- funktor-testapp/funktor_config/package.yml
+- funktor-testapp/funktor_config/resources/activity_table.yml
+- funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml
+- funktor-testapp/funktor_config/resources/default_queue.yml
+- funktor-testapp/funktor_config/resources/incoming_job_queue.yml
+- funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml
+- funktor-testapp/funktor_config/resources/jobs_table.yml
+- funktor-testapp/funktor_config/resources/single_thread_queue.yml
+- funktor-testapp/funktor_config/ruby_layer.yml
+- funktor-testapp/funktor_init.yml
+- funktor-testapp/lambda_event_handlers/default_queue_handler.rb
+- funktor-testapp/lambda_event_handlers/incoming_job_handler.rb
+- funktor-testapp/lambda_event_handlers/job_activator.rb
+- funktor-testapp/lambda_event_handlers/random_job_generator.rb
+- funktor-testapp/lambda_event_handlers/single_thread_queue_handler.rb
+- funktor-testapp/package-lock.json
+- funktor-testapp/package.json
+- funktor-testapp/serverless.yml
 - funktor.gemspec
 - lib/active_job/queue_adapters/funktor_adapter.rb
 - lib/funktor.rb
-- lib/funktor/active_job_handler.rb
+- lib/funktor/activity_tracker.rb
 - lib/funktor/aws/sqs/event.rb
 - lib/funktor/aws/sqs/record.rb
 - lib/funktor/cli/application.rb
@@ -148,20 +205,27 @@ files:
 - lib/funktor/cli/templates/funktor_config/boot.rb
 - lib/funktor/cli/templates/funktor_config/environment.yml
 - lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml
+- lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml
 - lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
 - lib/funktor/cli/templates/funktor_config/funktor.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
 - lib/funktor/cli/templates/funktor_config/package.yml
+- lib/funktor/cli/templates/funktor_config/resources/activity_table.yml
 - lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
 - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
 - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
+- lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml
 - lib/funktor/cli/templates/funktor_config/resources/work_queue.yml
 - lib/funktor/cli/templates/funktor_config/ruby_layer.yml
 - lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
 - lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
+- lib/funktor/cli/templates/lambda_event_handlers/job_activator.rb
 - lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/package.json
 - lib/funktor/cli/templates/serverless.yml
@@ -170,6 +234,7 @@ files:
 - lib/funktor/fake_job_queue.rb
 - lib/funktor/incoming_job_handler.rb
 - lib/funktor/job.rb
+- lib/funktor/job_activator.rb
 - lib/funktor/job_pusher.rb
 - lib/funktor/logger.rb
 - lib/funktor/middleware/metrics.rb
@@ -177,6 +242,7 @@ files:
 - lib/funktor/rails.rb
 - lib/funktor/testing.rb
 - lib/funktor/version.rb
+- lib/funktor/work_queue_handler.rb
 - lib/funktor/worker.rb
 - lib/funktor/worker/funktor_options.rb
 homepage: https://github.com/Octo-Labs/funktor