funktor 0.4.7 → 0.6.2

data/lib/funktor/web/views/processing.erb

@@ -0,0 +1,29 @@
+<h1>Processing</h1>
+<p>Showing <%= jobs.count %> of <%= @activity_stats["processing"] %></p>
+<table role="grid">
+  <tr>
+    <td>ID</td>
+    <td>Perform at</td>
+    <td>Worker</td>
+    <td>Queue</td>
+    <td>Params</td>
+    <td>Attempts</td>
+    <td>Last error</td>
+  </tr>
+  <% jobs.each do |job| %>
+    <tr>
+      <td><%= job.job_id %></td>
+      <td><%= job.perform_at %></td>
+      <td><%= job.worker_class_name %></td>
+      <td><%= job.queue %></td>
+      <td><%= job.worker_params %></td>
+      <td><%= job.retries %></td>
+      <td>
+        <%= job.error_class %> - <%= job.error_message %>
+        <br/>
+        <%= job.error_backtrace.first %>
+      </td>
+    </tr>
+  <% end %>
+</table>
+

data/lib/funktor/web/views/queued.erb

@@ -0,0 +1,29 @@
+<h1>Queued</h1>
+<p>Showing <%= jobs.count %> of <%= @activity_stats["queued"] %></p>
+<table role="grid">
+  <tr>
+    <td>ID</td>
+    <td>Perform at</td>
+    <td>Worker</td>
+    <td>Queue</td>
+    <td>Params</td>
+    <td>Attempts</td>
+    <td>Last error</td>
+  </tr>
+  <% jobs.each do |job| %>
+    <tr>
+      <td><%= job.job_id %></td>
+      <td><%= job.perform_at %></td>
+      <td><%= job.worker_class_name %></td>
+      <td><%= job.queue %></td>
+      <td><%= job.worker_params %></td>
+      <td><%= job.retries %></td>
+      <td>
+        <%= job.error_class %> - <%= job.error_message %>
+        <br/>
+        <%= job.error_backtrace.first %>
+      </td>
+    </tr>
+  <% end %>
+</table>
+

data/lib/funktor/web/views/retries.erb

@@ -0,0 +1,35 @@
+<h1>Retries</h1>
+<form action="/update_jobs" method="post" accept-charset="utf-8">
+  <input type="hidden" name="source" value="retry" />
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
+  <table role="grid">
+    <tr>
+      <td>&nbsp;</td>
+      <td>ID</td>
+      <td>Perform at</td>
+      <td>Worker</td>
+      <td>Queue</td>
+      <td>Params</td>
+      <td>Attempts</td>
+      <td>Last error</td>
+    </tr>
+    <% jobs.each do |job| %>
+      <tr>
+        <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
+        <td><%= job.job_id %></td>
+        <td><%= job.perform_at %></td>
+        <td><%= job.worker_class_name %></td>
+        <td><%= job.queue %></td>
+        <td><%= job.worker_params %></td>
+        <td><%= job.retries %></td>
+        <td>
+          <%= job.error_class %> - <%= job.error_message %>
+          <br/>
+          <%= job.error_backtrace.first %>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
+</form>
+

data/lib/funktor/web/views/scheduled.erb

@@ -0,0 +1,26 @@
+<h1>Scheduled</h1>
+<form action="/update_jobs" method="post" accept-charset="utf-8">
+  <input type="hidden" name="source" value="scheduled" />
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
+  <table role="grid">
+    <tr>
+      <td>&nbsp;</td>
+      <td>ID</td>
+      <td>Perform at</td>
+      <td>Worker</td>
+      <td>Queue</td>
+      <td>Params</td>
+    </tr>
+    <% jobs.each do |job| %>
+      <tr>
+        <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
+        <td><%= job.job_id %></td>
+        <td><%= job.perform_at %></td>
+        <td><%= job.worker_class_name %></td>
+        <td><%= job.queue %></td>
+        <td><%= job.worker_params %></td>
+      </tr>
+    <% end %>
+  </table>
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
+</form>

data/lib/funktor/web/views/stats.erb

@@ -0,0 +1,9 @@
+<table class="header">
+  <tr>
+    <td><h5>Funktor</h5></td>
+    <% @activity_stats.each_pair do |stat, value| %>
+      <td><a href="/<%= stat %>"><%= stat %>: <%= value %></a></td>
+    <% end %>
+    <td>Current time UTC: <%= Time.now.utc %></td>
+  </tr>
+</table>

data/lib/funktor/web/views/table_stats_with_buttons.erb

@@ -0,0 +1,11 @@
+<div class="grid">
+  <div>
+    Showing <%= jobs.count %> of <%= @activity_stats[stat_category] %>
+  </div>
+  <div>
+    <input type="submit" name="submit" value="Delete Selected Jobs" />
+  </div>
+  <div>
+    <input type="submit" name="submit" value="Queue Selected Jobs" />
+  </div>
+</div>

data/lib/funktor/web.rb

@@ -0,0 +1 @@
+require 'funktor/web/application'

data/lib/funktor/work_queue_handler.rb

@@ -0,0 +1,101 @@
+require 'aws-sdk-sqs'
+require 'aws-sdk-dynamodb'
+
+module Funktor
+  class WorkQueueHandler
+    include Funktor::ErrorHandler
+
+    def initialize
+      @failed_counter = Funktor::Counter.new('failed')
+      @processed_counter = Funktor::Counter.new('processed')
+      @tracker = Funktor::ActivityTracker.new
+    end
+
+    def call(event:, context:)
+      event = Funktor::Aws::Sqs::Event.new(event)
+      Funktor.logger.debug "event.jobs.count = #{event.jobs.count}"
+      event.jobs.each do |job|
+        dispatch(job)
+      end
+    end
+
+    def dynamodb_client
+      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+    end
+
+    def sqs_client
+      @sqs_client ||= ::Aws::SQS::Client.new
+    end
+
+    def dispatch(job)
+      begin
+        @tracker.track(:processingStarted, job)
+        update_job_category(job, "processing")
+        Funktor.work_queue_handler_middleware.invoke(job) do
+          job.execute
+        end
+        @processed_counter.incr(job)
+        @tracker.track(:processingComplete, job)
+        delete_job_from_dynamodb(job)
+      # rescue Funktor::Job::InvalidJsonError # TODO Make this work
+      rescue Exception => e
+        handle_error(e, job)
+        @failed_counter.incr(job)
+        job.error = e
+        if job.can_retry
+          @tracker.track(:retrying, job)
+          update_job_category(job, "retry")
+          trigger_retry(job)
+        else
+          @tracker.track(:bailingOut, job)
+          update_job_category(job, "dead")
+          Funktor.logger.error "We retried max times. We're bailing on this one."
+          Funktor.logger.error job.to_json
+        end
+        @tracker.track(:processingFailed, job)
+      end
+    end
+
+    def trigger_retry(job)
+      job.increment_retries
+      Funktor.logger.error "scheduling retry # #{job.retries} with delay of #{job.delay}"
+      Funktor.logger.error job.to_json
+      sqs_client.send_message({
+        queue_url: job.retry_queue_url,
+        message_body: job.to_json
+      })
+    end
+
+    def delayed_job_table
+      ENV['FUNKTOR_JOBS_TABLE']
+    end
+
+    def update_job_category(job, category)
+      dynamodb_client.update_item({
+        key: {
+          "jobShard" => job.shard,
+          "jobId" => job.job_id
+        },
+        table_name: delayed_job_table,
+        update_expression: "SET category = :category, queueable = :queueable",
+        expression_attribute_values: {
+          ":queueable" => "false",
+          ":category" => category
+        },
+        return_values: "ALL_OLD"
+      })
+    end
+
+    def delete_job_from_dynamodb(job)
+      dynamodb_client.delete_item({
+        key: {
+          "jobShard" => job.shard,
+          "jobId" => job.job_id
+        },
+        table_name: delayed_job_table,
+        return_values: "ALL_OLD"
+      })
+    end
+
+  end
+end

data/lib/funktor/worker/funktor_options.rb

@@ -1,10 +1,12 @@
+require 'active_support/core_ext/class/attribute'
+
 module Funktor
   module Worker
     module FunktorOptions
       def self.included(base)
         base.extend ClassMethods
         base.class_eval do
-          cattr_accessor :funktor_options_hash
+          class_attribute :funktor_options_hash
         end
       end
       module ClassMethods

data/lib/funktor/worker.rb

@@ -18,37 +18,27 @@ module Funktor::Worker
       self.perform_in(0, *worker_params)
     end
 
-    def perform_at(time, *worker_params)
-      delay = (time.utc - Time.now.utc).round
-      if delay < 0
-        delay = 0
-      end
-      self.perform_in(delay, *worker_params)
+    def perform_at(job_time, *worker_params)
+      self.push(job_time, *worker_params)
     end
 
     def perform_in(delay, *worker_params)
-      if delay > max_delay
-        raise Funktor::DelayTooLongError.new("The delay can't be longer than #{max_delay} seconds. This is a limitation of SQS. Funktor Pro has mechanisms to work around this limitation.")
-      end
-      self.push(delay, *worker_params)
+      job_time = Time.now.utc + delay
+      self.perform_at(job_time, *worker_params)
     end
 
-    def push(delay, *worker_params)
-      payload = build_job_payload(delay, *worker_params)
+    def push(job_time, *worker_params)
+      payload = build_job_payload(job_time, *worker_params)
       Funktor.job_pusher.push(payload)
     end
 
-    def max_delay
-      900
-    end
-
-    def build_job_payload(delay, *worker_params)
+    def build_job_payload(job_time, *worker_params)
       {
         worker: self.name,
         worker_params: worker_params,
         queue: self.work_queue,
         incoming_job_queue_url: self.queue_url,
-        delay: delay,
+        perform_at: job_time.utc,
         funktor_options: get_funktor_options
       }
     end

data/lib/funktor.rb

@@ -1,13 +1,15 @@
-require "funktor/version"
-require 'funktor/aws/sqs/event'
-require 'funktor/aws/sqs/record'
-require 'funktor/counter'
-require 'funktor/job'
-require 'funktor/job_pusher'
-require 'funktor/logger'
-require 'funktor/worker'
-require 'funktor/middleware_chain'
-require 'funktor/incoming_job_handler'
+require_relative "funktor/version"
+require_relative 'funktor/aws/sqs/event'
+require_relative 'funktor/aws/sqs/record'
+require_relative 'funktor/counter'
+require_relative 'funktor/job'
+require_relative 'funktor/job_pusher'
+require_relative 'funktor/logger'
+require_relative 'funktor/worker'
+require_relative 'funktor/middleware_chain'
+require_relative 'funktor/incoming_job_handler'
+require_relative 'funktor/job_activator'
+require_relative 'funktor/activity_tracker'
 
 require 'json'
 
@@ -33,14 +35,14 @@ module Funktor
     @job_pusher_chain
   end
 
-  def self.configure_active_job_handler
+  def self.configure_work_queue_handler
     yield self
   end
 
-  def self.active_job_handler_middleware
-    @active_job_handler_chain ||= MiddlewareChain.new
-    yield @active_job_handler_chain if block_given?
-    @active_job_handler_chain
+  def self.work_queue_handler_middleware
+    @work_queue_handler_chain ||= MiddlewareChain.new
+    yield @work_queue_handler_chain if block_given?
+    @work_queue_handler_chain
   end
 
   # TODO - Maybe we don't need this either? Maybe this should be a super dumb thing that also
@@ -58,6 +60,18 @@ module Funktor
     @incoming_job_handler_chain
   end
 
+  # TODO - Does this actually make any sense? Should the JobActivator even know about
+  # jobs/classes? Maybe it should be super dumb and just push JSON blobs around?
+  def self.configure_job_activator
+    yield self
+  end
+
+  def self.job_activator_middleware
+    @job_activator_chain ||= MiddlewareChain.new
+    yield @job_activator_chain if block_given?
+    @job_activator_chain
+  end
+
   def self.parse_json(string)
     JSON.parse(string)
   end
@@ -95,13 +109,31 @@ module Funktor
 
     @logger = logger
   end
+
+  # We have a raw_logger that doesn't add timestamps and what not. This is used to publish
+  # CloudWatch metrics that can be used in dashboards.
+  def self.raw_logger
+    @raw_logger ||= Funktor::Logger.new($stdout, level: options[:log_level], formatter: proc {|severity, datetime, progname, msg|
+      "#{msg}\n"
+    })
+  end
+
+  def self.raw_logger=(raw_logger)
+    if raw_logger.nil?
+      self.raw_logger.level = Logger::FATAL
+      return self.raw_logger
+    end
+
+    @raw_logger = raw_logger
+  end
 end
 
-# TODO - Should we require this by default or let people opt in?
 # TODO - Is it a code smell that we need to include these at the bottom, after
 # the main Funktor module is defined?
-require 'funktor/middleware/metrics'
-require 'funktor/error_handler'
-require 'funktor/active_job_handler'
+#
+# TODO - Should we require metrics by default or let people opt in?
+require_relative 'funktor/middleware/metrics'
+require_relative 'funktor/error_handler'
+require_relative 'funktor/work_queue_handler'
 
-require 'funktor/rails' if defined?(::Rails::Engine)
+require_relative 'funktor/rails' if defined?(::Rails::Engine)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.6.2
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-03 00:00:00.000000000 Z
+date: 2021-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.37'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-dynamodb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.62'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.62'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Background processing in AWS Lambda.
 email:
 - jeremy@octolabs.com
@@ -120,6 +162,7 @@ files:
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".tool-versions"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -131,10 +174,53 @@ files:
 - bin/setup
 - exe/funktor
 - exe/funktor-deploy
+- funktor-testapp/.envrc
+- funktor-testapp/.gitignore
+- funktor-testapp/Gemfile
+- funktor-testapp/Gemfile.lock
+- funktor-testapp/app/services/job_flood.rb
+- funktor-testapp/app/workers/audit_worker.rb
+- funktor-testapp/app/workers/greetings_worker.rb
+- funktor-testapp/app/workers/hello_worker.rb
+- funktor-testapp/app/workers/single_thread_audit_worker.rb
+- funktor-testapp/deploy-dev.sh
+- funktor-testapp/funktor_config/boot.rb
+- funktor-testapp/funktor_config/environment.yml
+- funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml
+- funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml
+- funktor-testapp/funktor_config/function_definitions/job_activator.yml
+- funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
+- funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
+- funktor-testapp/funktor_config/funktor.yml
+- funktor-testapp/funktor_config/iam_permissions/activity_table.yml
+- funktor-testapp/funktor_config/iam_permissions/default_queue.yml
+- funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
+- funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
+- funktor-testapp/funktor_config/iam_permissions/jobs_table_secondary_index.yml
+- funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
+- funktor-testapp/funktor_config/iam_permissions/ssm.yml
+- funktor-testapp/funktor_config/package.yml
+- funktor-testapp/funktor_config/resources/activity_table.yml
+- funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml
+- funktor-testapp/funktor_config/resources/default_queue.yml
+- funktor-testapp/funktor_config/resources/incoming_job_queue.yml
+- funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml
+- funktor-testapp/funktor_config/resources/jobs_table.yml
+- funktor-testapp/funktor_config/resources/low_concurrency_queue.yml
+- funktor-testapp/funktor_config/ruby_layer.yml
+- funktor-testapp/funktor_init.yml
+- funktor-testapp/lambda_event_handlers/default_queue_handler.rb
+- funktor-testapp/lambda_event_handlers/incoming_job_handler.rb
+- funktor-testapp/lambda_event_handlers/job_activator.rb
+- funktor-testapp/lambda_event_handlers/low_concurrency_queue_handler.rb
+- funktor-testapp/lambda_event_handlers/random_job_generator.rb
+- funktor-testapp/package-lock.json
+- funktor-testapp/package.json
+- funktor-testapp/serverless.yml
 - funktor.gemspec
 - lib/active_job/queue_adapters/funktor_adapter.rb
 - lib/funktor.rb
-- lib/funktor/active_job_handler.rb
+- lib/funktor/activity_tracker.rb
 - lib/funktor/aws/sqs/event.rb
 - lib/funktor/aws/sqs/record.rb
 - lib/funktor/cli/application.rb
@@ -148,20 +234,27 @@ files:
 - lib/funktor/cli/templates/funktor_config/boot.rb
 - lib/funktor/cli/templates/funktor_config/environment.yml
 - lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml
+- lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml
 - lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
 - lib/funktor/cli/templates/funktor_config/funktor.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
 - lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
 - lib/funktor/cli/templates/funktor_config/package.yml
+- lib/funktor/cli/templates/funktor_config/resources/activity_table.yml
 - lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
 - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
 - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
+- lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml
 - lib/funktor/cli/templates/funktor_config/resources/work_queue.yml
 - lib/funktor/cli/templates/funktor_config/ruby_layer.yml
 - lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
 - lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
+- lib/funktor/cli/templates/lambda_event_handlers/job_activator.rb
 - lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/package.json
 - lib/funktor/cli/templates/serverless.yml
@@ -170,13 +263,26 @@ files:
 - lib/funktor/fake_job_queue.rb
 - lib/funktor/incoming_job_handler.rb
 - lib/funktor/job.rb
+- lib/funktor/job_activator.rb
 - lib/funktor/job_pusher.rb
 - lib/funktor/logger.rb
 - lib/funktor/middleware/metrics.rb
 - lib/funktor/middleware_chain.rb
 - lib/funktor/rails.rb
+- lib/funktor/shard_utils.rb
 - lib/funktor/testing.rb
 - lib/funktor/version.rb
+- lib/funktor/web.rb
+- lib/funktor/web/application.rb
+- lib/funktor/web/views/index.erb
+- lib/funktor/web/views/layout.erb
+- lib/funktor/web/views/processing.erb
+- lib/funktor/web/views/queued.erb
+- lib/funktor/web/views/retries.erb
+- lib/funktor/web/views/scheduled.erb
+- lib/funktor/web/views/stats.erb
+- lib/funktor/web/views/table_stats_with_buttons.erb
+- lib/funktor/work_queue_handler.rb
 - lib/funktor/worker.rb
 - lib/funktor/worker/funktor_options.rb
 homepage: https://github.com/Octo-Labs/funktor