funktor 0.5.0 → 0.6.3

data/lib/funktor/web/views/processing.erb

@@ -0,0 +1,29 @@
+<h1>Processing</h1>
+<p>Showing <%= jobs.count %> of <%= @activity_stats["processing"] %></p>
+<table role="grid">
+  <tr>
+    <td>ID</td>
+    <td>Perform at</td>
+    <td>Worker</td>
+    <td>Queue</td>
+    <td>Params</td>
+    <td>Attempts</td>
+    <td>Last error</td>
+  </tr>
+  <% jobs.each do |job| %>
+    <tr>
+      <td><%= job.job_id %></td>
+      <td><%= job.perform_at %></td>
+      <td><%= job.worker_class_name %></td>
+      <td><%= job.queue %></td>
+      <td><%= job.worker_params %></td>
+      <td><%= job.retries %></td>
+      <td>
+        <%= job.error_class %> - <%= job.error_message %>
+        <br/>
+        <%= job.error_backtrace.first %>
+      </td>
+    </tr>
+  <% end %>
+</table>
+

data/lib/funktor/web/views/queued.erb

@@ -0,0 +1,29 @@
+<h1>Queued</h1>
+<p>Showing <%= jobs.count %> of <%= @activity_stats["queued"] %></p>
+<table role="grid">
+  <tr>
+    <td>ID</td>
+    <td>Perform at</td>
+    <td>Worker</td>
+    <td>Queue</td>
+    <td>Params</td>
+    <td>Attempts</td>
+    <td>Last error</td>
+  </tr>
+  <% jobs.each do |job| %>
+    <tr>
+      <td><%= job.job_id %></td>
+      <td><%= job.perform_at %></td>
+      <td><%= job.worker_class_name %></td>
+      <td><%= job.queue %></td>
+      <td><%= job.worker_params %></td>
+      <td><%= job.retries %></td>
+      <td>
+        <%= job.error_class %> - <%= job.error_message %>
+        <br/>
+        <%= job.error_backtrace.first %>
+      </td>
+    </tr>
+  <% end %>
+</table>
+

data/lib/funktor/web/views/retries.erb

@@ -0,0 +1,35 @@
+<h1>Retries</h1>
+<form action="/update_jobs" method="post" accept-charset="utf-8">
+  <input type="hidden" name="source" value="retry" />
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
+  <table role="grid">
+    <tr>
+      <td>&nbsp;</td>
+      <td>ID</td>
+      <td>Perform at</td>
+      <td>Worker</td>
+      <td>Queue</td>
+      <td>Params</td>
+      <td>Attempts</td>
+      <td>Last error</td>
+    </tr>
+    <% jobs.each do |job| %>
+      <tr>
+        <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
+        <td><%= job.job_id %></td>
+        <td><%= job.perform_at %></td>
+        <td><%= job.worker_class_name %></td>
+        <td><%= job.queue %></td>
+        <td><%= job.worker_params %></td>
+        <td><%= job.retries %></td>
+        <td>
+          <%= job.error_class %> - <%= job.error_message %>
+          <br/>
+          <%= job.error_backtrace.first %>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
+</form>
+

data/lib/funktor/web/views/scheduled.erb

@@ -0,0 +1,26 @@
+<h1>Scheduled</h1>
+<form action="/update_jobs" method="post" accept-charset="utf-8">
+  <input type="hidden" name="source" value="scheduled" />
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
+  <table role="grid">
+    <tr>
+      <td>&nbsp;</td>
+      <td>ID</td>
+      <td>Perform at</td>
+      <td>Worker</td>
+      <td>Queue</td>
+      <td>Params</td>
+    </tr>
+    <% jobs.each do |job| %>
+      <tr>
+        <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
+        <td><%= job.job_id %></td>
+        <td><%= job.perform_at %></td>
+        <td><%= job.worker_class_name %></td>
+        <td><%= job.queue %></td>
+        <td><%= job.worker_params %></td>
+      </tr>
+    <% end %>
+  </table>
+  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
+</form>

data/lib/funktor/web/views/stats.erb

@@ -0,0 +1,9 @@
+<table class="header">
+  <tr>
+    <td><h5>Funktor</h5></td>
+    <% @activity_stats.each_pair do |stat, value| %>
+      <td><a href="./<%= stat %>"><%= stat %>: <%= value %></a></td>
+    <% end %>
+    <td>Current time UTC: <%= Time.now.utc %></td>
+  </tr>
+</table>

data/lib/funktor/web/views/table_stats_with_buttons.erb

@@ -0,0 +1,11 @@
+<div class="grid">
+  <div>
+    Showing <%= jobs.count %> of <%= @activity_stats[stat_category] %>
+  </div>
+  <div>
+    <input type="submit" name="submit" value="Delete Selected Jobs" />
+  </div>
+  <div>
+    <input type="submit" name="submit" value="Queue Selected Jobs" />
+  </div>
+</div>

data/lib/funktor/web.rb

@@ -0,0 +1 @@
+require 'funktor/web/application'

data/lib/funktor/work_queue_handler.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-sqs'
+require 'aws-sdk-dynamodb'
 
 module Funktor
   class WorkQueueHandler
@@ -18,6 +19,10 @@ module Funktor
       end
     end
 
+    def dynamodb_client
+      @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+    end
+
     def sqs_client
       @sqs_client ||= ::Aws::SQS::Client.new
     end
@@ -25,20 +30,25 @@ module Funktor
     def dispatch(job)
       begin
         @tracker.track(:processingStarted, job)
+        update_job_category(job, "processing")
         Funktor.work_queue_handler_middleware.invoke(job) do
           job.execute
         end
         @processed_counter.incr(job)
         @tracker.track(:processingComplete, job)
+        delete_job_from_dynamodb(job)
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
         handle_error(e, job)
         @failed_counter.incr(job)
+        job.error = e
         if job.can_retry
           @tracker.track(:retrying, job)
+          update_job_category(job, "retry")
           trigger_retry(job)
         else
           @tracker.track(:bailingOut, job)
+          update_job_category(job, "dead")
           Funktor.logger.error "We retried max times. We're bailing on this one."
           Funktor.logger.error job.to_json
         end
@@ -56,5 +66,36 @@ module Funktor
       })
     end
 
+    def delayed_job_table
+      ENV['FUNKTOR_JOBS_TABLE']
+    end
+
+    def update_job_category(job, category)
+      dynamodb_client.update_item({
+        key: {
+          "jobShard" => job.shard,
+          "jobId" => job.job_id
+        },
+        table_name: delayed_job_table,
+        update_expression: "SET category = :category, queueable = :queueable",
+        expression_attribute_values: {
+          ":queueable" => "false",
+          ":category" => category
+        },
+        return_values: "ALL_OLD"
+      })
+    end
+
+    def delete_job_from_dynamodb(job)
+      dynamodb_client.delete_item({
+        key: {
+          "jobShard" => job.shard,
+          "jobId" => job.job_id
+        },
+        table_name: delayed_job_table,
+        return_values: "ALL_OLD"
+      })
+    end
+
   end
 end

data/lib/funktor/worker/funktor_options.rb

@@ -1,10 +1,12 @@
+require 'active_support/core_ext/class/attribute'
+
 module Funktor
   module Worker
     module FunktorOptions
       def self.included(base)
         base.extend ClassMethods
         base.class_eval do
-          cattr_accessor :funktor_options_hash
+          class_attribute :funktor_options_hash
         end
       end
       module ClassMethods

data/lib/funktor/worker.rb

@@ -18,30 +18,27 @@ module Funktor::Worker
       self.perform_in(0, *worker_params)
     end
 
-    def perform_at(time, *worker_params)
-      delay = (time.utc - Time.now.utc).round
-      if delay < 0
-        delay = 0
-      end
-      self.perform_in(delay, *worker_params)
+    def perform_at(job_time, *worker_params)
+      self.push(job_time, *worker_params)
     end
 
     def perform_in(delay, *worker_params)
-      self.push(delay, *worker_params)
+      job_time = Time.now.utc + delay
+      self.perform_at(job_time, *worker_params)
     end
 
-    def push(delay, *worker_params)
-      payload = build_job_payload(delay, *worker_params)
+    def push(job_time, *worker_params)
+      payload = build_job_payload(job_time, *worker_params)
       Funktor.job_pusher.push(payload)
     end
 
-    def build_job_payload(delay, *worker_params)
+    def build_job_payload(job_time, *worker_params)
       {
         worker: self.name,
         worker_params: worker_params,
         queue: self.work_queue,
         incoming_job_queue_url: self.queue_url,
-        delay: delay,
+        perform_at: job_time.utc,
         funktor_options: get_funktor_options
       }
     end

data/lib/funktor.rb

@@ -1,15 +1,15 @@
-require "funktor/version"
-require 'funktor/aws/sqs/event'
-require 'funktor/aws/sqs/record'
-require 'funktor/counter'
-require 'funktor/job'
-require 'funktor/job_pusher'
-require 'funktor/logger'
-require 'funktor/worker'
-require 'funktor/middleware_chain'
-require 'funktor/incoming_job_handler'
-require 'funktor/job_activator'
-require 'funktor/activity_tracker'
+require_relative "funktor/version"
+require_relative 'funktor/aws/sqs/event'
+require_relative 'funktor/aws/sqs/record'
+require_relative 'funktor/counter'
+require_relative 'funktor/job'
+require_relative 'funktor/job_pusher'
+require_relative 'funktor/logger'
+require_relative 'funktor/worker'
+require_relative 'funktor/middleware_chain'
+require_relative 'funktor/incoming_job_handler'
+require_relative 'funktor/job_activator'
+require_relative 'funktor/activity_tracker'
 
 require 'json'
 
@@ -132,8 +132,8 @@ end
 # the main Funktor module is defined?
 #
 # TODO - Should we require metrics by default or let people opt in?
-require 'funktor/middleware/metrics'
-require 'funktor/error_handler'
-require 'funktor/work_queue_handler'
+require_relative 'funktor/middleware/metrics'
+require_relative 'funktor/error_handler'
+require_relative 'funktor/work_queue_handler'
 
-require 'funktor/rails' if defined?(::Rails::Engine)
+require_relative 'funktor/rails' if defined?(::Rails::Engine)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.3
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-12 00:00:00.000000000 Z
+date: 2021-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -122,6 +122,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Background processing in AWS Lambda.
 email:
 - jeremy@octolabs.com
@@ -154,21 +182,22 @@ files:
 - funktor-testapp/app/workers/audit_worker.rb
 - funktor-testapp/app/workers/greetings_worker.rb
 - funktor-testapp/app/workers/hello_worker.rb
+- funktor-testapp/app/workers/single_thread_audit_worker.rb
 - funktor-testapp/deploy-dev.sh
 - funktor-testapp/funktor_config/boot.rb
 - funktor-testapp/funktor_config/environment.yml
 - funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml
 - funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml
 - funktor-testapp/funktor_config/function_definitions/job_activator.yml
+- funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
 - funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
-- funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml
 - funktor-testapp/funktor_config/funktor.yml
 - funktor-testapp/funktor_config/iam_permissions/activity_table.yml
 - funktor-testapp/funktor_config/iam_permissions/default_queue.yml
 - funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
 - funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
 - funktor-testapp/funktor_config/iam_permissions/jobs_table_secondary_index.yml
-- funktor-testapp/funktor_config/iam_permissions/single_thread_queue.yml
+- funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
 - funktor-testapp/funktor_config/iam_permissions/ssm.yml
 - funktor-testapp/funktor_config/package.yml
 - funktor-testapp/funktor_config/resources/activity_table.yml
@@ -177,14 +206,14 @@ files:
 - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
 - funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml
 - funktor-testapp/funktor_config/resources/jobs_table.yml
-- funktor-testapp/funktor_config/resources/single_thread_queue.yml
+- funktor-testapp/funktor_config/resources/low_concurrency_queue.yml
 - funktor-testapp/funktor_config/ruby_layer.yml
 - funktor-testapp/funktor_init.yml
 - funktor-testapp/lambda_event_handlers/default_queue_handler.rb
 - funktor-testapp/lambda_event_handlers/incoming_job_handler.rb
 - funktor-testapp/lambda_event_handlers/job_activator.rb
+- funktor-testapp/lambda_event_handlers/low_concurrency_queue_handler.rb
 - funktor-testapp/lambda_event_handlers/random_job_generator.rb
-- funktor-testapp/lambda_event_handlers/single_thread_queue_handler.rb
 - funktor-testapp/package-lock.json
 - funktor-testapp/package.json
 - funktor-testapp/serverless.yml
@@ -240,8 +269,19 @@ files:
 - lib/funktor/middleware/metrics.rb
 - lib/funktor/middleware_chain.rb
 - lib/funktor/rails.rb
+- lib/funktor/shard_utils.rb
 - lib/funktor/testing.rb
 - lib/funktor/version.rb
+- lib/funktor/web.rb
+- lib/funktor/web/application.rb
+- lib/funktor/web/views/index.erb
+- lib/funktor/web/views/layout.erb
+- lib/funktor/web/views/processing.erb
+- lib/funktor/web/views/queued.erb
+- lib/funktor/web/views/retries.erb
+- lib/funktor/web/views/scheduled.erb
+- lib/funktor/web/views/stats.erb
+- lib/funktor/web/views/table_stats_with_buttons.erb
 - lib/funktor/work_queue_handler.rb
 - lib/funktor/worker.rb
 - lib/funktor/worker/funktor_options.rb

data/funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml

@@ -1,11 +0,0 @@
-handler: lambda_event_handlers/single_thread_queue_handler.call
-timeout: ${self:custom.funktor.SingleThreadQueueHandler.timeout, 900}
-reservedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.reservedConcurrency, null}
-provisionedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.provisionedConcurrency, null}
-memorySize: ${self:custom.funktor.SingleThreadQueueHandler.memorySize, 256}
-events:
-  - sqs:
-      arn:
-        Fn::GetAtt:
-          - SingleThreadQueue
-          - Arn