funktor 0.5.0 → 0.6.3

data/lib/funktor/cli/templates/serverless.yml

@@ -26,6 +26,7 @@ provider:
   lambdaHashingVersion: 20201221
   environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
+  logRetentionInDays: 7
   iamRoleStatements:
     <%- all_iam_permissions.each do |iam_permission| -%>
     - ${file(<%= iam_permission %>)}

data/lib/funktor/incoming_job_handler.rb

@@ -30,6 +30,11 @@ module Funktor
         # TODO : This number should be configurable via ENV var
         if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
           Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
+          # We push to the jobs table first becauase the work queue handler will expect to be able
+          # to update the stats of a record that's already in the table.
+          # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
+          # work queue handler to be ok with a job that's not yet in the table?
+          push_to_jobs_table(job, "queued")
           push_to_work_queue(job)
           if job.is_retry?
             @tracker.track(:retryActivated, job)
@@ -38,7 +43,7 @@ module Funktor
           end
         else
           Funktor.logger.debug "pushing to jobs table for delay = #{job.delay}"
-          push_to_jobs_table(job)
+          push_to_jobs_table(job, nil)
           if job.is_retry?
             # do nothing for tracking
           else
@@ -49,18 +54,10 @@ module Funktor
       end
     end
 
-    def queue_for_job(job)
-      queue_name = job.queue || 'default'
-      queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
-      Funktor.logger.debug "queue_constant = #{queue_constant}"
-      Funktor.logger.debug "ENV value = #{ENV[queue_constant]}"
-      ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
-    end
-
     def push_to_work_queue(job)
       Funktor.logger.debug "job = #{job.to_json}"
       sqs_client.send_message({
-        queue_url: queue_for_job(job),
+        queue_url: job.work_queue_url,
         message_body: job.to_json,
         delay_seconds: job.delay
       })
@@ -70,16 +67,15 @@ module Funktor
       ENV['FUNKTOR_JOBS_TABLE']
     end
 
-    def push_to_jobs_table(job)
-      perform_at = (Time.now + job.delay).utc
+    def push_to_jobs_table(job, category = nil)
       resp = dynamodb_client.put_item({
         item: {
           payload: job.to_json,
           jobId: job.job_id,
-          performAt: perform_at.iso8601,
+          performAt: job.perform_at.iso8601,
           jobShard: job.shard,
-          dummy: "dummy",
-          category: job.is_retry? ? "retry" : "scheduled"
+          queueable: category.present? ? "false" : "true",
+          category: category || (job.is_retry? ? "retry" : "scheduled")
         },
         table_name: delayed_job_table
       })

data/lib/funktor/job.rb

@@ -1,5 +1,8 @@
+require_relative 'shard_utils'
+
 module Funktor
   class Job
+    include ShardUtils
     attr_accessor :job_string
     attr_accessor :job_data
     def initialize(job_string)
@@ -11,7 +14,15 @@ module Funktor
     end
 
     def queue
-      job_data["queue"]
+      job_data["queue"] || 'default'
+    end
+
+    def work_queue_url
+      queue_name = self.queue
+      queue_constant = "FUNKTOR_#{queue_name.underscore.upcase}_QUEUE"
+      Funktor.logger.debug "queue_constant = #{queue_constant}"
+      Funktor.logger.debug "ENV value = #{ENV[queue_constant]}"
+      ENV[queue_constant] || ENV['FUNKTOR_DEFAULT_QUEUE']
     end
 
     def worker_class_name
@@ -23,8 +34,7 @@ module Funktor
     end
 
     def shard
-      # TODO - Should the number of shards be configurable?
-      job_data["job_id"].hash % 64
+      calculate_shard(job_data["job_id"])
     end
 
     def worker_params
@@ -43,12 +53,43 @@ module Funktor
       job_data["retries"] = retries
     end
 
+    def perform_at
+      if job_data["perform_at"].present?
+        job_data["perform_at"].is_a?(Time) ? job_data["perform_at"] : Time.parse(job_data["perform_at"])
+      else
+        Time.now.utc
+      end
+    end
+
     def delay
-      job_data["delay"] || 0
+      delay = (perform_at - Time.now.utc).to_i
+      if delay < 0
+        delay = 0
+      end
+      return delay
     end
 
     def delay=(delay)
-      job_data["delay"] = delay
+      job_data["perform_at"] = Time.now.utc + delay
+    end
+
+    def error_class
+      job_data["error_class"]
+    end
+
+    def error_message
+      job_data["error_message"]
+    end
+
+    def error_backtrace
+      job_data["error_backtrace"].present? ? Funktor.parse_json(job_data["error_backtrace"]) : []
+    end
+
+    def error=(error)
+      # TODO We should maybe compress this?
+      job_data["error_class"] = error.class.name
+      job_data["error_message"] = error.message
+      job_data["error_backtrace"] = Funktor.dump_json(error.backtrace)
     end
 
     def execute
@@ -84,6 +125,10 @@ module Funktor
 
     def retry_queue_url
       worker_class&.custom_queue_url || ENV['FUNKTOR_INCOMING_JOB_QUEUE']
+    rescue NameError, TypeError
+      # In the web ui we may not have access to the the worker classes
+      # TODO : We should mayb handle this differently somehow? This just feels a bit icky...
+      ENV['FUNKTOR_INCOMING_JOB_QUEUE']
     end
   end
 end

data/lib/funktor/job_activator.rb

@@ -16,23 +16,25 @@ module Funktor
       @sqs_client ||= ::Aws::SQS::Client.new
     end
 
-    def active_job_queue
-      ENV['FUNKTOR_ACTIVE_JOB_QUEUE']
-    end
-
     def delayed_job_table
       ENV['FUNKTOR_JOBS_TABLE']
     end
 
     def jobs_to_activate
-      target_time = (Time.now + 90).utc
+      # TODO : The lookahead time here should be configurable
+      # If this doesn't match the setting in the IncomingJobHandler some jobs
+      # might be activated and then immediately re-scheduled instead of being
+      # queued, which leads to kind of confusing stats for the "incoming" stat.
+      # (Come to think of it, the incoming stat is kind of confusting anyway since
+      # it reflects retries and scheduled jobs activations...)
+      target_time = (Time.now + 60).utc
       query_params = {
         expression_attribute_values: {
-          ":dummy" => "dummy",
+          ":queueable" => "true",
           ":targetTime" => target_time.iso8601
         },
-        key_condition_expression: "dummy = :dummy AND performAt < :targetTime",
-        projection_expression: "payload, performAt, jobId, jobShard",
+        key_condition_expression: "queueable = :queueable AND performAt < :targetTime",
+        projection_expression: "jobId, jobShard, category",
         table_name: delayed_job_table,
         index_name: "performAtIndex"
       }
@@ -49,38 +51,62 @@ module Funktor
     end
 
     def handle_item(item)
-      job = Funktor::Job.new(item["payload"])
-      Funktor.logger.debug "we created a job from payload"
-      Funktor.logger.debug item["payload"]
-      delay = (Time.parse(item["performAt"]) - Time.now.utc).to_i
-      if delay < 0
-        delay = 0
-      end
+      job_shard = item["jobShard"]
+      job_id = item["jobId"]
+      current_category = item["category"]
       Funktor.logger.debug "jobShard = #{item['jobShard']}"
       Funktor.logger.debug "jobId = #{item['jobId']}"
-      # First we delete the item from Dynamo to be sure that another scheduler hasn't gotten to it,
-      # and if that works then send to SQS. This is basically how Sidekiq scheduler works.
-      response = dynamodb_client.delete_item({
+      Funktor.logger.debug "current_category = #{current_category}"
+      activate_job(job_shard, job_id, current_category)
+    end
+
+    def activate_job(job_shard, job_id, current_category, queue_immediately = false)
+      # First we conditionally update the item in  Dynamo to be sure that another scheduler hasn't gotten
+      # to it, and if that works then send to SQS. This is basically how Sidekiq scheduler works.
+      response = dynamodb_client.update_item({
         key: {
-          "jobShard" => item["jobShard"],
-          "jobId" => item["jobId"]
+          "jobShard" => job_shard,
+          "jobId" => job_id
+        },
+        update_expression: "SET category = :category, queueable = :queueable",
+        condition_expression: "category = :current_category",
+        expression_attribute_values: {
+          ":current_category" => current_category,
+          ":queueable" => "false",
+          ":category" => "queued"
         },
         table_name: delayed_job_table,
         return_values: "ALL_OLD"
       })
-      if response.attributes # this means the record was still there
+      if response.attributes # this means the record was still there in the state we expected
+        Funktor.logger.debug "response.attributes ====== "
+        Funktor.logger.debug response.attributes
+        job = Funktor::Job.new(response.attributes["payload"])
+        Funktor.logger.debug "we created a job from payload"
+        Funktor.logger.debug response.attributes["payload"]
+        Funktor.logger.debug "queueing to #{job.retry_queue_url}"
+        if queue_immediately
+          job.delay = 0
+        end
         sqs_client.send_message({
-          # TODO : How to get this URL...
-          queue_url: queue_for_job(job),
-          message_body: item["payload"],
-          delay_seconds: delay
+          queue_url: job.retry_queue_url,
+          message_body: job.to_json
+          #delay_seconds: job.delay
         })
         if job.is_retry?
-          @tracker.track(:retryActivated, job)
+          # We don't track here because we send stuff back to the incoming job queue and we track the
+          # :retryActivated even there.
+          # TODO - Once we're sure this is all working right we can delete the commented out line.
+          #@tracker.track(:retryActivated, job)
         else
           @tracker.track(:scheduledJobActivated, job)
         end
       end
+    rescue ::Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
+      # This means that a different instance of the JobActivator (or someone doing stuff in the web UI)
+      # got to the job first.
+      Funktor.logger.debug "#{e.to_s} : #{e.message}"
+      Funktor.logger.debug e.backtrace.join("\n")
     end
 
     def call(event:, context:)

data/lib/funktor/shard_utils.rb

@@ -0,0 +1,6 @@
+module ShardUtils
+  def calculate_shard(job_id)
+    # TODO - Should the number of shards be configurable?
+    job_id.sum % 64
+  end
+end

data/lib/funktor/testing.rb

@@ -1,4 +1,5 @@
 require 'funktor/worker'
+require 'funktor/job_pusher'
 require 'funktor/fake_job_queue'
 
 module Funktor

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.5.0"
+  VERSION = "0.6.3"
 end

data/lib/funktor/web/application.rb

@@ -0,0 +1,139 @@
+require 'sinatra/base'
+require 'aws-sdk-dynamodb'
+require_relative '../../funktor'
+require_relative '../../funktor/shard_utils'
+require_relative '../../funktor/activity_tracker'
+
+module Funktor
+  module Web
+    class Application < Sinatra::Base
+      include ShardUtils
+
+      get '/' do
+        erb :index, layout: :layout, locals: {
+          activity_data: get_activity_data
+        }
+      end
+
+      get '/scheduled' do
+        erb :scheduled, layout: :layout, locals: {
+          activity_data: get_activity_data,
+          jobs: get_jobs('scheduled')
+        }
+      end
+
+      get '/retries' do
+        erb :retries, layout: :layout, locals: {
+          activity_data: get_activity_data,
+          jobs: get_jobs('retry')
+        }
+      end
+
+      get '/queued' do
+        erb :queued, layout: :layout, locals: {
+          activity_data: get_activity_data,
+          jobs: get_jobs('queued')
+        }
+      end
+
+      get '/processing' do
+        erb :processing, layout: :layout, locals: {
+          activity_data: get_activity_data,
+          jobs: get_jobs('processing')
+        }
+      end
+
+      post '/update_jobs' do
+        job_ids = params[:job_id]
+        if job_ids.is_a?(String)
+          job_ids = [job_ids]
+        end
+        job_ids ||= []
+        puts "params[:submit] = #{params[:submit]}"
+        puts "job_ids = #{job_ids}"
+        puts "params[:source] = #{params[:source]}"
+        if params[:submit] == "Delete Selected Jobs"
+          delete_jobs(job_ids, params[:source])
+        elsif params[:submit] == "Queue Selected Jobs"
+          queue_jobs(job_ids, params[:source])
+        end
+        redirect request.referrer
+      end
+
+      def get_jobs(category)
+        "Jobs of type #{category}"
+        query_params = {
+          expression_attribute_values: {
+            ":category" => category
+          },
+          key_condition_expression: "category = :category",
+          projection_expression: "payload, performAt, jobId, jobShard",
+          table_name: ENV['FUNKTOR_JOBS_TABLE'],
+          index_name: "categoryIndex"
+        }
+        resp = dynamodb_client.query(query_params)
+        @items = resp.items
+        @jobs = @items.map{ |item| Funktor::Job.new(item["payload"]) }
+        return @jobs
+      end
+
+      def get_activity_data
+        query_params = {
+          expression_attribute_values: {
+            ":category" => "stat"
+          },
+          key_condition_expression: "category = :category",
+          projection_expression: "statName, stat_value",
+          table_name: ENV['FUNKTOR_ACTIVITY_TABLE']
+        }
+        resp = dynamodb_client.query(query_params)
+        @activity_stats = {}
+        resp.items.each do |item|
+          @activity_stats[item["statName"]] = item["stat_value"].to_i
+        end
+        return @activity_stats
+      end
+
+      def queue_jobs(job_ids, source)
+        job_activator = Funktor::JobActivator.new
+        job_ids.each do |job_id|
+          job_shard = calculate_shard(job_id)
+          job_activator.activate_job(job_shard, job_id, source, true)
+        end
+      end
+
+      def delete_jobs(job_ids, source)
+        @tracker = Funktor::ActivityTracker.new
+        job_ids.each do |job_id|
+          delete_single_job(job_id, source)
+        end
+      end
+
+      def delete_single_job(job_id, source)
+        response = dynamodb_client.delete_item({
+          key: {
+            "jobShard" => calculate_shard(job_id),
+            "jobId" => job_id
+          },
+          table_name: ENV['FUNKTOR_JOBS_TABLE'],
+          return_values: "ALL_OLD"
+        })
+        if response.attributes # this means the record was still there
+          if source == "scheduled"
+            @tracker.track(:scheduledJobDeleted, nil)
+          elsif source == "retry"
+            @tracker.track(:retryDeleted, nil)
+          end
+        end
+      end
+
+      def dynamodb_client
+        @dynamodb_client ||= ::Aws::DynamoDB::Client.new
+      end
+
+      # start the server if ruby file executed directly
+      run! if app_file == $0
+    end
+  end
+end
+

data/lib/funktor/web/views/index.erb

@@ -0,0 +1,3 @@
+<p>Hellerrrrrrrrr!</p>
+<p>Activity data = <%= activity_data %></p>
+

data/lib/funktor/web/views/layout.erb

@@ -0,0 +1,58 @@
+<html>
+  <head>
+    <link rel="stylesheet" href="https://unpkg.com/@picocss/pico@latest/css/pico.min.css">
+    <style type="text/css" media="screen">
+      /* Green Light scheme (Default) */
+      /* Can be forced with data-theme="light" */
+      [data-theme="light"],
+      :root:not([data-theme="dark"]) {
+        --primary: #43a047;
+        --primary-hover: #388e3c;
+        --primary-focus: rgba(67, 160, 71, 0.125);
+        --primary-inverse: #FFF;
+      }
+
+      /* Green Dark scheme (Auto) */
+      /* Automatically enabled if user has Dark mode enabled */
+      @media only screen and (prefers-color-scheme: dark) {
+        :root:not([data-theme="light"]) {
+          --primary: #43a047;
+          --primary-hover: #4caf50;
+          --primary-focus: rgba(67, 160, 71, 0.25);
+          --primary-inverse: #FFF;
+        }
+      }
+
+      /* Green Dark scheme (Forced) */
+      /* Enabled if forced with data-theme="dark" */
+      [data-theme="dark"] {
+        --primary: #43a047;
+        --primary-hover: #4caf50;
+        --primary-focus: rgba(67, 160, 71, 0.25);
+        --primary-inverse: #FFF;
+      }
+
+      /* Green (Common styles) */
+      :root {
+        --form-element-active-border-color: var(--primary);
+        --form-element-focus-color: var(--primary-focus);
+        --switch-color: var(--primary-inverse);
+        --switch-checked-background-color: var(--primary);
+      }
+
+      /* custom stuff for the funktor dashboard */
+      table.header h5{
+        margin-bottom: 0;
+      }
+      body > main{
+        padding: 0;
+      }
+    </style>
+  </head>
+  <body>
+    <%= erb :stats %>
+    <main class="container-fluid">
+      <%= yield %>
+    </main>
+  </body>
+</html>