crono_trigger 0.3.2 → 0.3.4

data/lib/crono_trigger/schedulable.rb

@@ -11,29 +11,37 @@ module CronoTrigger
     DEFAULT_RETRY_INTERVAL = 4
     DEFAULT_EXECUTE_LOCK_TIMEOUT = 600
 
+    class NoRestrictedUnlockError < StandardError; end
+
+    @included_by = []
+
+    def self.included_by
+      @included_by
+    end
+
     class AbortExecution < StandardError; end
 
     extend ActiveSupport::Concern
     include ActiveSupport::Callbacks
 
     included do
+      CronoTrigger::Schedulable.included_by << self
       class_attribute :crono_trigger_options, :executable_conditions
       self.crono_trigger_options ||= {}
       self.executable_conditions ||= []
 
       define_model_callbacks :execute
 
-      scope :executables, ->(from: Time.current, primary_key_offset: nil, limit: 1000) do
+      scope :executables, ->(from: Time.current, limit: CronoTrigger.config.executor_thread * 3 || 100, including_locked: false) do
         t = arel_table
 
         rel = where(t[crono_trigger_column_name(:next_execute_at)].lteq(from))
-          .where(t[crono_trigger_column_name(:execute_lock)].lteq(from.to_i - execute_lock_timeout))
+        rel = rel.where(t[crono_trigger_column_name(:execute_lock)].lteq(from.to_i - execute_lock_timeout)) unless including_locked
 
         rel = rel.where(t[crono_trigger_column_name(:started_at)].lteq(from)) if column_names.include?(crono_trigger_column_name(:started_at))
         rel = rel.where(t[crono_trigger_column_name(:finished_at)].gt(from).or(t[crono_trigger_column_name(:finished_at)].eq(nil)))  if column_names.include?(crono_trigger_column_name(:finished_at))
-        rel = rel.where(t[primary_key].gt(primary_key_offset)) if primary_key_offset
 
-        rel = rel.order(Arel.sql("#{quoted_table_name}.#{quoted_primary_key} ASC")).limit(limit)
+        rel = rel.order(crono_trigger_column_name(:next_execute_at) => :asc).limit(limit)
 
         rel = executable_conditions.reduce(rel) do |merged, pr|
           if pr.arity == 0
@@ -52,12 +60,15 @@ module CronoTrigger
     end
 
     module ClassMethods
-      def executables_with_lock(primary_key_offset: nil, limit: 1000)
+      def executables_with_lock(limit: CronoTrigger.config.executor_thread * 3 || 100)
         records = nil
         transaction do
-          records = executables(primary_key_offset: primary_key_offset, limit: limit).lock.to_a
+          records = executables(limit: limit).lock.to_a
           unless records.empty?
-            where(id: records).update_all(crono_trigger_column_name(:execute_lock) => Time.current.to_i)
+            where(id: records).update_all(
+              crono_trigger_column_name(:execute_lock) => Time.current.to_i,
+              crono_trigger_column_name(:locked_by) => CronoTrigger.config.worker_id
+            )
           end
           records
         end
@@ -71,6 +82,18 @@ module CronoTrigger
         (crono_trigger_options[:execute_lock_timeout] || DEFAULT_EXECUTE_LOCK_TIMEOUT)
       end
 
+      def crono_trigger_unlock_all!
+        wheres = all.where_values_hash
+        if wheres.empty?
+          raise NoRestrictedUnlockError, "Need `where` filter at least one, because this method is danger"
+        else
+          update_all(
+            crono_trigger_column_name(:execute_lock) => 0,
+            crono_trigger_column_name(:locked_by) => nil,
+          )
+        end
+      end
+
       private
 
       def add_executable_conditions(pr)
@@ -137,7 +160,11 @@ module CronoTrigger
 
       now = Time.current
       wait = crono_trigger_options[:exponential_backoff] ? retry_interval * [2 * (retry_count - 1), 1].max : retry_interval
-      attributes = {crono_trigger_column_name(:next_execute_at) => now + wait, crono_trigger_column_name(:execute_lock) => 0}
+      attributes = {
+        crono_trigger_column_name(:next_execute_at) => now + wait,
+        crono_trigger_column_name(:execute_lock) => 0,
+        crono_trigger_column_name(:locked_by) => nil,
+      }
 
       if self.class.column_names.include?("retry_count")
         attributes.merge!(retry_count: retry_count.to_i + 1)
@@ -149,7 +176,11 @@ module CronoTrigger
     def reset!(update_last_executed_at = true)
       logger.info "Reset execution schedule #{self.class}-#{id}" if logger
 
-      attributes = {crono_trigger_column_name(:next_execute_at) => calculate_next_execute_at, crono_trigger_column_name(:execute_lock) => 0}
+      attributes = {
+        crono_trigger_column_name(:next_execute_at) => calculate_next_execute_at,
+        crono_trigger_column_name(:execute_lock) => 0,
+        crono_trigger_column_name(:locked_by) => nil,
+      }
 
       if update_last_executed_at && self.class.column_names.include?(crono_trigger_column_name(:last_executed_at))
         attributes.merge!(crono_trigger_column_name(:last_executed_at) => Time.current)
@@ -162,18 +193,39 @@ module CronoTrigger
       update_columns(attributes)
     end
 
-    def assume_executing?
-      execute_lock_timeout = self.class.execute_lock_timeout
-      locking? &&
-        self[crono_trigger_column_name(:execute_lock)] + execute_lock_timeout >= Time.now.to_i
+    def crono_trigger_unlock!
+      update_columns(
+        crono_trigger_column_name(:execute_lock) => 0,
+        crono_trigger_column_name(:locked_by) => nil,
+      )
     end
 
-    def locking?
-      self[crono_trigger_column_name(:execute_lock)] > 0
+    def crono_trigger_status
+      case
+      when locking?
+        :locked
+      when waiting?
+        :waiting
+      when not_scheduled?
+        :not_scheduled
+      end
+    end
+
+    def waiting?
+      !!self[crono_trigger_column_name(:next_execute_at)]
+    end
+
+    def not_scheduled?
+      self[crono_trigger_column_name(:next_execute_at)].nil? && last_executed_at.nil?
+    end
+
+    def locking?(at: Time.now)
+      self[crono_trigger_column_name(:execute_lock)] > 0 && 
+        self[crono_trigger_column_name(:execute_lock)] > at.to_i - self.class.execute_lock_timeout
     end
 
-    def idling?
-      !locking?
+    def assume_executing?
+      locking?
     end
 
     def crono_trigger_column_name(name)

data/lib/crono_trigger/version.rb

@@ -1,3 +1,3 @@
 module CronoTrigger
-  VERSION = "0.3.2"
+  VERSION = "0.3.4"
 end

data/lib/crono_trigger/web.rb

@@ -0,0 +1,163 @@
+require "crono_trigger"
+require "sinatra/base"
+require "rack/contrib/post_body_content_type_parser"
+
+module CronoTrigger
+  class Web < Sinatra::Application
+    use Rack::PostBodyContentTypeParser
+
+    set :root, File.expand_path("../../../web", __FILE__)
+    set :public_folder, Proc.new { File.join(root, "public") }
+    set :views, proc { File.join(root, "views") }
+
+    get "/" do
+      redirect to("/workers")
+    end
+
+    get "/workers.:format" do
+      if params[:format] == "json"
+        content_type :json
+        @workers = CronoTrigger::Models::Worker.alive_workers
+        Oj.dump({
+          records: @workers,
+        }, mode: :compat)
+      else
+        raise "unknown format"
+      end
+    end
+
+    get "/workers" do
+      erb :index
+    end
+
+    post "/signals" do
+      worker_id = params[:worker_id]
+      sig = params[:signal]
+      if worker_id && sig
+        if CronoTrigger::Models::Signal.send_signal(sig, worker_id)
+          status 200
+          body ""
+        else
+          status 422
+          Oj.dump({error: "#{sig} signal is not supported"}, mode: :compat)
+        end
+      else
+        status 422
+        Oj.dump({error: "Must set worker_id and signal"}, mode: :compat)
+      end
+    end
+
+    get "/signals.:format" do
+      if params[:format] == "json"
+        content_type :json
+        @signals = CronoTrigger::Models::Signal.order(sent_at: :desc).limit(30)
+        Oj.dump({
+          records: @signals,
+        }, mode: :compat)
+      else
+        raise "unknown format"
+      end
+    end
+
+    get "/signals" do
+      erb :index
+    end
+
+    post "/models/:name/:id/unlock" do
+      model_class = CronoTrigger::Schedulable.included_by.find { |c| c.name == params[:name] }
+      if model_class
+        model_class.find(params[:id]).crono_trigger_unlock!
+        status 200
+        body ""
+      else
+        status 404
+        "Model Class is not found"
+      end
+    end
+
+    post "/models/:name/:id/reset" do
+      model_class = CronoTrigger::Schedulable.included_by.find { |c| c.name == params[:name] }
+      if model_class
+        model_class.find(params[:id]).reset!
+        status 200
+        body ""
+      else
+        status 404
+        "Model Class is not found"
+      end
+    end
+
+    post "/models/:name/:id/retry" do
+      model_class = CronoTrigger::Schedulable.included_by.find { |c| c.name == params[:name] }
+      if model_class
+        model_class.find(params[:id]).retry!
+        status 200
+        body ""
+      else
+        status 404
+        "Model Class is not found"
+      end
+    end
+
+    get "/models/:name.:format" do
+      if params[:format] == "json"
+        content_type :json
+        model_class = CronoTrigger::Schedulable.included_by.find { |c| c.name == params[:name] }
+        if model_class
+          after_minute = params[:after] ? Integer(params[:after]) : 10
+          @scheduled_records = model_class.executables(from: Time.now.since(after_minute.minutes), limit: 100, including_locked: true).reorder(next_execute_at: :desc)
+          @scheduled_records.where!(locked_by: params[:worker_id]) if params[:worker_id]
+          now = Time.now
+          records = @scheduled_records.map do |r|
+            {
+              "crono_trigger_status" => r.crono_trigger_status,
+              "id" => r.id,
+              "cron" => r[r.crono_trigger_column_name(:cron)],
+              "next_execute_at" => r[r.crono_trigger_column_name(:next_execute_at)],
+              "last_executed_at" => r[r.crono_trigger_column_name(:last_executed_at)],
+              "timezone" => r[r.crono_trigger_column_name(:timezone)],
+              "execute_lock" => r[r.crono_trigger_column_name(:execute_lock)],
+              "locked_by" => r[r.crono_trigger_column_name(:locked_by)],
+              "started_at" => r[r.crono_trigger_column_name(:started_at)],
+              "finished_at" => r[r.crono_trigger_column_name(:finished_at)],
+              "last_error_name" => r[r.crono_trigger_column_name(:last_error_name)],
+              "last_error_reason" => r[r.crono_trigger_column_name(:last_error_reason)],
+              "last_error_time" => r[r.crono_trigger_column_name(:last_error_time)],
+              "retry_count" => r[r.crono_trigger_column_name(:retry_count)],
+              "time_to_unlock" => [(r.class.execute_lock_timeout + r[r.crono_trigger_column_name(:execute_lock)]) - now.to_i, 0].max,
+              "delay_sec" => r.locking?(at: now) ? 0 : (now - r[r.crono_trigger_column_name(:next_execute_at)]).to_i,
+            }
+          end
+          Oj.dump({
+            records: records,
+          }, mode: :compat)
+        else
+          status 404
+          "Model Class is not found"
+        end
+      else
+        raise "unknown format"
+      end
+    end
+
+    get "/models/:name" do
+      erb :index
+    end
+
+    get "/models.:format" do
+      if params[:format] == "json"
+        content_type :json
+        @models = CronoTrigger::Schedulable.included_by.map(&:name).sort
+        Oj.dump({
+          models: @models,
+        }, mode: :compat)
+      else
+        raise "unknown format"
+      end
+    end
+
+    get "/models" do
+      erb :index
+    end
+  end
+end

data/lib/crono_trigger/worker.rb

@@ -2,31 +2,141 @@ require "active_support/core_ext/string"
 
 module CronoTrigger
   module Worker
+    HEARTBEAT_INTERVAL = 60
+    SIGNAL_FETCH_INTERVAL = 10
+    EXECUTOR_SHUTDOWN_TIMELIMIT = 300
+    OTHER_THREAD_SHUTDOWN_TIMELIMIT = 120
+    attr_reader :polling_threads
+
     def initialize
+      @crono_trigger_worker_id = CronoTrigger.config.worker_id
       @stop_flag = ServerEngine::BlockingFlag.new
+      @heartbeat_stop_flag = ServerEngine::BlockingFlag.new
+      @signal_fetch_stop_flag = ServerEngine::BlockingFlag.new
       @model_queue = Queue.new
-      CronoTrigger.config.model_names.each do |model_name|
-        model = model_name.classify.constantize
-        @model_queue << model
+      @model_names = CronoTrigger.config.model_names || CronoTrigger::Schedulable.included_by
+      @model_names.each do |model_name|
+        @model_queue << model_name
       end
       @executor = Concurrent::ThreadPoolExecutor.new(
         min_threads: 1,
         max_threads: CronoTrigger.config.executor_thread,
+        max_queue: CronoTrigger.config.executor_thread * 2,
       )
-      ActiveRecord::Base.logger = logger
+      @execution_counter = Concurrent::AtomicFixnum.new
+      @logger = Logger.new(STDOUT) unless @logger
+      ActiveRecord::Base.logger = @logger
     end
 
     def run
-      polling_threads = CronoTrigger.config.polling_thread.times.map { PollingThread.new(@model_queue, @stop_flag, logger, @executor) }
-      polling_threads.each(&:run)
-      polling_threads.each(&:join)
+      @heartbeat_thread = run_heartbeat_thread
+      @signal_fetcn_thread = run_signal_fetch_thread
+
+      polling_thread_count = CronoTrigger.config.polling_thread || [@model_names.size, Concurrent.processor_count].min
+      # Assign local variable for Signal handling
+      polling_threads = polling_thread_count.times.map { PollingThread.new(@model_queue, @stop_flag, @logger, @executor, @execution_counter) }
+      @polling_threads = polling_threads
+      @polling_threads.each(&:run)
+
+      ServerEngine::SignalThread.new do |st|
+        st.trap(:TSTP) do
+          @logger.info("[worker_id:#{@crono_trigger_worker_id}] Transit to quiet mode")
+          polling_threads.each(&:quiet)
+          heartbeat
+        end
+      end
+
+      @polling_threads.each(&:join)
 
       @executor.shutdown
-      @executor.wait_for_termination
+      @executor.wait_for_termination(EXECUTOR_SHUTDOWN_TIMELIMIT)
+      @heartbeat_thread.join(OTHER_THREAD_SHUTDOWN_TIMELIMIT)
+      @signal_fetcn_thread.join(OTHER_THREAD_SHUTDOWN_TIMELIMIT)
+
+      unregister
     end
 
     def stop
       @stop_flag.set!
+      @heartbeat_stop_flag.set!
+      @signal_fetch_stop_flag.set!
+    end
+
+    def stopped?
+      @stop_flag.set?
+    end
+
+    def quiet?
+      @polling_threads&.all?(&:quiet?)
+    end
+
+    private
+
+    def run_heartbeat_thread
+      heartbeat
+      Thread.start do
+        until @heartbeat_stop_flag.wait_for_set(HEARTBEAT_INTERVAL)
+          heartbeat
+        end
+      end
+    end
+
+    def run_signal_fetch_thread
+      Thread.start do
+        until @signal_fetch_stop_flag.wait_for_set(SIGNAL_FETCH_INTERVAL)
+          handle_signal_from_rdb
+        end
+      end
+    end
+
+    def heartbeat
+      CronoTrigger::Models::Worker.connection_pool.with_connection do
+        begin
+          worker_record = CronoTrigger::Models::Worker.find_or_initialize_by(worker_id: @crono_trigger_worker_id)
+          worker_record.max_thread_size = @executor.max_length
+          worker_record.current_executing_size = @executor.scheduled_task_count
+          worker_record.current_queue_size = @execution_counter.value
+          worker_record.executor_status = executor_status
+          worker_record.polling_model_names = @model_names
+          worker_record.last_heartbeated_at = Time.current
+          @logger.info("[worker_id:#{@crono_trigger_worker_id}] Send heartbeat to database")
+          worker_record.save
+        rescue => ex
+          p ex
+          stop
+        end
+      end
+    end
+
+    def executor_status
+      case
+      when @executor.shutdown?
+        "shutdown"
+      when @executor.shuttingdown?
+        "shuttingdown"
+      when @executor.running?
+        if quiet?
+          "quiet"
+        else
+          "running"
+        end
+      end
+    end
+
+    def unregister
+      @logger.info("[worker_id:#{@crono_trigger_worker_id}] Unregister worker from database")
+      CronoTrigger::Models::Worker.connection_pool.with_connection do
+        CronoTrigger::Models::Worker.find_by(worker_id: @crono_trigger_worker_id)&.destroy
+      end
+    end
+
+    def handle_signal_from_rdb
+      CronoTrigger::Models::Signal.connection_pool.with_connection do
+        CronoTrigger::Models::Signal.sent_to_me.take(1)[0]&.tap do |s|
+          @logger.info("[worker_id:#{@crono_trigger_worker_id}] Receive Signal #{s.signal} from database")
+          s.kill_me(to_supervisor: s.signal != "TSTP")
+        end
+      end
     end
   end
 end