lowkiq 1.0.0

data/lib/lowkiq/redis_info.rb

@@ -0,0 +1,21 @@
+module Lowkiq
+  class RedisInfo
+    def initialize(redis_pool)
+      @redis_pool = redis_pool
+    end
+
+    def call
+      @redis_pool.with do |redis|
+        info = redis.info
+        {
+          url: redis.connection[:id],
+          version: info["redis_version"],
+          uptime_in_days: info["uptime_in_days"],
+          connected_clients: info["connected_clients"],
+          used_memory_human: info["used_memory_human"],
+          used_memory_peak_human: info["used_memory_peak_human"],
+        }
+      end
+    end
+  end
+end

data/lib/lowkiq/schedulers/lag.rb

@@ -0,0 +1,27 @@
+module Lowkiq
+  module Schedulers
+    class Lag
+      def initialize(wait, metrics)
+        @metrics = metrics
+        @wait = wait
+      end
+
+      def build_job(shard_handlers)
+        Proc.new do
+          identifiers = shard_handlers.map { |sh| { queue_name: sh.queue_name, shard: sh.shard_index } }
+          metrics = @metrics.call identifiers
+          shard_handler, _lag =
+                         shard_handlers.zip(metrics.map(&:lag))
+                           .select { |(_, lag)| lag > 0 }
+                           .max_by { |(_, lag)| lag }
+
+          if shard_handler
+            shard_handler.process
+          else
+            @wait.call
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lowkiq/schedulers/seq.rb

@@ -0,0 +1,28 @@
+module Lowkiq
+  module Schedulers
+    class Seq
+      def initialize(wait)
+        @wait = wait
+      end
+
+      def build_job(shard_handlers)
+        shard_enumerator = shard_handlers.cycle
+        processed = []
+
+        lambda do
+          if processed.length == shard_handlers.length
+            all_failed = processed.all? { |ok| !ok }
+            processed.clear
+            if all_failed
+              @wait.call
+              return
+            end
+          end
+
+          shard_handler = shard_enumerator.next
+          processed << shard_handler.process
+        end
+      end
+    end
+  end
+end

data/lib/lowkiq/server.rb

@@ -0,0 +1,54 @@
+module Lowkiq
+  class Server
+    def self.build(options)
+      require options[:require]
+      Lowkiq.on_server_init.call
+
+      splitter = Lowkiq.build_splitter.call
+      shard_handlers_by_thread = splitter.call Lowkiq.shard_handlers
+      scheduler = Lowkiq.build_scheduler.call
+      new shard_handlers_by_thread, scheduler
+    end
+
+    def initialize(shard_handlers_by_thread, scheduler)
+      @shard_handlers_by_thread = shard_handlers_by_thread
+      @scheduler = scheduler
+      @threads = []
+    end
+
+    def start
+      @shard_handlers_by_thread.each do |handlers|
+        handlers.each(&:restore)
+      end
+
+      @threads = @shard_handlers_by_thread.map do |handlers|
+        job = @scheduler.build_job handlers
+        Thread.new do
+          job.call until exit_from_thread?
+        end
+      end
+    end
+
+    def stop
+      @stopped = true
+    end
+
+    def join
+      @threads.each(&:join)
+    end
+
+    def exit_from_thread?
+      stopped? || failed?
+    end
+
+    def stopped?
+      @stopped
+    end
+
+    def failed?
+      @threads.map(&:status).any? do |status|
+        status != "run" && status != "sleep"
+      end
+    end
+  end
+end

data/lib/lowkiq/shard_handler.rb

@@ -0,0 +1,110 @@
+module Lowkiq
+  class ShardHandler
+    def self.build_many(worker, wrapper)
+      (0...worker.shards_count).map do |shard_index|
+        new shard_index, worker, wrapper
+      end
+    end
+
+    attr_reader :shard_index, :queue_name, :worker
+
+    def initialize(shard_index, worker, wrapper)
+      @shard_index = shard_index
+      @queue_name = worker.queue_name
+      @worker = worker
+      @wrapper = wrapper
+      @timestamp = Utils::Timestamp.method(:now)
+      @queue = Queue::Queue.new Lowkiq.server_redis_pool,
+                                worker.queue_name,
+                                worker.shards_count
+    end
+
+    def process
+      data = @queue.pop @shard_index, limit: @worker.batch_size
+
+      return false if data.empty?
+
+      begin
+        batch = batch_from_data data
+
+        @wrapper.call @worker, batch do
+          @worker.perform batch
+        end
+
+        @queue.ack @shard_index, :success
+        true
+      rescue => ex
+        fail! data, ex
+        back, morgue = separate data
+
+        @queue.push_back back
+        @queue.push_to_morgue morgue
+        @queue.ack @shard_index, :fail
+        false
+      end
+    end
+
+    def restore
+      data = @queue.processing_data @shard_index
+      return if data.nil?
+      @queue.push_back data
+      @queue.ack @shard_index
+    end
+
+    private
+
+    def batch_from_data(data)
+      data.each_with_object({}) do |job, h|
+        id = job.fetch(:id)
+        payloads = job.fetch(:payloads).map(&:first)
+        h[id] = payloads
+      end
+    end
+
+    def fail!(data, ex)
+      data.map! do |job|
+        job[:retry_count] += 1
+        job[:perform_in] = @timestamp.call + @worker.retry_in(job[:retry_count])
+        job[:error] = ex.message
+        job
+      end
+    end
+
+    def separate(data)
+      back = []
+      morgue = []
+
+      data.each do |job|
+        id          = job.fetch(:id)
+        payloads    = job.fetch(:payloads)
+        retry_count = job.fetch(:retry_count)
+        perform_in  = job.fetch(:perform_in)
+        error       = job.fetch(:error, nil)
+
+        morgue_payload = payloads.shift if retry_count >= @worker.max_retry_count
+
+        if payloads.any?
+          job = {
+            id: id,
+            payloads: payloads,
+            retry_count: morgue_payload ? 0 : retry_count,
+            perform_in:  morgue_payload ? @timestamp.call : perform_in,
+            error: error,
+          }.compact
+          back << job
+        end
+
+        if morgue_payload
+          job = {
+            id: id,
+            payloads: [morgue_payload],
+            error: error,
+          }.compact
+          morgue << job
+        end
+      end
+
+      [back, morgue]
+    end
+  end
+end

data/lib/lowkiq/splitters/by_node.rb

@@ -0,0 +1,19 @@
+module Lowkiq
+  module Splitters
+    class ByNode
+      def initialize(number_of_nodes, node_number, threads_per_node)
+        @number_of_nodes = number_of_nodes
+        @node_number = node_number
+        @threads_per_node = threads_per_node
+      end
+
+      def call(shard_handlers)
+        groups_for_nodes = Utils::Array.new(shard_handlers).in_transposed_groups(@number_of_nodes)
+        groups_for_node = groups_for_nodes[@node_number]
+        Utils::Array.new(groups_for_node)
+          .in_transposed_groups(@threads_per_node)
+          .reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/lowkiq/splitters/default.rb

@@ -0,0 +1,15 @@
+module Lowkiq
+  module Splitters
+    class Default
+      def initialize(threads_per_node)
+        @threads_per_node = threads_per_node
+      end
+
+      def call(shard_handlers)
+        Utils::Array.new(shard_handlers)
+          .in_transposed_groups(@threads_per_node)
+          .reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/lowkiq/utils.rb

@@ -0,0 +1,36 @@
+module Lowkiq
+  module Utils
+    class Array
+      def initialize(array)
+        @array = array.to_a
+      end
+
+      def in_transposed_groups(number)
+        result = number.times.map { [] }
+
+        @array.each_with_index do |item, index|
+          group = index % number
+          result[group] << item
+        end
+
+        result
+      end
+    end
+
+    class Redis
+      def initialize(redis)
+        @redis = redis
+      end
+
+      def zresetscores(key)
+        @redis.zunionstore key, [key], weights: [0.0]
+      end
+    end
+
+    module Timestamp
+      def self.now
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/lowkiq/version.rb

@@ -0,0 +1,3 @@
+module Lowkiq
+  VERSION = "1.0.0"
+end

data/lib/lowkiq/web.rb

@@ -0,0 +1,45 @@
+require "rack"
+require "lowkiq/web/action"
+require "lowkiq/web/api"
+
+module Lowkiq
+  module Web
+    HTML = Proc.new do |env|
+      root_path = env['SCRIPT_NAME']
+      [200, {}, [<<-HTML]]
+        <!DOCTYPE html>
+        <html>
+          <head>
+            <meta charset="UTF-8">
+            <title>Lowkiq</title>
+          </head>
+          <body>
+            <div id="root"></div>
+            <script type="text/javascript">
+              window.lowkiqRoot="#{root_path}";
+            </script>
+            <script type="text/javascript" src="#{root_path}/assets/#{VERSION}/app.js"></script>
+          </body>
+        </html>
+      HTML
+    end
+
+    ASSETS = File.expand_path("#{File.dirname(__FILE__)}/../../assets")
+
+    APP = Rack::Builder.new do
+      map "/api" do
+        run Api
+      end
+
+      map "/assets/#{VERSION}" do
+        run Rack::File.new ASSETS, { 'Cache-Control' => 'public, max-age=86400' }
+      end
+
+      run HTML
+    end
+
+    def self.call(env)
+      APP.call env
+    end
+  end
+end

data/lib/lowkiq/web/action.rb

@@ -0,0 +1,31 @@
+module Lowkiq
+  module Web
+    class Action
+      def self.segments_to_regex(segments)
+        prepared = segments.map do |segment|
+          case segment
+          when Symbol
+            "(?<#{segment}>[^\/]+)"
+          else
+            segment
+          end
+        end.join( '/' )
+        Regexp.new '\A' + '/' + prepared + '\z'
+      end
+
+      def initialize(request_method, segments, &body)
+        @request_method = request_method
+        @url_pattern = self.class.segments_to_regex(segments)
+        @body = body
+      end
+
+      def call(req)
+        return if @request_method != req.request_method
+        match = @url_pattern.match req.path_info
+        return unless match
+        data = @body.call req, match
+        [200, {}, [JSON.generate(data)]]
+      end
+    end
+  end
+end

data/lib/lowkiq/web/api.rb

@@ -0,0 +1,142 @@
+module Lowkiq
+  module Web
+    module Api
+      GET = 'GET'.freeze
+      POST = 'POST'.freeze
+
+      ACTIONS = [
+        Action.new(GET, ['v1', 'stats']) do |_, _|
+          worker_names = Lowkiq.workers.map(&:name)
+          queue_names = Lowkiq.workers.map(&:queue_name)
+
+          metrics = Lowkiq::Queue::QueueMetrics
+                      .new(Lowkiq.client_redis_pool)
+                      .call(queue_names)
+          by_worker = worker_names.zip(metrics).each_with_object({}) do |(name, m), o|
+            o[name] = m.to_h.slice(:length, :morgue_length, :lag)
+          end
+          total = {
+            length:        metrics.map(&:length).reduce(&:+).to_i,
+            morgue_length: metrics.map(&:morgue_length).reduce(&:+).to_i,
+            lag:           metrics.map(&:lag).max.to_i,
+          }
+          {
+            total: total,
+            by_worker: by_worker,
+          }
+        end,
+
+        Action.new(GET, ['web', 'dashboard']) do |_, _|
+          worker_names = Lowkiq.workers.map(&:name)
+          queue_names = Lowkiq.workers.map(&:queue_name)
+
+          metrics = Lowkiq::Queue::QueueMetrics
+                      .new(Lowkiq.client_redis_pool)
+                      .call(queue_names)
+
+          queues = worker_names.zip(metrics).map do |(name, m)|
+            {
+              name: name,
+              lag: m.lag,
+              processed: m.processed,
+              failed: m.failed,
+              busy: m.busy,
+              enqueued: m.length, # fresh + retries
+              fresh: m.fresh,
+              retries: m.retries,
+              dead: m.morgue_length,
+            }
+          end
+
+          redis_info = Lowkiq::RedisInfo.new(Lowkiq.client_redis_pool).call
+
+          {
+            queues: queues,
+            redis_info: redis_info,
+          }
+        end,
+
+        %w[ range_by_id range_by_perform_in range_by_retry_count
+            morgue_range_by_id morgue_range_by_updated_at
+        ].map do |method|
+          Action.new(GET, ['web', :worker, method]) do |req, match|
+            min = req.params['min']
+            max = req.params['max']
+
+            queries = match_to_worker(match).client_queries
+            queries.public_send method, min, max, limit: 100
+          end
+        end,
+
+        %w[ rev_range_by_id rev_range_by_perform_in rev_range_by_retry_count
+            morgue_rev_range_by_id morgue_rev_range_by_updated_at
+        ].map do |method|
+          Action.new(GET, ['web', :worker, method]) do |req, match|
+            min = req.params['min']
+            max = req.params['max']
+
+            queries = match_to_worker(match).client_queries
+            queries.public_send method, max, min, limit: 100
+          end
+        end,
+
+        Action.new(GET, ['web', :worker, 'processing_data']) do |_, match|
+          queue = match_to_worker(match).client_queue
+
+          queue.shards.flat_map do |shard|
+            queue.processing_data shard
+          end
+        end,
+
+        %w[ morgue_delete ].map do |method|
+          Action.new(POST, ['web', :worker, method]) do |req, match|
+            ids = req.params['ids']
+            Thread.new do
+              queue = match_to_worker(match).client_queue
+              queue.public_send method, ids
+            end
+            :ok
+          end
+        end,
+
+        %w[ morgue_queue_up ].map do |method|
+          Action.new(POST, ['web', :worker, method]) do |req, match|
+            ids = req.params['ids']
+            Thread.new do
+              actions = match_to_worker(match).client_actions
+              actions.public_send method, ids
+            end
+            :ok
+          end
+        end,
+
+        %w[ morgue_queue_up_all_jobs morgue_delete_all_jobs
+            perform_all_jobs_now kill_all_failed_jobs delete_all_failed_jobs].map do |method|
+          Action.new(POST, ['web', :worker, method]) do |_, match|
+            Thread.new do
+              actions = match_to_worker(match).client_actions
+              actions.public_send method
+            end
+            :ok
+          end
+        end,
+
+      ].flatten
+
+      def self.match_to_worker(match)
+        Lowkiq.workers.find { |w| w.name == match[:worker] }
+      end
+
+      def self.call(env)
+        req = Rack::Request.new env
+
+        ACTIONS.each do |action|
+          resp = action.call req
+          return resp if resp
+        end
+
+        [404, {}, ["not found"]]
+      end
+    end
+  end
+end