lowkiq 1.0.0

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lowkiq"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/deploy.md

@@ -0,0 +1,3 @@
++ bump version in `lib/lowkiq/version.rb`
++ build frontend: `npm run build`
++ build gem: `gem build lowkiq.gemspec`

data/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+volumes:
+  bundle:
+
+services:
+  app:
+    image: ruby
+    environment:
+      REDIS_URL: redis://redis:6379
+    ports:
+      - "8080:8080" # http
+    working_dir: /usr/src/app
+    volumes:
+      - ./:/usr/src/app
+      - bundle:/usr/local/bundle
+    depends_on:
+      - redis
+  redis:
+    image: redis:5-alpine
+
+  frontend:
+    image: node
+    ports:
+      - "8081:8081"
+    working_dir: /usr/src/app
+    volumes:
+      - ./frontend/:/usr/src/app
+      - ./assets:/usr/src/app/build

data/exe/lowkiq

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require 'lowkiq'
+
+options = Lowkiq::OptionParser.call ARGV
+server = Lowkiq::Server.build options
+
+Signal.trap('TERM') { server.stop }
+
+Signal.trap('INT') do
+  if server.stopped?
+    Process.exit
+  else
+    server.stop
+  end
+end
+
+Signal.trap('TTIN') do
+  file = "/tmp/lowkiq_ttin.txt"
+
+  File.delete file if File.exists? file
+
+  File.open(file, 'w') do |file|
+    Thread.list.each_with_index do |thread, idx|
+      file.write "== thread #{idx} == \n"
+      if thread.backtrace.nil?
+        file.write "<no backtrace available> \n"
+      else
+        thread.backtrace.each do |line|
+          file.write "#{line} \n"
+        end
+      end
+    end
+  end
+end
+
+begin
+  server.start
+  server.join
+rescue => ex
+  Lowkiq.last_words.call ex
+  raise ex
+end

data/lib/lowkiq.rb

@@ -0,0 +1,111 @@
+require "connection_pool"
+require "redis"
+require "zlib"
+require "json"
+require "ostruct"
+require "optparse"
+
+require "lowkiq/version"
+require "lowkiq/utils"
+
+require "lowkiq/extend_tracker"
+require "lowkiq/option_parser"
+
+require "lowkiq/splitters/default"
+require "lowkiq/splitters/by_node"
+
+require "lowkiq/schedulers/lag"
+require "lowkiq/schedulers/seq"
+
+require "lowkiq/server"
+
+require "lowkiq/queue/marshal"
+require "lowkiq/queue/keys"
+require "lowkiq/queue/fetch"
+require "lowkiq/queue/queue"
+require "lowkiq/queue/queue_metrics"
+require "lowkiq/queue/shard_metrics"
+require "lowkiq/queue/queries"
+require "lowkiq/queue/actions"
+require "lowkiq/worker"
+require "lowkiq/shard_handler"
+
+require "lowkiq/redis_info"
+
+require "lowkiq/web"
+
+module Lowkiq
+  class << self
+    attr_accessor :poll_interval, :threads_per_node,
+                  :redis, :client_pool_size, :pool_timeout,
+                  :server_middlewares, :on_server_init,
+                  :build_scheduler, :build_splitter,
+                  :last_words
+
+    def server_redis_pool
+      @server_redis_pool ||= ConnectionPool.new(size: threads_per_node, timeout: pool_timeout, &redis)
+    end
+
+    def client_redis_pool
+      @client_redis_pool ||= ConnectionPool.new(size: client_pool_size, timeout: pool_timeout, &redis)
+    end
+
+    def server_wrapper
+      null = -> (worker, batch, &block) { block.call }
+      server_middlewares.reduce(null) do |wrapper, m|
+        -> (worker, batch, &block) do
+          wrapper.call worker, batch do
+            m.call worker, batch, &block
+          end
+        end
+      end
+    end
+
+    def workers
+      Worker.extended_modules
+    end
+
+    def shard_handlers
+      self.workers.flat_map do |w|
+        ShardHandler.build_many w, self.server_wrapper
+      end
+    end
+
+    def build_lag_scheduler
+      Schedulers::Lag.new(
+        ->() { sleep Lowkiq.poll_interval },
+        Queue::ShardMetrics.new(self.server_redis_pool)
+      )
+    end
+
+    def build_seq_scheduler
+      Schedulers::Seq.new(
+        ->() { sleep Lowkiq.poll_interval }
+      )
+    end
+
+    def build_default_splitter
+      Lowkiq::Splitters::Default.new Lowkiq.threads_per_node
+    end
+
+    def build_by_node_splitter(number_of_nodes, node_number)
+      Lowkiq::Splitters::ByNode.new(
+        number_of_nodes,
+        node_number,
+        Lowkiq.threads_per_node,
+      )
+    end
+  end
+
+  # defaults
+  self.poll_interval = 1
+  self.threads_per_node = 5
+  self.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL') }
+  self.client_pool_size = 5
+  self.pool_timeout = 5
+  self.server_middlewares = []
+  self.on_server_init = ->() {}
+  self.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
+  self.build_splitter = ->() { Lowkiq.build_default_splitter }
+  self.last_words = ->(ex) {}
+end

data/lib/lowkiq/extend_tracker.rb

@@ -0,0 +1,13 @@
+module Lowkiq
+  module ExtendTracker
+    def extended(mod)
+      @extended_modules ||= []
+      @extended_modules << mod
+      @extended_modules.sort_by! &:name
+    end
+
+    def extended_modules
+      @extended_modules
+    end
+  end
+end

data/lib/lowkiq/option_parser.rb

@@ -0,0 +1,24 @@
+module Lowkiq
+  module OptionParser
+    module_function
+
+    def call(args)
+      options = {
+      }
+      ::OptionParser.new do |parser|
+        parser.on("-r", "--require PATH") do |path|
+          options[:require] = path
+        end
+
+        parser.on("-h", "--help", "Prints this help") do
+          puts parser
+          exit
+        end
+      end.parse!(args)
+
+      fail "--require is required option" if options[:require].nil? || options[:require].empty?
+
+      options
+    end
+  end
+end

data/lib/lowkiq/queue/actions.rb

@@ -0,0 +1,63 @@
+module Lowkiq
+  module Queue
+    class Actions
+      def initialize(queue, queries)
+        @queue = queue
+        @queries = queries
+
+        @pool = queue.pool
+        @keys = Keys.new queue.name
+      end
+
+      def perform_all_jobs_now
+        @pool.with do |redis|
+          uredis = Utils::Redis.new redis
+          redis.multi do
+            uredis.zresetscores @keys.all_ids_scored_by_perform_in_zset
+            @queue.shards.each do |shard|
+              uredis.zresetscores @keys.ids_scored_by_perform_in_zset(shard)
+            end
+          end
+        end
+      end
+
+      def kill_all_failed_jobs
+        until (jobs = @queries.range_by_retry_count('0', '+inf', limit: 100); jobs.empty?)
+          @queue.push_to_morgue jobs
+          ids = jobs.map { |j| j[:id] }
+          @queue.delete ids
+        end
+      end
+
+      def delete_all_failed_jobs
+        until (jobs = @queries.range_by_retry_count('0', '+inf', limit: 100); jobs.empty?)
+          ids = jobs.map { |j| j[:id] }
+          @queue.delete ids
+        end
+      end
+
+      def morgue_queue_up(ids)
+        jobs = @queries.morgue_fetch ids
+        return if jobs.empty?
+
+        @queue.push_back jobs
+        @queue.morgue_delete ids
+      end
+
+      def morgue_queue_up_all_jobs
+        until (jobs = @queries.morgue_range_by_id('-', '+', limit: 100); jobs.empty?)
+          @queue.push_back jobs
+          ids = jobs.map { |j| j[:id] }
+          @queue.morgue_delete ids
+        end
+      end
+
+      def morgue_delete_all_jobs
+        until (jobs = @queries.morgue_range_by_id('-', '+', limit: 100); jobs.empty?)
+          ids = jobs.map { |j| j[:id] }
+          @queue.morgue_delete ids
+        end
+      end
+    end
+  end
+end

data/lib/lowkiq/queue/fetch.rb

@@ -0,0 +1,51 @@
+module Lowkiq
+  module Queue
+    class Fetch
+      def initialize(name)
+        @keys = Keys.new name
+      end
+
+      def fetch(redis, strategy, ids)
+        resp = redis.public_send strategy do
+          ids.each do |id|
+            redis.zscore @keys.all_ids_scored_by_perform_in_zset, id
+            redis.zscore @keys.all_ids_scored_by_retry_count_zset, id
+            redis.zrange @keys.payloads_zset(id), 0, -1, with_scores: true
+            redis.hget   @keys.errors_hash, id
+          end
+        end
+
+        ids.zip(resp.each_slice(4)).map do |x|
+          next if x[1][0].nil? # пропускаем id, если его уже нет в очереди
+          res = {
+            id: x[0],
+            perform_in: x[1][0],
+            retry_count: x[1][1],
+            payloads: x[1][2].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            error: x[1][3],
+          }.compact
+        end.compact
+      end
+
+      def morgue_fetch(redis, strategy, ids)
+        resp = redis.public_send strategy do
+          ids.each do |id|
+            redis.zscore @keys.morgue_all_ids_scored_by_updated_at_zset, id
+            redis.zrange @keys.morgue_payloads_zset(id), 0, -1, with_scores: true
+            redis.hget   @keys.morgue_errors_hash, id
+          end
+        end
+
+        ids.zip(resp.each_slice(3)).map do |x|
+          next if x[1][0].nil? # пропускаем id, если его уже нет в очереди
+          {
+            id: x[0],
+            updated_at: x[1][0],
+            payloads: x[1][1].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            error: x[1][2],
+          }.compact
+        end.compact
+      end
+    end
+  end
+end

data/lib/lowkiq/queue/keys.rb

@@ -0,0 +1,67 @@
+module Lowkiq
+  module Queue
+    class Keys
+      PREFIX = 'lowkiq:A:v1'.freeze
+
+      def initialize(name)
+        @prefix = [PREFIX, name].join(':').freeze
+      end
+
+      def processed_key
+        [@prefix, :processed].join(':')
+      end
+
+      def failed_key
+        [@prefix, :failed].join(':')
+      end
+
+      def all_ids_lex_zset
+        [@prefix, :all_ids_lex].join(':')
+      end
+
+      def all_ids_scored_by_perform_in_zset
+        [@prefix, :all_ids_scored_by_perfrom_in].join(':')
+      end
+
+      def all_ids_scored_by_retry_count_zset
+        [@prefix, :all_ids_scored_by_retry_count].join(':')
+      end
+
+      def ids_scored_by_perform_in_zset(shard)
+        [@prefix, :ids_scored_by_perform_in, shard].join(':')
+      end
+
+      def payloads_zset(id)
+        [@prefix, :payloads, id].join(':')
+      end
+
+      def errors_hash
+        [@prefix, :errors].join(':')
+      end
+
+      def processing_key(shard)
+        [@prefix, :processing, shard].join(':')
+      end
+
+      def processing_length_by_shard_hash
+        [@prefix, :processing_length_by_shard].join(':')
+      end
+
+      def morgue_all_ids_lex_zset
+        [@prefix, :morgue, :all_ids_lex].join(':')
+      end
+
+      def morgue_all_ids_scored_by_updated_at_zset
+        [@prefix, :morgue, :all_ids_scored_by_updated_at].join(':')
+      end
+
+      def morgue_payloads_zset(id)
+        [@prefix, :morgue, :payloads, id].join(':')
+      end
+
+      def morgue_errors_hash
+        [@prefix, :morgue, :errors].join(':')
+      end
+    end
+  end
+end