lowkiq 1.0.0

data/lib/lowkiq/queue/marshal.rb

@@ -0,0 +1,23 @@
+module Lowkiq
+  module Queue
+    module Marshal
+      class << self
+        def dump_payload(data)
+          ::Marshal.dump data
+        end
+
+        def load_payload(str)
+          ::Marshal.load str
+        end
+
+        def dump_data(data)
+          ::Marshal.dump data
+        end
+
+        def load_data(str)
+          ::Marshal.load str
+        end
+      end
+    end
+  end
+end

data/lib/lowkiq/queue/queries.rb

@@ -0,0 +1,143 @@
+module Lowkiq
+  module Queue
+    class Queries
+      def initialize(redis_pool, name)
+        @pool = redis_pool
+        @keys = Keys.new name
+        @fetch = Fetch.new name
+      end
+
+      def range_by_id(min, max, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrangebylex(
+            @keys.all_ids_lex_zset,
+            min, max,
+            limit: [0, limit]
+          )
+          _fetch redis, ids
+        end
+      end
+
+      def rev_range_by_id(max, min, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrevrangebylex(
+            @keys.all_ids_lex_zset,
+            max, min,
+            limit: [0, limit]
+          )
+          _fetch redis, ids
+        end
+      end
+
+      def range_by_perform_in(min, max, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrangebyscore(
+            @keys.all_ids_scored_by_perform_in_zset,
+            min, max,
+            limit: [0, limit]
+          )
+          _fetch redis, ids
+        end
+      end
+
+      def rev_range_by_perform_in(max, min, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrevrangebyscore(
+            @keys.all_ids_scored_by_perform_in_zset,
+            max, min,
+            limit: [0, limit]
+          )
+          _fetch redis, ids
+        end
+      end
+
+      def range_by_retry_count(min, max, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrangebyscore(
+            @keys.all_ids_scored_by_retry_count_zset,
+            min, max,
+            limit: [0, limit]
+          )
+          _fetch redis, ids
+        end
+      end
+
+      def rev_range_by_retry_count(max, min, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrevrangebyscore(
+            @keys.all_ids_scored_by_retry_count_zset,
+            max, min,
+            limit: [0, limit]
+          )
+          _fetch redis, ids
+        end
+      end
+
+      def morgue_range_by_id(min, max, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrangebylex(
+            @keys.morgue_all_ids_lex_zset,
+            min, max,
+            limit: [0, limit]
+          )
+          _morgue_fetch redis, ids
+        end
+      end
+
+      def morgue_rev_range_by_id(max, min, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrevrangebylex(
+            @keys.morgue_all_ids_lex_zset,
+            max, min,
+            limit: [0, limit]
+          )
+          _morgue_fetch redis, ids
+        end
+      end
+
+      def morgue_range_by_updated_at(min, max, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrangebyscore(
+            @keys.morgue_all_ids_scored_by_updated_at_zset,
+            min, max,
+            limit: [0, limit]
+          )
+          _morgue_fetch redis, ids
+        end
+      end
+
+      def morgue_rev_range_by_updated_at(max, min, limit: 10)
+        @pool.with do |redis|
+          ids = redis.zrevrangebyscore(
+            @keys.morgue_all_ids_scored_by_updated_at_zset,
+            max, min,
+            limit: [0, limit]
+          )
+          _morgue_fetch redis, ids
+        end
+      end
+
+      def fetch(ids)
+        @pool.with do |redis|
+          _fetch redis, ids
+        end
+      end
+
+      def morgue_fetch(ids)
+        @pool.with do |redis|
+          _morgue_fetch redis, ids
+        end
+      end
+
+      private
+
+      def _fetch(redis, ids)
+        @fetch.fetch(redis, :multi, ids)
+      end
+
+      def _morgue_fetch(redis, ids)
+        @fetch.morgue_fetch(redis, :multi, ids)
+      end
+    end
+  end
+end

data/lib/lowkiq/queue/queue.rb

@@ -0,0 +1,177 @@
+module Lowkiq
+  module Queue
+    class Queue
+      attr_reader :name, :pool
+
+      def initialize(redis_pool, name, shards_count)
+        @pool = redis_pool
+        @name = name
+        @shards_count = shards_count
+        @timestamp = Utils::Timestamp.method(:now)
+        @keys = Keys.new name
+        @fetch = Fetch.new name
+      end
+
+      def push(batch)
+        @pool.with do |redis|
+          redis.multi do
+            batch.each do |job|
+              id          = job.fetch(:id)
+              perform_in  = job.fetch(:perform_in, @timestamp.call)
+              retry_count = job.fetch(:retry_count, -1) # for testing
+              payload     = job.fetch(:payload, "")
+              score       = job.fetch(:score, @timestamp.call)
+
+              shard = id_to_shard id
+
+              redis.zadd @keys.all_ids_lex_zset, 0, id
+              redis.zadd @keys.all_ids_scored_by_perform_in_zset, perform_in, id, nx: true
+              redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id, nx: true
+
+              redis.zadd @keys.ids_scored_by_perform_in_zset(shard), perform_in, id, nx: true
+              redis.zadd @keys.payloads_zset(id), score, Marshal.dump_payload(payload), nx: true
+            end
+          end
+        end
+      end
+
+      def pop(shard, limit:)
+        @pool.with do |redis|
+          data = nil
+          tx = redis.watch @keys.ids_scored_by_perform_in_zset(shard) do
+            ids = redis.zrangebyscore @keys.ids_scored_by_perform_in_zset(shard),
+                                      0, @timestamp.call,
+                                      limit: [0, limit]
+
+            if ids.empty?
+              redis.unwatch
+              return []
+            end
+
+            data = @fetch.fetch(redis, :pipelined, ids)
+
+            redis.multi do
+              _delete redis, ids
+              redis.set  @keys.processing_key(shard), Marshal.dump_data(data)
+              redis.hset @keys.processing_length_by_shard_hash, shard, data.length
+            end
+          end until tx
+
+          data
+        end
+      end
+
+      def push_back(batch)
+        @pool.with do |redis|
+          batch.each do |job|
+            id          = job.fetch(:id)
+            perform_in  = job.fetch(:perform_in, @timestamp.call)
+            retry_count = job.fetch(:retry_count, -1)
+            payloads    = job.fetch(:payloads).map do |(payload, score)|
+              [score, Marshal.dump_payload(payload)]
+            end
+            error       = job.fetch(:error, nil)
+
+            shard = id_to_shard id
+
+            redis.multi do
+              redis.zadd @keys.all_ids_lex_zset, 0, id
+              redis.zadd @keys.all_ids_scored_by_perform_in_zset, perform_in, id
+              redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id
+
+              redis.zadd @keys.ids_scored_by_perform_in_zset(shard), perform_in, id
+              redis.zadd @keys.payloads_zset(id), payloads, nx: true
+
+              redis.hset @keys.errors_hash, id, error unless error.nil?
+            end
+          end
+        end
+      end
+
+      def ack(shard, result = nil)
+        @pool.with do |redis|
+          length = redis.hget(@keys.processing_length_by_shard_hash, shard).to_i
+          redis.multi do
+            redis.del  @keys.processing_key(shard)
+            redis.hdel @keys.processing_length_by_shard_hash, shard
+
+            redis.incrby @keys.processed_key, length if result == :success
+            redis.incrby @keys.failed_key, length    if result == :fail
+          end
+        end
+      end
+
+      def processing_data(shard)
+        data = @pool.with do |redis|
+          redis.get @keys.processing_key(shard)
+        end
+        return [] if data.nil?
+
+        Marshal.load_data data
+      end
+
+      def push_to_morgue(batch)
+        @pool.with do |redis|
+          batch.each do |job|
+            id       = job.fetch(:id)
+            payloads = job.fetch(:payloads).map do |(payload, score)|
+              [score, Marshal.dump_payload(payload)]
+            end
+            error    = job.fetch(:error, nil)
+
+            redis.multi do
+              redis.zadd @keys.morgue_all_ids_lex_zset, 0, id
+              redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, @timestamp.call, id
+              redis.zadd @keys.morgue_payloads_zset(id), payloads, nx: true
+
+              redis.hset @keys.morgue_errors_hash, id, error unless error.nil?
+            end
+          end
+        end
+      end
+
+      def morgue_delete(ids)
+        @pool.with do |redis|
+          redis.multi do
+            ids.each do |id|
+              redis.zrem @keys.morgue_all_ids_lex_zset, id
+              redis.zrem @keys.morgue_all_ids_scored_by_updated_at_zset, id
+              redis.del  @keys.morgue_payloads_zset(id)
+              redis.hdel @keys.morgue_errors_hash, id
+            end
+          end
+        end
+      end
+
+      def delete(ids)
+        @pool.with do |redis|
+          redis.multi do
+            _delete redis, ids
+          end
+        end
+      end
+
+      def shards
+        (0...@shards_count)
+      end
+
+      private
+
+      def id_to_shard(id)
+        Zlib.crc32(id.to_s) % @shards_count
+      end
+
+      def _delete(redis, ids)
+        ids.each do |id|
+          shard = id_to_shard id
+          redis.zrem @keys.all_ids_lex_zset, id
+          redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
+          redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
+          redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
+          redis.del  @keys.payloads_zset(id)
+          redis.hdel @keys.errors_hash, id
+        end
+      end
+    end
+  end
+end

data/lib/lowkiq/queue/queue_metrics.rb

@@ -0,0 +1,80 @@
+module Lowkiq
+  module Queue
+    class QueueMetrics
+      def initialize(redis_pool)
+        @redis_pool = redis_pool
+        @timestamp = Utils::Timestamp.method(:now)
+      end
+
+      def call(queues)
+        result = @redis_pool.with do |redis|
+          #redis.pipelined do
+          redis.multi do
+            queues.each { |queue| pipeline redis, queue }
+          end
+        end
+
+        result.each_slice(pipeline_count).map do |res|
+          coerce(res)
+        end
+      end
+
+      private
+
+      def pipeline(redis, name)
+        keys = Keys.new name
+
+        # fresh
+        redis.zcount keys.all_ids_scored_by_retry_count_zset, -1, -1
+
+        # retries
+        redis.zcount keys.all_ids_scored_by_retry_count_zset, 0, '+inf'
+
+        # morgue_length
+        redis.zcard keys.morgue_all_ids_scored_by_updated_at_zset
+
+        # lag [id, score]
+        redis.zrange keys.all_ids_scored_by_perform_in_zset,
+                     0, 0, with_scores: true
+        # processed
+        redis.get keys.processed_key
+
+        # failed
+        redis.get keys.failed_key
+
+        # busy []
+        redis.hvals keys.processing_length_by_shard_hash
+      end
+
+      def pipeline_count
+        7
+      end
+
+      def coerce(result)
+        length = result[0] + result[1]
+        OpenStruct.new length:        length,
+                       fresh:         result[0],
+                       retries:       result[1],
+                       morgue_length: result[2],
+                       lag:           coerce_lag(result[3]),
+                       processed:     result[4].to_i,
+                       failed:        result[5].to_i,
+                       busy:          coerce_busy(result[6])
+      end
+
+      def coerce_lag(res)
+        _id, score = res.first
+
+        return 0 if score.nil?
+        return 1 if score == 0 # на случай Actions#perform_all_jobs_now
+        lag = @timestamp.call - score.to_i
+        return 0 if lag < 0
+        lag
+      end
+
+      def coerce_busy(res)
+        res.map(&:to_i).reduce(0, &:+)
+      end
+    end
+  end
+end

data/lib/lowkiq/queue/shard_metrics.rb

@@ -0,0 +1,52 @@
+module Lowkiq
+  module Queue
+    class ShardMetrics
+      def initialize(redis_pool)
+        @redis_pool = redis_pool
+        @timestamp = Utils::Timestamp.method(:now)
+      end
+
+      def call(queues)
+        result = @redis_pool.with do |redis|
+          redis.pipelined do
+            queues.each { |queue| pipeline redis, queue }
+          end
+        end
+
+        result.each_slice(pipeline_count).map do |res|
+          coerce(res)
+        end
+      end
+
+      private
+
+      def pipeline(redis, id)
+        name = id[:queue_name]
+        shard = id[:shard]
+
+        keys = Keys.new name
+
+        # lag [id, score]
+        redis.zrange keys.ids_scored_by_perform_in_zset(shard),
+                     0, 0, with_scores: true
+      end
+
+      def pipeline_count
+        1
+      end
+
+      def coerce(result)
+        OpenStruct.new lag: coerce_lag(result[0])
+      end
+
+      def coerce_lag(res)
+        _id, score = res.first
+
+        return 0 if score.nil?
+        lag = @timestamp.call - score.to_i
+        return 0 if lag < 0
+        lag
+      end
+    end
+  end
+end