gitlab-sidekiq-fetcher 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: f10159f679879ed622c3bbd336bb04d0da0c54d536b7270c8c81dcc1243cedfc
-  data.tar.gz: cbca779aacb710b4f8de222799ca88860f3745666d22b51ea3d7da7f5677f8a0
+SHA1:
+  metadata.gz: 00056b1430ab40c06094813fcf06296ef24eef40
+  data.tar.gz: 1ee1138aeaed7c353e2eaddab6982d7a1d384d82
 SHA512:
-  metadata.gz: dd8376c3c379c325db87519e43f3ea36f96063b03b8884dc45363ff949cf3cda5341b20641f27dbb90be8a2188c0588ec92eb048b4055702aac6f7419f595a49
-  data.tar.gz: 02b0b4746cbe42960b89fc3b1b7f861f87b45317f6ae1f1f76d737333f80ac67c1553f6a8fdd3d44cacda87c08de88f9dee9b227a6d4b021d8bd58cacaa7fdee
+  metadata.gz: 5c141c0c16201c5b788268b3d829cae2b1820682fb907f56f61e75aa205c223d83c519c544ac3bf0d0c8f7dd5758c3f754e93bbfe184e92b80c2bd9e134b06a3
+  data.tar.gz: d1cef7c3751a5b6dd04155a21d4ef9742bb274552eb66ab41cf2044cbcc94f21ca494aac7ee3e6db01fd9891e259e8bb2e52bc26c5d3299dd72b84f5088fe0f0

data/.gitignore

@@ -1,2 +1,2 @@
+Gemfile.lock
 *.gem
-coverage

data/README.md

@@ -5,11 +5,10 @@ gitlab-sidekiq-fetcher
 fetches from Redis.
 
 It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
+At this time we only added Sidekiq 5+ support to it.
 
-There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
-semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
-However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.
-
+It implements in Sidekiq the reliable queue pattern using [Redis' rpoplpush
+command](http://redis.io/commands/rpoplpush#pattern-reliable-queue).
 
 ## Installation
 
@@ -25,22 +24,12 @@ Enable reliable fetches by calling this gem from your Sidekiq configuration:
 
 ```ruby
 Sidekiq.configure_server do |config|
-  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+  Sidekiq::ReliableFetcher.setup_reliable_fetch!(config)
 
   # …
 end
 ```
 
-There is an additional parameter `config.options[:semi_reliable_fetch]` you can use to switch between two strategies:
-
-```ruby
-Sidekiq.configure_server do |config|
-  config.options[:semi_reliable_fetch] = true # Default value is false
-
-  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
-end
-```
-
 ## License
 
 LGPL-3.0, see the LICENSE file.

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,14 +1,16 @@
 Gem::Specification.new do |s|
   s.name        = 'gitlab-sidekiq-fetcher'
-  s.version     = '0.1.0'
+  s.version     = '0.3.0'
   s.authors     = ['TEA', 'GitLab']
   s.email       = 'valery@gitlab.com'
   s.license     = 'LGPL-3.0'
-  s.homepage    = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
+  s.homepage    = 'https://github.com/TEA-ebook/sidekiq-reliable-fetch'
   s.summary     = 'Reliable fetch extension for Sidekiq'
   s.description = 'Redis reliable queue pattern implemented in Sidekiq'
   s.require_paths = ['lib']
+
   s.files = `git ls-files`.split($\)
   s.test_files  = []
+
   s.add_dependency 'sidekiq', '~> 5'
 end

data/lib/sidekiq-reliable-fetch.rb

@@ -1,5 +1,4 @@
 require 'sidekiq'
 
-require_relative 'sidekiq/base_reliable_fetch'
-require_relative 'sidekiq/reliable_fetch'
-require_relative 'sidekiq/semi_reliable_fetch'
+require_relative 'sidekiq/reliable_fetcher'
+require_relative 'sidekiq-reliable-fetch/web'

data/lib/sidekiq-reliable-fetch/api.rb

@@ -0,0 +1,56 @@
+module SidekiqReliableFetch
+  ##
+  # Encapsulates a working queue within Sidekiq.
+  # Allows enumeration of all jobs within the queue.
+  #
+  #   queue = SidekiqReliableFetch::WorkingQueue.new("mailer")
+  #   queue.each do |job|
+  #     job.klass # => 'MyWorker'
+  #     job.args # => [1, 2, 3]
+  #   end
+  #
+  class WorkingQueue
+    include Enumerable
+
+    def self.all
+      Sidekiq.redis { |c| c.keys('queue:*:working') }
+        .sort
+        .map { |q| SidekiqReliableFetch::WorkingQueue.new(q) }
+    end
+
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def size
+      Sidekiq.redis { |con| con.llen(@name) }
+    end
+
+    def each
+      initial_size = size
+      deleted_size = 0
+      page = 0
+      page_size = 50
+
+      loop do
+        range_start = page * page_size - deleted_size
+        range_end   = page * page_size - deleted_size + (page_size - 1)
+        entries = Sidekiq.redis do |conn|
+          conn.lrange @name, range_start, range_end
+        end
+        break if entries.empty?
+        page += 1
+        entries.each do |entry|
+          yield Sidekiq::Job.new(entry, @name)
+        end
+        deleted_size = initial_size - size
+      end
+    end
+
+    def find_job(jid)
+      detect { |j| j.jid == jid }
+    end
+  end
+end

data/lib/sidekiq-reliable-fetch/web.rb

@@ -0,0 +1,24 @@
+require_relative 'api'
+
+module SidekiqReliableFetch
+  # Hook into *Sidekiq::Web* Sinatra app which adds a new '/working' page
+  module Web
+    VIEW_PATH = File.expand_path('../../../web/views', __FILE__)
+
+    def self.registered(app)
+      app.get '/working' do
+        @queues = SidekiqReliableFetch::WorkingQueue.all
+        erb File.read(File.join(VIEW_PATH, 'working_queues.erb'))
+      end
+
+      app.get '/working/:queue' do
+        @queue = SidekiqReliableFetch::WorkingQueue.new(params[:queue])
+        erb File.read(File.join(VIEW_PATH, 'working_queue.erb'))
+      end
+    end
+  end
+end
+
+require 'sidekiq/web' unless defined?(Sidekiq::Web)
+Sidekiq::Web.register(SidekiqReliableFetch::Web)
+Sidekiq::Web.tabs['Working'] = 'working'

data/lib/sidekiq/reliable_fetcher.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+module Sidekiq
+  class ReliableFetcher
+    WORKING_QUEUE = 'working'
+    DEFAULT_DEAD_AFTER = 60 * 60 * 24 # 24 hours
+    DEFAULT_CLEANING_INTERVAL = 5000 # clean each N processed jobs
+    IDLE_TIMEOUT = 5 # seconds
+
+    UnitOfWork = Struct.new(:queue, :job) do
+      def acknowledge
+        # NOTE LREM is O(n), so depending on the type of jobs and their average
+        # duration, another data structure might be more suited.
+        # But as there should not be too much jobs in this queue in the same time,
+        # it's probably ok.
+        Sidekiq.redis { |conn| conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job) }
+      end
+
+      def queue_name
+        queue.sub(/.*queue:/, '')
+      end
+
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.lpush(queue, job)
+            conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
+          end
+        end
+      end
+    end
+
+    def self.setup_reliable_fetch!(config)
+      config.options[:fetch] = Sidekiq::ReliableFetcher
+      config.on(:startup) do
+        requeue_on_startup!(config.options[:queues])
+      end
+    end
+
+    def initialize(options)
+      queues = options[:queues].map { |q| "queue:#{q}" }
+
+      @unique_queues = queues.uniq
+      @queues_iterator = queues.shuffle.cycle
+      @queues_size  = queues.size
+
+      @nb_fetched_jobs = 0
+      @cleaning_interval = options[:cleaning_interval] || DEFAULT_CLEANING_INTERVAL
+      @consider_dead_after = options[:consider_dead_after] || DEFAULT_DEAD_AFTER
+
+      Sidekiq.logger.info { "GitLab reliable fetch activated!" }
+    end
+
+    def retrieve_work
+      clean_working_queues! if @cleaning_interval != -1 && @nb_fetched_jobs >= @cleaning_interval
+
+      @queues_size.times do
+        queue = @queues_iterator.next
+        work = Sidekiq.redis { |conn| conn.rpoplpush(queue, "#{queue}:#{WORKING_QUEUE}") }
+
+        if work
+          @nb_fetched_jobs += 1
+          return UnitOfWork.new(queue, work)
+        end
+      end
+
+      # We didn't find a job in any of the configured queues. Let's sleep a bit
+      # to avoid uselessly burning too much CPU
+      sleep(IDLE_TIMEOUT)
+
+      nil
+    end
+
+    def self.requeue_on_startup!(queues)
+      Sidekiq.logger.debug { "Re-queueing working jobs" }
+
+      counter = 0
+
+      Sidekiq.redis do |conn|
+        queues.uniq.each do |queue|
+          while conn.rpoplpush("queue:#{queue}:#{WORKING_QUEUE}", "queue:#{queue}")
+            counter += 1
+          end
+        end
+      end
+
+      Sidekiq.logger.debug { "Re-queued #{counter} jobs" }
+    end
+
+    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
+    # an instance method will make it async to the Fetcher actor
+    def self.bulk_requeue(inprogress, options)
+      return if inprogress.empty?
+
+      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+
+      Sidekiq.redis do |conn|
+        conn.pipelined do
+          inprogress.each do |unit_of_work|
+            conn.lpush("#{unit_of_work.queue}", unit_of_work.job)
+            conn.lrem("#{unit_of_work.queue}:#{WORKING_QUEUE}", 1, unit_of_work.job)
+          end
+        end
+      end
+
+      Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
+    rescue => ex
+      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+    end
+
+    private
+
+    # Detect "old" jobs and requeue them because the worker they were assigned
+    # to probably failed miserably.
+    # NOTE Potential problem here if a specific job always make a worker
+    # really fail.
+    def clean_working_queues!
+      Sidekiq.logger.debug "Cleaning working queues"
+
+      @unique_queues.each do |queue|
+        clean_working_queue!(queue)
+      end
+
+      @nb_fetched_jobs = 0
+    end
+
+    def clean_working_queue!(queue)
+      Sidekiq.redis do |conn|
+        working_jobs = conn.lrange("#{queue}:#{WORKING_QUEUE}", 0, -1)
+        working_jobs.each do |job|
+          enqueued_at = Sidekiq.load_json(job)['enqueued_at'].to_i
+          job_duration = Time.now.to_i - enqueued_at
+
+          next if job_duration < @consider_dead_after
+
+          Sidekiq.logger.info "Requeued a dead job from #{queue}:#{WORKING_QUEUE}"
+
+          conn.lpush("#{queue}", job)
+          conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
+        end
+      end
+    end
+  end
+end

data/web/views/working_queue.erb

@@ -0,0 +1,25 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3><%= t('Working jobs') %></h3>
+  </div>
+</header>
+
+<% if @queue.size > 0 %>
+  <table class="table table-striped table-bordered table-white">
+    <thead>
+      <tr>
+        <th><%= t('Job') %></th>
+        <th><%= t('Arguments') %></th>
+      </tr>
+    </thead>
+    <% @queue.each do |entry| %>
+        <td><%= entry.display_class %></td>
+        <td>
+           <div class="args"><%= display_args(entry.display_args) %></div>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+<% else %>
+  <div class="alert alert-success"><%= t('No working job found') %></div>
+<% end %>

data/web/views/working_queues.erb

@@ -0,0 +1,17 @@
+<h3><%= t('Working queues') %></h3>
+
+<div class="table_container">
+  <table class="queues table table-hover table-bordered table-striped table-white">
+    <thead>
+      <th><%= t('Queue') %></th>
+      <th><%= t('Size') %></th>
+    </thead>
+    <% @queues.each do |queue| %>
+      <tr>
+        <td>
+          <a href="<%= root_path %>working/<%= queue.name %>"><%= queue.name %></a>
+        </td>
+        <td><%= number_with_delimiter(queue.size) %></td>
+      </tr>
+    <% end %>
+</table>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - TEA
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-14 00:00:00.000000000 Z
+date: 2018-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -32,28 +32,17 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".gitlab-ci.yml"
-- ".rspec"
-- Gemfile
-- Gemfile.lock
 - LICENSE
+- README-GITLAB.md
 - README.md
-- RELEASE-GITLAB.md
 - gitlab-sidekiq-fetcher.gemspec
 - lib/sidekiq-reliable-fetch.rb
-- lib/sidekiq/base_reliable_fetch.rb
-- lib/sidekiq/reliable_fetch.rb
-- lib/sidekiq/semi_reliable_fetch.rb
-- spec/base_reliable_fetch_spec.rb
-- spec/fetch_shared_examples.rb
-- spec/reliable_fetch_spec.rb
-- spec/semi_reliable_fetch_spec.rb
-- spec/spec_helper.rb
-- test/README.md
-- test/config.rb
-- test/reliability_test.rb
-- test/worker.rb
-homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
+- lib/sidekiq-reliable-fetch/api.rb
+- lib/sidekiq-reliable-fetch/web.rb
+- lib/sidekiq/reliable_fetcher.rb
+- web/views/working_queue.erb
+- web/views/working_queues.erb
+homepage: https://github.com/TEA-ebook/sidekiq-reliable-fetch
 licenses:
 - LGPL-3.0
 metadata: {}
@@ -73,7 +62,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq

data/.gitlab-ci.yml

@@ -1,53 +0,0 @@
-image: "ruby:2.5"
-
-before_script:
-  - ruby -v
-  - which ruby
-  - gem install bundler --no-ri --no-rdoc
-  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
-
-variables:
-  REDIS_URL: "redis://redis"
-
-rspec:
-  stage: test
-  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
-  script:
-    - bundle exec rspec
-  services:
-    - redis:alpine
-  artifacts:
-    expire_in: 31d
-    when: always
-    paths:
-      - coverage/
-
-.integration:
-  stage: test
-  script:
-    - cd test
-    - bundle exec ruby reliability_test.rb
-  services:
-    - redis:alpine
-
-integration_semi:
-  extends: .integration
-  variables:
-    JOB_FETCHER: semi
-
-integration_reliable:
-  extends: .integration
-  variables:
-    JOB_FETCHER: reliable
-
-
-integration_basic:
-  extends: .integration
-  allow_failure: yes
-  variables:
-    JOB_FETCHER: basic
-
-
-# rubocop:
-#   script:
-#     - bundle exec rubocop