gitlab-sidekiq-fetcher 0.3.0 → 0.4.0

data/test/worker.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class TestWorker
+  include Sidekiq::Worker
+
+  def perform
+    # To mimic long running job and to increase the probability of losing the job
+    sleep 1
+
+    Sidekiq.redis do |redis|
+      redis.lpush(REDIS_FINISHED_LIST, get_sidekiq_job_id)
+    end
+  end
+
+  def get_sidekiq_job_id
+    context_data = Thread.current[:sidekiq_context]&.first
+
+    return unless context_data
+
+    index = context_data.index('JID-')
+
+    return unless index
+
+    context_data[index + 4..-1]
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - TEA
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-13 00:00:00.000000000 Z
+date: 2018-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -32,17 +32,28 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".gitlab-ci.yml"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
 - LICENSE
-- README-GITLAB.md
 - README.md
+- RELEASE-GITLAB.md
 - gitlab-sidekiq-fetcher.gemspec
 - lib/sidekiq-reliable-fetch.rb
-- lib/sidekiq-reliable-fetch/api.rb
-- lib/sidekiq-reliable-fetch/web.rb
-- lib/sidekiq/reliable_fetcher.rb
-- web/views/working_queue.erb
-- web/views/working_queues.erb
-homepage: https://github.com/TEA-ebook/sidekiq-reliable-fetch
+- lib/sidekiq/base_reliable_fetch.rb
+- lib/sidekiq/reliable_fetch.rb
+- lib/sidekiq/semi_reliable_fetch.rb
+- spec/base_reliable_fetch_spec.rb
+- spec/fetch_shared_examples.rb
+- spec/reliable_fetch_spec.rb
+- spec/semi_reliable_fetch_spec.rb
+- spec/spec_helper.rb
+- test/README.md
+- test/config.rb
+- test/reliability_test.rb
+- test/worker.rb
+homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
 metadata: {}
@@ -62,7 +73,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq

data/lib/sidekiq-reliable-fetch/api.rb

@@ -1,56 +0,0 @@
-module SidekiqReliableFetch
-  ##
-  # Encapsulates a working queue within Sidekiq.
-  # Allows enumeration of all jobs within the queue.
-  #
-  #   queue = SidekiqReliableFetch::WorkingQueue.new("mailer")
-  #   queue.each do |job|
-  #     job.klass # => 'MyWorker'
-  #     job.args # => [1, 2, 3]
-  #   end
-  #
-  class WorkingQueue
-    include Enumerable
-
-    def self.all
-      Sidekiq.redis { |c| c.keys('queue:*:working') }
-        .sort
-        .map { |q| SidekiqReliableFetch::WorkingQueue.new(q) }
-    end
-
-    attr_reader :name
-
-    def initialize(name)
-      @name = name
-    end
-
-    def size
-      Sidekiq.redis { |con| con.llen(@name) }
-    end
-
-    def each
-      initial_size = size
-      deleted_size = 0
-      page = 0
-      page_size = 50
-
-      loop do
-        range_start = page * page_size - deleted_size
-        range_end   = page * page_size - deleted_size + (page_size - 1)
-        entries = Sidekiq.redis do |conn|
-          conn.lrange @name, range_start, range_end
-        end
-        break if entries.empty?
-        page += 1
-        entries.each do |entry|
-          yield Sidekiq::Job.new(entry, @name)
-        end
-        deleted_size = initial_size - size
-      end
-    end
-
-    def find_job(jid)
-      detect { |j| j.jid == jid }
-    end
-  end
-end

data/lib/sidekiq-reliable-fetch/web.rb

@@ -1,24 +0,0 @@
-require_relative 'api'
-
-module SidekiqReliableFetch
-  # Hook into *Sidekiq::Web* Sinatra app which adds a new '/working' page
-  module Web
-    VIEW_PATH = File.expand_path('../../../web/views', __FILE__)
-
-    def self.registered(app)
-      app.get '/working' do
-        @queues = SidekiqReliableFetch::WorkingQueue.all
-        erb File.read(File.join(VIEW_PATH, 'working_queues.erb'))
-      end
-
-      app.get '/working/:queue' do
-        @queue = SidekiqReliableFetch::WorkingQueue.new(params[:queue])
-        erb File.read(File.join(VIEW_PATH, 'working_queue.erb'))
-      end
-    end
-  end
-end
-
-require 'sidekiq/web' unless defined?(Sidekiq::Web)
-Sidekiq::Web.register(SidekiqReliableFetch::Web)
-Sidekiq::Web.tabs['Working'] = 'working'

data/lib/sidekiq/reliable_fetcher.rb

@@ -1,143 +0,0 @@
-# frozen_string_literal: true
-module Sidekiq
-  class ReliableFetcher
-    WORKING_QUEUE = 'working'
-    DEFAULT_DEAD_AFTER = 60 * 60 * 24 # 24 hours
-    DEFAULT_CLEANING_INTERVAL = 5000 # clean each N processed jobs
-    IDLE_TIMEOUT = 5 # seconds
-
-    UnitOfWork = Struct.new(:queue, :job) do
-      def acknowledge
-        # NOTE LREM is O(n), so depending on the type of jobs and their average
-        # duration, another data structure might be more suited.
-        # But as there should not be too much jobs in this queue in the same time,
-        # it's probably ok.
-        Sidekiq.redis { |conn| conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job) }
-      end
-
-      def queue_name
-        queue.sub(/.*queue:/, '')
-      end
-
-      def requeue
-        Sidekiq.redis do |conn|
-          conn.pipelined do
-            conn.lpush(queue, job)
-            conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
-          end
-        end
-      end
-    end
-
-    def self.setup_reliable_fetch!(config)
-      config.options[:fetch] = Sidekiq::ReliableFetcher
-      config.on(:startup) do
-        requeue_on_startup!(config.options[:queues])
-      end
-    end
-
-    def initialize(options)
-      queues = options[:queues].map { |q| "queue:#{q}" }
-
-      @unique_queues = queues.uniq
-      @queues_iterator = queues.shuffle.cycle
-      @queues_size  = queues.size
-
-      @nb_fetched_jobs = 0
-      @cleaning_interval = options[:cleaning_interval] || DEFAULT_CLEANING_INTERVAL
-      @consider_dead_after = options[:consider_dead_after] || DEFAULT_DEAD_AFTER
-
-      Sidekiq.logger.info { "GitLab reliable fetch activated!" }
-    end
-
-    def retrieve_work
-      clean_working_queues! if @cleaning_interval != -1 && @nb_fetched_jobs >= @cleaning_interval
-
-      @queues_size.times do
-        queue = @queues_iterator.next
-        work = Sidekiq.redis { |conn| conn.rpoplpush(queue, "#{queue}:#{WORKING_QUEUE}") }
-
-        if work
-          @nb_fetched_jobs += 1
-          return UnitOfWork.new(queue, work)
-        end
-      end
-
-      # We didn't find a job in any of the configured queues. Let's sleep a bit
-      # to avoid uselessly burning too much CPU
-      sleep(IDLE_TIMEOUT)
-
-      nil
-    end
-
-    def self.requeue_on_startup!(queues)
-      Sidekiq.logger.debug { "Re-queueing working jobs" }
-
-      counter = 0
-
-      Sidekiq.redis do |conn|
-        queues.uniq.each do |queue|
-          while conn.rpoplpush("queue:#{queue}:#{WORKING_QUEUE}", "queue:#{queue}")
-            counter += 1
-          end
-        end
-      end
-
-      Sidekiq.logger.debug { "Re-queued #{counter} jobs" }
-    end
-
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
-      return if inprogress.empty?
-
-      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-
-      Sidekiq.redis do |conn|
-        conn.pipelined do
-          inprogress.each do |unit_of_work|
-            conn.lpush("#{unit_of_work.queue}", unit_of_work.job)
-            conn.lrem("#{unit_of_work.queue}:#{WORKING_QUEUE}", 1, unit_of_work.job)
-          end
-        end
-      end
-
-      Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
-    rescue => ex
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
-    end
-
-    private
-
-    # Detect "old" jobs and requeue them because the worker they were assigned
-    # to probably failed miserably.
-    # NOTE Potential problem here if a specific job always make a worker
-    # really fail.
-    def clean_working_queues!
-      Sidekiq.logger.debug "Cleaning working queues"
-
-      @unique_queues.each do |queue|
-        clean_working_queue!(queue)
-      end
-
-      @nb_fetched_jobs = 0
-    end
-
-    def clean_working_queue!(queue)
-      Sidekiq.redis do |conn|
-        working_jobs = conn.lrange("#{queue}:#{WORKING_QUEUE}", 0, -1)
-        working_jobs.each do |job|
-          enqueued_at = Sidekiq.load_json(job)['enqueued_at'].to_i
-          job_duration = Time.now.to_i - enqueued_at
-
-          next if job_duration < @consider_dead_after
-
-          Sidekiq.logger.info "Requeued a dead job from #{queue}:#{WORKING_QUEUE}"
-
-          conn.lpush("#{queue}", job)
-          conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
-        end
-      end
-    end
-  end
-end

data/web/views/working_queue.erb

@@ -1,25 +0,0 @@
-<header class="row">
-  <div class="col-sm-5">
-    <h3><%= t('Working jobs') %></h3>
-  </div>
-</header>
-
-<% if @queue.size > 0 %>
-  <table class="table table-striped table-bordered table-white">
-    <thead>
-      <tr>
-        <th><%= t('Job') %></th>
-        <th><%= t('Arguments') %></th>
-      </tr>
-    </thead>
-    <% @queue.each do |entry| %>
-        <td><%= entry.display_class %></td>
-        <td>
-           <div class="args"><%= display_args(entry.display_args) %></div>
-        </td>
-      </tr>
-    <% end %>
-  </table>
-<% else %>
-  <div class="alert alert-success"><%= t('No working job found') %></div>
-<% end %>

data/web/views/working_queues.erb

@@ -1,17 +0,0 @@
-<h3><%= t('Working queues') %></h3>
-
-<div class="table_container">
-  <table class="queues table table-hover table-bordered table-striped table-white">
-    <thead>
-      <th><%= t('Queue') %></th>
-      <th><%= t('Size') %></th>
-    </thead>
-    <% @queues.each do |queue| %>
-      <tr>
-        <td>
-          <a href="<%= root_path %>working/<%= queue.name %>"><%= queue.name %></a>
-        </td>
-        <td><%= number_with_delimiter(queue.size) %></td>
-      </tr>
-    <% end %>
-</table>