sidekiq-reliable-fetch 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 35c020c4d73f87e39a4b5fa52930e42505c8e866
+  data.tar.gz: 1d344483e74a309efe9db2355450f20a0d5f4487
+SHA512:
+  metadata.gz: 755c4623df1d7d672bb4c857ca53cb4b2e12d9d3e0f1cbbb37b7eb42a4f93dbba639bf91f8588607e7570eef27d50996e637f3a71d6920a8aaacdb62b9d273f5
+  data.tar.gz: 6c76f3015bfc44f03285d151625cf3ee88024b97c8df6c970b72140290948b8625a550fccbf93f90f3291483175dd382d63f8527742269b54046271ac3be62e6

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+*.gem

data/LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/lib/sidekiq/reliable_fetcher.rb

@@ -0,0 +1,134 @@
+module Sidekiq
+  class ReliableFetcher
+    WORKING_QUEUE = 'working'
+    DEFAULT_DEAD_AFTER = 60 * 60 * 24 # 24 hours
+    DEFAULT_CLEANING_INTERVAL = 60 * 30 # 30 minutes
+
+    def self.setup_reliable_fetch!(config)
+      config.options[:fetch] = Sidekiq::ReliableFetcher
+      config.on(:startup) do
+        requeue_on_startup!(config.options[:queues])
+      end
+    end
+
+    def initialize(options)
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues].map { |q| "queue:#{q}" }
+      @unique_queues = @queues.uniq
+      @last_clean = Time.now.to_i
+      @cleaning_interval = options[:cleaning_interval] || DEFAULT_CLEANING_INTERVAL
+      @consider_dead_after = options[:consider_dead_after] || DEFAULT_DEAD_AFTER
+    end
+
+    def retrieve_work
+      clean_working_queues! if @cleaning_interval != -1 && Time.now.to_i - @last_clean > @cleaning_interval
+
+      queue = pick_queue
+      work = Sidekiq.redis { |conn| conn.brpoplpush(queue, "#{queue}:#{WORKING_QUEUE}", Sidekiq::Fetcher::TIMEOUT) }
+      UnitOfWork.new(queue, work) if work
+    end
+
+    def self.requeue_on_startup!(queues)
+      Sidekiq.logger.debug { "Re-queueing working jobs" }
+
+      counter = 0
+
+      Sidekiq.redis do |conn|
+        queues.uniq.each do |queue|
+          while conn.rpoplpush("queue:#{queue}:#{WORKING_QUEUE}", "queue:#{queue}")
+            counter += 1
+          end
+        end
+      end
+
+      Sidekiq.logger.debug { "Re-queued #{counter} jobs" }
+    end
+
+    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
+    # an instance method will make it async to the Fetcher actor
+    def self.bulk_requeue(inprogress, options)
+      return if inprogress.empty?
+
+      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+
+      Sidekiq.redis do |conn|
+        conn.pipelined do
+          inprogress.each do |unit_of_work|
+            conn.rpush("#{unit_of_work.queue}", unit_of_work.message)
+            conn.lrem("#{unit_of_work.queue}:#{WORKING_QUEUE}", 1, unit_of_work.message)
+          end
+        end
+      end
+
+      Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
+    rescue => ex
+      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+    end
+
+    UnitOfWork = Struct.new(:queue, :message) do
+      def acknowledge
+        # NOTE LREM is O(n), so depending on the type of jobs and their average
+        # duration, another data structure might be more suited.
+        # But as there should not be too much jobs in this queue in the same time,
+        # it's probably ok.
+        Sidekiq.redis { |conn| conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, message) }
+      end
+
+      def queue_name
+        queue.gsub(/.*queue:/, '')
+      end
+
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.rpush(queue, message)
+            conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, message)
+          end
+        end
+      end
+    end
+
+    private
+
+    # Creating the Redis#brpoplpush command takes into account any
+    # configured queue weights. By default Redis#brpoplpush returns
+    # data from the first queue that has pending elements. We
+    # choose the queue each time we invoke Redis#brpoplpush to honor weights
+    # and avoid queue starvation.
+    def pick_queue
+      queues = @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq
+      queues.first
+    end
+
+    # Detect "old" jobs and requeue them because the worker they were assigned
+    # to probably failed miserably.
+    # NOTE Potential problem here if a specific job always make a worker
+    # really fail.
+    def clean_working_queues!
+      Sidekiq.logger.debug "Cleaning working queues"
+
+      @unique_queues.each do |queue|
+        clean_working_queue!(queue)
+      end
+
+      @last_clean = Time.now.to_i
+    end
+
+    def clean_working_queue!(queue)
+      Sidekiq.redis do |conn|
+        working_jobs = conn.lrange("#{queue}:#{WORKING_QUEUE}", 0, -1)
+        working_jobs.each do |job|
+          enqueued_at = Sidekiq.load_json(job)['enqueued_at'].to_i
+          job_duration = Time.now.to_i - enqueued_at
+
+          if job_duration > @consider_dead_after
+            Sidekiq.logger.info "Requeued a dead job found in #{queue}:#{WORKING_QUEUE}"
+
+            conn.rpush("#{queue}", job)
+            conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-reliable-fetch/api.rb

@@ -0,0 +1,56 @@
+module SidekiqReliableFetch
+  ##
+  # Encapsulates a working queue within Sidekiq.
+  # Allows enumeration of all jobs within the queue.
+  #
+  #   queue = SidekiqReliableFetch::WorkingQueue.new("mailer")
+  #   queue.each do |job|
+  #     job.klass # => 'MyWorker'
+  #     job.args # => [1, 2, 3]
+  #   end
+  #
+  class WorkingQueue
+    include Enumerable
+
+    def self.all
+      Sidekiq.redis { |c| c.keys('queue:*:working') }
+        .sort
+        .map { |q| SidekiqReliableFetch::WorkingQueue.new(q) }
+    end
+
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def size
+      Sidekiq.redis { |con| con.llen(@name) }
+    end
+
+    def each
+      initial_size = size
+      deleted_size = 0
+      page = 0
+      page_size = 50
+
+      loop do
+        range_start = page * page_size - deleted_size
+        range_end   = page * page_size - deleted_size + (page_size - 1)
+        entries = Sidekiq.redis do |conn|
+          conn.lrange @name, range_start, range_end
+        end
+        break if entries.empty?
+        page += 1
+        entries.each do |entry|
+          yield Sidekiq::Job.new(entry, @name)
+        end
+        deleted_size = initial_size - size
+      end
+    end
+
+    def find_job(jid)
+      detect { |j| j.jid == jid }
+    end
+  end
+end

data/lib/sidekiq-reliable-fetch/web.rb

@@ -0,0 +1,25 @@
+require_relative 'api'
+
+module SidekiqReliableFetch
+  # Hook into *Sidekiq::Web* Sinatra app which adds a new '/working' page
+  module Web
+    VIEW_PATH = File.expand_path('../../../web/views', __FILE__)
+
+    def self.registered(app)
+      app.get '/working' do
+        puts VIEW_PATH.inspect
+        @queues = SidekiqReliableFetch::WorkingQueue.all
+        erb File.read(File.join(VIEW_PATH, 'working_queues.erb'))
+      end
+
+      app.get '/working/:queue' do
+        @queue = SidekiqReliableFetch::WorkingQueue.new(params[:queue])
+        erb File.read(File.join(VIEW_PATH, 'working_queue.erb'))
+      end
+    end
+  end
+end
+
+require 'sidekiq/web' unless defined?(Sidekiq::Web)
+Sidekiq::Web.register(SidekiqReliableFetch::Web)
+Sidekiq::Web.tabs['Working'] = 'working'

data/lib/sidekiq-reliable-fetch.rb

@@ -0,0 +1,4 @@
+require 'sidekiq'
+
+require_relative 'sidekiq/reliable_fetcher'
+require_relative 'sidekiq-reliable-fetch/web'

data/sidekiq-reliable-fetch.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name        = 'sidekiq-reliable-fetch'
+  s.version     = '0.1.0'
+  s.authors     = ['TEA']
+  s.email       = 'technique@tea-ebook.com'
+  s.license     = 'LGPL-3.0'
+  s.homepage    = 'https://github.com/TEA-ebook/sidekiq-reliable-fetch'
+  s.summary     = 'Reliable fetch extension for Sidekiq'
+  s.description = 'Redis reliable queue pattern implemented in Sidekiq'
+  s.require_paths = ['lib']
+
+  s.files = `git ls-files`.split($\)
+  s.test_files  = []
+
+  s.add_dependency 'sidekiq', '~> 3'
+end

data/web/views/working_queue.erb

@@ -0,0 +1,25 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3><%= t('Working jobs') %></h3>
+  </div>
+</header>
+
+<% if @queue.size > 0 %>
+  <table class="table table-striped table-bordered table-white">
+    <thead>
+      <tr>
+        <th><%= t('Job') %></th>
+        <th><%= t('Arguments') %></th>
+      </tr>
+    </thead>
+    <% @queue.each do |entry| %>
+        <td><%= entry.display_class %></td>
+        <td>
+           <div class="args"><%= display_args(entry.display_args) %></div>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+<% else %>
+  <div class="alert alert-success"><%= t('No working job found') %></div>
+<% end %>

data/web/views/working_queues.erb

@@ -0,0 +1,17 @@
+<h3><%= t('Working queues') %></h3>
+
+<div class="table_container">
+  <table class="queues table table-hover table-bordered table-striped table-white">
+    <thead>
+      <th><%= t('Queue') %></th>
+      <th><%= t('Size') %></th>
+    </thead>
+    <% @queues.each do |queue| %>
+      <tr>
+        <td>
+          <a href="<%= root_path %>working/<%= queue.name %>"><%= queue.name %></a>
+        </td>
+        <td><%= number_with_delimiter(queue.size) %></td>
+      </tr>
+    <% end %>
+</table>

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-reliable-fetch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- TEA
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+description: Redis reliable queue pattern implemented in Sidekiq
+email: technique@tea-ebook.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE
+- lib/sidekiq-reliable-fetch.rb
+- lib/sidekiq-reliable-fetch/api.rb
+- lib/sidekiq-reliable-fetch/web.rb
+- lib/sidekiq/reliable_fetcher.rb
+- sidekiq-reliable-fetch.gemspec
+- web/views/working_queue.erb
+- web/views/working_queues.erb
+homepage: https://github.com/TEA-ebook/sidekiq-reliable-fetch
+licenses:
+- LGPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Reliable fetch extension for Sidekiq
+test_files: []
+has_rdoc: