resque-fifo-queue 0.1.1

data/lib/resque/plugins/fifo/queue/version.rb

@@ -0,0 +1,9 @@
+module Resque
+  module Plugins
+    module Fifo
+      module Queue
+        VERSION = "0.1.1"
+      end
+    end
+  end
+end

data/lib/resque/plugins/fifo/server.rb

@@ -0,0 +1,83 @@
+require 'resque/server'
+
+module Resque
+  module Plugins
+    module Fifo
+      module Server
+        VIEW_PATH = File.join(File.dirname(__FILE__), 'server', 'views')
+
+        module Helpers
+          def poll
+            if @polling
+              text = "Last Updated: #{Time.now.strftime("%H:%M:%S")}"
+            else
+              text = "<a href='#{u(request.path_info)}.poll' rel='poll'>Live Poll</a>"
+            end
+            "<p class='poll'>#{text}</p>"
+          end
+
+          def show_page(page, layout = true)
+            response["Cache-Control"] = "max-age=0, private, must-revalidate"
+            begin
+              erb(File.read(File.join(VIEW_PATH, page)), {:layout => layout})
+            rescue Errno::ECONNREFUSED
+              erb :error, {:layout => false}, :error => "Can't connect to Redis! (#{Resque.redis_id})"
+            end
+          end
+        end
+
+        class << self
+          def registered(app)
+            app.get '/fifo_queues' do
+              @refresh_requested = false
+              @manager = Resque::Plugins::Fifo::Queue::Manager.new
+              @queue_with_slices = @manager.dump_queues_with_slices
+              @orphaned_queues = @manager.orphaned_queues
+              show_page('fifo_queues.erb')
+            end
+
+            app.get '/fifo_queues.poll' do
+              @polling = true
+              @refresh_requested = false
+              @manager = Resque::Plugins::Fifo::Queue::Manager.new
+              @orphaned_queues = @manager.orphaned_queues
+              @queue_with_slices = @manager.dump_queues_with_slices
+              show_page('fifo_queues.erb', false)
+            end
+
+            app.post '/shared_finder' do
+              @manager = Resque::Plugins::Fifo::Queue::Manager.new
+              @queue_name = @manager.compute_queue_name(params[:key])
+              @worker = @manager.worker_for_queue(@queue_name)
+              erb(File.read(File.join(VIEW_PATH, 'shared_finder.erb')))
+            end
+
+            app.post '/request_update' do
+              @manager = Resque::Plugins::Fifo::Queue::Manager.new
+              @manager.request_refresh
+              @refresh_requested = true
+              @queue_with_slices = @manager.dump_queues_with_slices
+              redirect '/resque/fifo_queues'
+            end
+
+            app.post '/clear_stats' do
+              @manager = Resque::Plugins::Fifo::Queue::Manager.new
+              @manager.clear_stats
+              @refresh_requested = true
+              @queue_with_slices = @manager.dump_queues_with_slices
+              redirect '/resque/fifo_queues'
+            end
+            # We have little choice in using this funky name - Resque
+            # already has a "Stats" tab, and it doesn't like
+            # tab names with spaces in it (it translates the url as job%20stats)
+            app.tabs << "FIFO_Queues"
+
+            app.helpers(Helpers)
+          end
+        end
+      end
+    end
+  end
+end
+
+Resque::Server.register Resque::Plugins::Fifo::Server

data/lib/resque/plugins/fifo/server/views/fifo_queues.erb

@@ -0,0 +1,109 @@
+<h1>Resque FIFO Managed Queues version: <%= Resque::Plugins::Fifo::Queue::VERSION %></h1>
+<% if @refresh_requested %>
+<p class="intro">
+  Updating worker list... Refresh to view updates.
+</p>
+<% end %>
+<p class="intro">
+  This page displays statistics about available FIFO Managed Queues and its partitioning information.
+</p>
+<form method="POST" action="<%=u "/shared_finder" %>" >
+  <label for="key">Key</label>
+  <input id="key" type='text' name='key'/>
+  <input type='submit' name='' value='Find Shard' />
+</form>
+<h1>Perfomance Statistics</h1>
+<p>Contains statistics related to the performance of the FIFO queues</p>
+<form method="POST" action="<%=u "/clear_stats" %>" >
+  <input type='submit' name='' value='Reset Performance statistics' />
+</form>
+<table>
+  <thead>
+    <th>Name</th>
+    <th>Value</th>
+    <th>Description</th>
+  </thead>
+  <tbody>
+    <tr>
+      <td>Max Job Delay</td>
+      <td><%= @manager.get_stats_max_delay %></td>
+      <td>The worst case in ms that a job had waited before it gets processed.</td>
+    </tr>
+    <tr>
+      <td>Avg Job Delay</td>
+      <td><%= @manager.get_stats_avg_delay %></td>
+      <td>Average wait time in ms, before a job gets processed</td>
+    </tr>
+    <tr>
+      <td>Total DHT rehash</td>
+      <td><%= @manager.dht_times_rehashed %></td>
+      <td>Total number of times the hash table was rehashed</td>
+    </tr>
+    <tr>
+      <td>Avg DHT recalculate time</td>
+      <td><%= @manager.get_stats_avg_dht_recalc %></td>
+      <td>Average time it takes the system to rehash the DHT</td>
+    </tr>
+  </tbody>
+</table>
+<h1>Pending Queue Information</h1>
+<p> This queue contains jobs that are to be resharded</p>
+<table>
+  <thead>
+    <th>Queue Name</th>
+    <th>Stored Jobs</th>
+  </thead>
+  <tbody>
+    <tr>
+      <td><a class="queue" href="<%= u "queues/#{@manager.pending_queue_name}" %>"><%= @manager.pending_queue_name %></a></td>
+      <td><%= @manager.pending_total %></td>
+  </tbody>
+</table>
+<h1 class='wi'> <%= @queue_with_slices.size %> worker instances</h1>
+<form method="POST" action="<%=u "/request_update" %>" >
+  <input type='submit' name='' value='Request Force Update' />
+</form>
+<table>
+  <thead>
+    <th>Slice (0 -- <%= 2**32 %>)</th>
+    <th>Queue Name</th>
+    <th>Worker Host</th>
+    <th>Status</th>
+    <th>Started</th>
+    <th>Heartbeat</th>
+    <th>Assigned</th>
+    <th>Queued</th>
+  </thead>
+  <tbody>
+    <% @queue_with_slices.each do |slice_info| %>
+      <tr>
+        <td>#<%= slice_info[0] %></td>
+        <td><a class="queue" href="<%= u "queues/#{slice_info[1]}" %>"><%= slice_info[1] %></a></td>
+        <td><a href="<%=u "workers/#{slice_info[2]}"%>"><%= slice_info[2] %>:<%= slice_info[3] %></a></td>
+        <td><%= slice_info[4] %></td>
+        <td><%= slice_info[5] %></td>
+        <td><%= slice_info[6] %></td>
+        <td><%= slice_info[7] %></td>
+        <td><%= slice_info[8] %></td>
+      </tr>
+    <% end %>
+    <tr></tr>
+  </tbody>
+</table>
+<% if @orphaned_queues.size > 0 %>
+  <h1 class="wi">Orphaned Queues</h1>
+  <table>
+    <thead>
+      <th>Queue Name</th>
+    </thead>
+    <tbody>
+      <% @orphaned_queues.each do |queue| %>
+        <tr>
+          <td><a class="queue" href="<%= u "queues/#{queue}" %>"><%= queue %></a></td>
+        </tr>
+      <% end %>
+      <tr></tr>
+    </tbody>
+  </table>
+<% end %>
+<%= poll %>

data/lib/resque/plugins/fifo/server/views/shared_finder.erb

@@ -0,0 +1,21 @@
+<form method="POST" action="<%=u "/shared_finder" %>" >
+  <label for="key">Key</label>
+  <input id="key" type='text' name='key'/>
+  <input type='submit' name='' value='Find Shard' />
+</form>
+<% if @worker %>
+<table>
+  <thead>
+    <th>Queue Name</th>
+    <th>Host Name</th>
+  </thead>
+  <tbody>
+    <tr>
+      <td><%= @queue_anme %></td>
+      <td><%= @worker.hostname %></td>
+    </tr>
+  </tbody>
+</table>
+<% else %>
+No worker found
+<% end %>

data/lib/resque/plugins/fifo/worker.rb

@@ -0,0 +1,73 @@
+require 'securerandom'
+
+module Resque
+  module Plugins
+    module Fifo
+      class Worker < Resque::Worker
+        UPDATE_DELAY = 10
+        attr_accessor :main_queue_name
+
+        def queues=(queues)
+          queues = queues.empty? ? (ENV["QUEUES"] || ENV['QUEUE']).to_s.split(',') : queues
+          @main_queue_name = "#{manager.queue_prefix}-#{SecureRandom.hex(10)}"
+
+          @queues = ([:fifo_refresh, main_queue_name] + queues).map { |queue| queue.to_s.strip }
+          unless ['*', '?', '{', '}', '[', ']'].any? { |char| @queues.join.include?(char) }
+            @static_queues = @queues.flatten.uniq
+          end
+          validate_queues
+        end
+
+        # Attempts to grab a job off one of the provided queues. Returns
+        # nil if no job can be found.
+        def reserve
+          queues.each do |queue|
+            log_with_severity :debug, "Checking #{queue}"
+            if job = Resque.reserve(queue)
+              log_with_severity :debug, "Found job on #{queue}"
+
+              if job.payload['enqueue_ts']
+                delay_ts = Time.now.to_i - job.payload['enqueue_ts'].to_i
+                max_delay = Resque.redis.get("fifo-stats-max-delay") || 0
+                Resque.redis.incrby("fifo-stats-accumulated-delay", delay_ts)
+                Resque.redis.incr("fifo-stats-accumulated-count")
+                if (delay_ts > max_delay.to_i)
+                  Resque.redis.set("fifo-stats-max-delay", max_delay)
+                end
+              end
+              return job
+            end
+          end
+
+          nil
+        rescue Exception => e
+          log_with_severity :error, "Error reserving job: #{e.inspect}"
+          log_with_severity :error, e.backtrace.join("\n")
+          raise e
+        end
+
+        # Registers ourself as a worker. Useful when entering the worker
+        # lifecycle on startup.
+        def register_worker
+          super
+
+          puts "Fifo Startup - Updating worker list"
+          manager.request_refresh
+        end
+
+        def unregister_worker(exception = nil)
+          super(exception)
+
+          puts "Fifo Shutdown - Updating worker list"
+          manager.request_refresh
+        end
+
+        private
+
+        def manager
+          @manager ||=  Resque::Plugins::Fifo::Queue::Manager.new
+        end
+      end
+    end
+  end
+end

data/lib/tasks/resque_fifo.rake

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+
+require 'resque/fifo/queue'
+require 'resque/fifo/tasks'

data/resque-fifo-queue.gemspec

@@ -0,0 +1,45 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque/plugins/fifo/queue/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "resque-fifo-queue"
+  spec.version       = Resque::Plugins::Fifo::Queue::VERSION
+  spec.authors       = ["Joseph Emmanuel Dayo"]
+  spec.email         = ["joseph.dayo@gmail.com"]
+
+  spec.summary       = %q{Implements a federated fifo queue with consistent hashing on top of resque}
+  spec.description   = %q{Implements a federated fifo queue with consistent hashing on top of resque}
+  spec.homepage      = "https://github.com/jedld/resque-fifo-queue"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "timecop"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_dependency "redis"
+  spec.add_dependency "resque", "~> 1.27"
+  spec.add_dependency "resque-scheduler"
+  spec.add_dependency "resque-pause"
+  spec.add_dependency "resque_solo"
+  spec.add_dependency "xxhash"
+  spec.add_dependency "redlock"
+end

metadata

@@ -0,0 +1,237 @@
+--- !ruby/object:Gem::Specification
+name: resque-fifo-queue
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Joseph Emmanuel Dayo
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-11-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.27'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.27'
+- !ruby/object:Gem::Dependency
+  name: resque-scheduler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resque-pause
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resque_solo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: xxhash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Implements a federated fifo queue with consistent hashing on top of resque
+email:
+- joseph.dayo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- init.rb
+- lib/resque/fifo/constants.rb
+- lib/resque/fifo/queue.rb
+- lib/resque/fifo/tasks.rb
+- lib/resque/plugins/fifo/extensions.rb
+- lib/resque/plugins/fifo/queue/drain_worker.rb
+- lib/resque/plugins/fifo/queue/manager.rb
+- lib/resque/plugins/fifo/queue/version.rb
+- lib/resque/plugins/fifo/server.rb
+- lib/resque/plugins/fifo/server/views/fifo_queues.erb
+- lib/resque/plugins/fifo/server/views/shared_finder.erb
+- lib/resque/plugins/fifo/worker.rb
+- lib/tasks/resque_fifo.rake
+- resque-fifo-queue.gemspec
+homepage: https://github.com/jedld/resque-fifo-queue
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.10
+signing_key: 
+specification_version: 4
+summary: Implements a federated fifo queue with consistent hashing on top of resque
+test_files: []