sidekiq-dynamic-queues 0.5.0

data/.gitignore

@@ -0,0 +1,9 @@
+*.gem
+*.DS_Store
+.bundle
+Gemfile.lock
+pkg/*
+.idea
+spec/dump.rdb
+spec/redis-server.log
+spec/redis.pid

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - jruby-19mode # JRuby in 1.9 mode
+  # - rbx-19mode
+script: bundle exec rspec spec

data/CHANGELOG

@@ -0,0 +1,5 @@
+0.5.0
+-----
+
+Initial version ported from resque-dynamic-queues 0.8.1
+

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in sidekiq-dynamic-queues.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+Copyright (c) 2013 Matt Conway (matt@conwaysplace.com)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+A sidekiq plugin for specifying the queues a worker pulls from with wildcards, negations, or dynamic look up from redis.
+
+Authored against Sidekiq 2.9.0, so it at least works with that - try running the tests if you use a different version of sidekiq
+
+[![Build Status](https://secure.travis-ci.org/wr0ngway/sidekiq-dynamic-queues.png)](http://travis-ci.org/wr0ngway/sidekiq-dynamic-queues)
+
+Usage:
+
+If creating a gem of your own that uses sidekiq-dynamic-queues, you may have to add an explicit require statement at the top of your Rakefile:
+
+    require 'sidekiq-dynamic-queues'
+
+Configure by setting Sidekiq.options[:fetch] = Sidekiq::DynamicQueues::Fetch
+
+Start your workers with a QUEUE that can contain '\*' for zero-or more of any character, '!' to exclude the following pattern, or @key to look up the patterns from redis.  Some examples help:
+
+    sidekiq -q foo
+
+Pulls jobs from the queue 'foo'
+
+    sidekiq -q *
+
+Pulls jobs from any queue
+
+    sidekiq -q *foo
+
+Pulls jobs from queues that end in foo
+
+    sidekiq -q *foo*
+
+Pulls jobs from queues whose names contain foo
+
+    sidekiq -q *foo* -q !foobar
+
+Pulls jobs from queues whose names contain foo except the foobar queue
+
+    sidekiq -q *foo* -q !*bar
+
+Pulls jobs from queues whose names contain foo except queues whose names end in bar
+
+    QUEUE='@key' sidekiq -q @key
+
+Pulls jobs from queue names stored in redis (use Sidekiq::DynamicQueues::Attributes.set\_dynamic\_queue("key", ["queuename1", "queuename2"]) to set them)
+
+    sidekiq -q * -q !@key
+
+Pulls jobs from any queue except ones stored in redis
+
+    sidekiq -q @
+
+Pulls jobs from queue names stored in redis using the hostname of the worker
+
+    Sidekiq::DynamicQueues::Attributes.set_dynamic_queue("key", ["*foo*", "!*bar"])
+    sidekiq -q @key 
+
+Pulls jobs from queue names stored in redis, with wildcards/negations
+
+
+There is also a tab in the sidekiq-web UI that allows you to define the dynamic queues  To activate it, you need to require 'sidekiq-dynamic-queues-server' in whatever initializer you use to bring up sidekiq-web.
+
+
+Contributors:
+
+Matt Conway ( https://github.com/wr0ngway )

data/Rakefile

@@ -0,0 +1,67 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+task :my_release => ['changelog', 'release'] do
+end
+
+task :changelog do
+
+  helper = Bundler::GemHelper.new(Dir.pwd)
+  version = "v#{helper.gemspec.version}"
+
+  changelog_file = 'CHANGELOG'
+  entries = ""
+
+  # Get a list of current tags
+  tags = `git tag -l`.split
+  tags = tags.sort_by {|t| t[1..-1].split(".").collect {|s| s.to_i } }
+  newest_tag = tags[-1]
+  
+  if version == newest_tag
+    puts "You need to update version, same as most recent tag: #{version}"
+    exit
+  end
+
+  # If we already have a changelog, make the last tag be the
+  # last one in the changelog, and the next one be the one
+  # following that in the tag list
+  newest_changelog_version = nil
+  if File.exist?(changelog_file)
+    entries = File.read(changelog_file)
+    head = entries.split.first
+    if head =~ /\d\.\d\.\d/
+      newest_changelog_version = "v#{head}"
+
+      if version == newest_changelog_version
+        puts "You need to update version, same as most recent changelog: #{version}"
+        exit
+      end
+
+    end
+  end
+
+  # Generate changelog from repo
+  log=`git log --pretty='format:%s <%h> [%cn]' #{newest_tag}..HEAD`
+
+  # Strip out maintenance entries
+  log = log.lines.to_a.delete_if do |l|
+     l =~ /^Regenerated? gemspec/ ||
+         l =~ /^version bump/i ||
+         l =~ /^Updated changelog/ ||
+         l =~ /^Merged? branch/
+  end
+
+  # Write out changelog file
+  File.open(changelog_file, 'w') do |out|
+    out.puts version.gsub(/^v/, '')
+    out.puts "-----"
+    out.puts "\n"
+    out.puts log
+    out.puts "\n"
+    out.puts entries
+  end
+
+  # Commit and push
+  sh "git ci -m'Updated changelog' #{changelog_file}"
+  sh "git push"
+end

data/config.ru

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+require 'logger'
+
+$LOAD_PATH.unshift ::File.expand_path(::File.dirname(__FILE__) + '/lib')
+require 'sidekiq/web'
+require 'sidekiq-dynamic-queues-server'
+
+use Rack::ShowExceptions
+run Sidekiq::Web

data/init.rb

@@ -0,0 +1 @@
+require 'sidekiq-dynamic-queues'

data/lib/sidekiq-dynamic-queues-server.rb

@@ -0,0 +1,5 @@
+require 'sidekiq-dynamic-queues'
+require 'sidekiq/web'
+require 'sidekiq/dynamic_queues/server'
+
+Sidekiq::Web.register Sidekiq::DynamicQueues::Server

data/lib/sidekiq-dynamic-queues.rb

@@ -0,0 +1,3 @@
+require 'sidekiq'
+require 'sidekiq/dynamic_queues/attributes'
+require 'sidekiq/dynamic_queues/fetch'

data/lib/sidekiq/dynamic_queues/attributes.rb

@@ -0,0 +1,65 @@
+module Sidekiq
+  module DynamicQueues
+
+    DYNAMIC_QUEUE_KEY = "dynamic_queue"
+    FALLBACK_KEY = "default"
+
+    module Attributes
+      extend self
+
+      def json_encode(data)
+        Sidekiq.dump_json(data)
+      end
+      
+      def json_decode(data)
+        return nil unless data
+        Sidekiq.load_json(data)
+      end
+      
+      def get_dynamic_queue(key, fallback=['*'])
+        data = Sidekiq.redis {|r| r.hget(DYNAMIC_QUEUE_KEY, key) }
+        queue_names = json_decode(data)
+
+        if queue_names.nil? || queue_names.size == 0
+          data = Sidekiq.redis {|r| r.hget(DYNAMIC_QUEUE_KEY, FALLBACK_KEY) }
+          queue_names = json_decode(data)
+        end
+        
+        if queue_names.nil? || queue_names.size == 0
+          queue_names = fallback
+        end
+
+        return queue_names
+      end
+
+      def set_dynamic_queue(key, values)
+        if values.nil? or values.size == 0
+          Sidekiq.redis {|r| r.hdel(DYNAMIC_QUEUE_KEY, key) }
+        else
+          Sidekiq.redis {|r| r.hset(DYNAMIC_QUEUE_KEY, key, json_encode(values)) }
+        end
+      end
+      
+      def set_dynamic_queues(dynamic_queues)
+        Sidekiq.redis do |r|
+          r.multi do
+            r.del(DYNAMIC_QUEUE_KEY)
+            dynamic_queues.each do |k, v|
+              set_dynamic_queue(k, v)
+            end
+          end
+        end
+      end
+
+      def get_dynamic_queues
+        result = {}
+        queues = Sidekiq.redis {|r| r.hgetall(DYNAMIC_QUEUE_KEY) }
+        queues.each {|k, v| result[k] = json_decode(v) }
+        result[FALLBACK_KEY] ||= ['*']
+        return result
+      end
+
+    end
+    
+  end
+end

data/lib/sidekiq/dynamic_queues/fetch.rb

@@ -0,0 +1,84 @@
+require 'sidekiq/fetch'
+
+module Sidekiq
+  module DynamicQueues
+
+    # enable with Sidekiq.options[:fetch] = Sidekiq::DynamicQueues::Fetch
+    class Fetch < Sidekiq::BasicFetch
+
+      include Sidekiq::Util
+      include Sidekiq::DynamicQueues::Attributes
+      
+      def initialize(options)
+        super
+        @dynamic_queues = options[:queues]
+      end
+  
+      # overriding Sidekiq::BasicFetch#queues_cmd
+      def queues_cmd
+        if @dynamic_queues.grep(/(^!)|(^@)|(\*)/).size == 0
+          super
+        else
+          queues = expanded_queues
+          queues = @strictly_ordered_queues ? queues : queues.shuffle
+          queues << Sidekiq::Fetcher::TIMEOUT
+        end
+      end
+      
+      # Returns a list of queues to use when searching for a job.
+      #
+      # A splat ("*") means you want every queue (in alpha order) - this
+      # can be useful for dynamically adding new queues.
+      #
+      # The splat can also be used as a wildcard within a queue name,
+      # e.g. "*high*", and negation can be indicated with a prefix of "!"
+      #
+      # An @key can be used to dynamically look up the queue list for key from redis.
+      # If no key is supplied, it defaults to the worker's hostname, and wildcards
+      # and negations can be used inside this dynamic queue list.   Set the queue
+      # list for a key with
+      # Sidekiq::DynamicQueues::Attributes.set_dynamic_queue(key, ["q1", "q2"]
+      #
+      def expanded_queues
+        queue_names = @dynamic_queues.dup
+
+        real_queues = Sidekiq::Client.registered_queues
+        matched_queues = []
+
+        while q = queue_names.shift
+          q = q.to_s
+
+          if q =~ /^(!)?@(.*)/
+            key = $2.strip
+            key = hostname if key.size == 0
+
+            add_queues = get_dynamic_queue(key)
+            add_queues.map! { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
+
+            queue_names.concat(add_queues)
+            next
+          end
+
+          if q =~ /^!/
+            negated = true
+            q = q[1..-1]
+          end
+
+          patstr = q.gsub(/\*/, ".*")
+          pattern = /^#{patstr}$/
+          if negated
+            matched_queues -= matched_queues.grep(pattern)
+          else
+            matches = real_queues.grep(/^#{pattern}$/)
+            matches = [q] if matches.size == 0 && q == patstr
+            matched_queues.concat(matches)
+          end
+        end
+
+        return matched_queues.collect { |q| "queue:#{q}" }.uniq.sort
+      end
+      
+    end
+    
+  end
+end

data/lib/sidekiq/dynamic_queues/server.rb

@@ -0,0 +1,69 @@
+require 'sidekiq-dynamic-queues'
+
+module Sidekiq
+  module DynamicQueues
+    module Server
+
+      Attr = Sidekiq::DynamicQueues::Attributes
+
+      def self.registered(app)
+        
+        app.helpers do
+          
+          def find_template(view,*a,&b)
+            dir = File.expand_path("../server/views/", __FILE__)
+            super(dir,*a,&b)
+            super
+          end
+          
+        end
+        
+        app.get "/dynamicqueue" do
+          @queues = []
+          dqueues = Attr.get_dynamic_queues
+          dqueues.each do |k, v|
+            fetch = Fetch.new(:queues => ["@#{k}"], :strict => true)
+            expanded = fetch.queues_cmd
+            expanded.pop
+            expanded = expanded.collect {|q| q.split(":").last }
+            view_data = {
+                'name' => k,
+                'value' => Array(v).join(", "),
+                'expanded' => expanded.join(", ")
+            }
+            @queues << view_data
+          end
+          
+          @queues.sort! do |a, b|
+            an = a['name']
+            bn = b['name']
+            if an == 'default'
+              1
+            elsif bn == 'default'
+              -1
+            else
+              an <=> bn
+            end
+          end
+          
+          slim :dynamicqueue
+        end
+
+        app.post "/dynamicqueue" do
+          dynamic_queues = Array(params['queues'])
+          queues = {}
+          dynamic_queues.each do |queue|
+            key = queue['name']
+            values = queue['value'].to_s.split(',').collect{|q| q.gsub(/\s/, '') }
+            queues[key] = values
+          end
+          Attr.set_dynamic_queues(queues)
+          redirect "/dynamicqueue"
+        end
+
+        app.tabs["DynamicQueues"] = "dynamicqueue"
+      end
+    end
+    
+  end
+end

data/lib/sidekiq/dynamic_queues/server/views/dynamicqueue.slim

@@ -0,0 +1,63 @@
+h3 Dynamic Queues
+
+p class="intro"
+  | The list below shows the dynamic queues currently defined.  When you start
+    a worker with a dynamic queue key (@key_name), that key is looked up from
+    the list below to determine the actual queues the worker should pull from.
+    Wildcards (*) and negation (leading !) can be used to select the queues the
+    worker should process. There is always a fallback key - @default, which
+    workers will use if the key for that worker is empty.  If both the key and
+    the fallback are empty, the worker defaults to processing '*'
+
+form action="/dynamicqueue" method="POST"
+
+  table class="queues table table-hover table-bordered table-striped table-white"
+    thead
+      th Name
+      th Value
+      th Expanded
+      th
+    - @queues.each_with_index do |data, i|
+      tr class="line"
+        td
+          input type="text" id="input-#{{i}}-name" name="queues[][name]" value="#{{data['name']}}"
+        td
+          input type="text" id="input-#{{i}}-value" name="queues[][value]" value="#{{data['value']}}"
+        td class="expanded"
+          = data['expanded']
+        td
+          a href="#remove" class="remove" Remove
+
+  a href="#add" class="add" Add
+
+  input type="submit" value="Save"
+
+javascript:                    
+  function markDirty()
+  {
+    $("input[type=submit]").css({border:"3px orange solid"});
+  }
+  
+  jQuery(function($) {
+
+    $("input").live("keypress", markDirty);
+    
+    $("a.add").live("click", function(e) {
+      e.preventDefault();
+      var $table = $("table.queues");
+      var $newRow = $table.find("tr.line:first").clone();
+      $newRow.find("input[type=text]").attr("value", "");
+      $newRow.find("td.expanded").html("")
+      $newRow.appendTo($table);
+      markDirty();
+    });
+
+    $("a.remove").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      $link.parents("tr").remove();
+      markDirty();
+    });
+
+ 
+  });