qtrix 0.0.1

data/lib/qtrix/cli.rb

@@ -0,0 +1,80 @@
+require 'mixlib/cli'
+require 'yaml'
+
+module Qtrix
+  module CLI
+    class Base
+      include Mixlib::CLI
+      attr_reader :stdout, :stderr
+
+      def initialize(stdout=STDOUT, stderr=STDERR)
+        super()
+        @stdout = stdout
+        @stderr = stderr
+      end
+
+      def exec
+        Qtrix.connection_config(config)
+        unless exec_behavior
+          msg = "no appropriate combination of options.\n\n"
+          msg += "Type 'bundle exec [command] --help' for usage"
+          error(msg)
+        end
+      rescue StandardError => e
+        error("Failure: #{e}")
+      end
+
+      private
+      def to_args(*args)
+        args.compact
+      end
+
+      def write(msg)
+        stdout.write("#{msg}\n")
+        true
+      end
+
+      def error(msg)
+        stderr.write("Failure: #{msg}\n")
+        false
+      end
+    end
+
+    class Default
+      include Mixlib::CLI
+
+      banner <<-EOS
+
+Usage: bundle exec qtrix [sub-command] [options]
+
+Where available sub-commands are:
+
+  config-sets:  Perform operations on configuration sets.
+  queues:       Perform operations on queues within a
+                configuration set.
+  overrides:    Perform operations on overrides within a
+                configuration set.
+
+For more information about the subcommands, try:
+
+  bundle exec qtrix [subcommand] --help
+
+      EOS
+    end
+
+    require 'qtrix/cli/config_sets'
+    require 'qtrix/cli/queues'
+    require 'qtrix/cli/overrides'
+
+    @commands = {
+      config_sets: ConfigSets,
+      overrides:   Overrides,
+      queues:      Queues
+    }
+
+    def self.get_command_class(str)
+      @commands.fetch(str.downcase.to_sym) if str
+    end
+  end
+end
+

data/lib/qtrix/cli/config_sets.rb

@@ -0,0 +1,87 @@
+module Qtrix
+  module CLI
+    class ConfigSets < Base
+      include Mixlib::CLI
+
+      banner <<-EOS
+
+Usage: bundle exec qtrix config_sets [options]
+
+Allows interaction with the configuration sets in the qtrix system.
+With this, you can:
+
+    * View all configuration sets.
+    * View the current configuration set.
+    * Add a configuration set.
+    * Specify the current configuration set.
+    * Remove a configuration set.
+
+Options include:
+
+      EOS
+
+      option :list,
+        short:       '-l',
+        long:        '--list',
+        description: 'The list of known config sets'
+
+      option :current_configuration_set,
+        short:       '-c',
+        long:        '--current',
+        description: 'List current config set'
+
+      option :add_configuration_set,
+        long:        '--create CONFIG_SET_NAME',
+        description: 'Create a new config set'
+
+      option :activate_configuration_set,
+        short:       '-a CONFIG_SET_NAME',
+        long:        '--activate CONFIG_SET_NAME',
+        description: 'Activate a configuration set'
+
+      option :remove_configuration_set,
+        short:       '-d CONFIG_SET_NAME',
+        long:        '--delete CONFIG_SET_NAME',
+        description: 'Delete a non-active config set'
+
+      option :host,
+        short:       '-h HOST',
+        description: 'The host where redis is located',
+        default:     'localhost'
+
+      option :port,
+        short:       '-p PORT',
+        description: 'The host which redis is listening on',
+        default:     6379
+
+      option :db,
+        short:       '-n DB',
+        description: 'The redis DB where the action should occur',
+        default:     0
+
+      def exec_behavior
+        if config[:list]
+          config_sets = Qtrix.configuration_sets
+          msg = "Known configuration sets: #{config_sets.join(", ")}"
+          write(msg)
+        elsif config[:current_configuration_set]
+          current_config_set = Qtrix.current_configuration_set
+          msg = "Current configuration set: #{current_config_set}"
+          write(msg)
+        elsif config[:add_configuration_set]
+          config_set = config[:add_configuration_set]
+          Qtrix.create_configuration_set(config_set)
+          write("Configuration set created successfully: #{config_set}")
+        elsif config[:activate_configuration_set]
+          config_set = config[:activate_configuration_set].to_sym
+          Qtrix.activate_configuration_set!(config_set)
+          write("Activated configuration set successfully: #{config_set}")
+        elsif config[:remove_configuration_set]
+          config_set = config[:remove_configuration_set]
+          Qtrix.remove_configuration_set!(config_set.to_sym)
+          write("Configuration set removed successfully: #{config_set}")
+        end
+      end
+    end
+  end
+end

data/lib/qtrix/cli/overrides.rb

@@ -0,0 +1,121 @@
+module Qtrix
+  module CLI
+    class Overrides < Base
+      banner <<-EOS
+
+Usage: bundle exec qtrix overrides [options]
+
+Allows interaction with the overrides in a configuration set.  With this
+you can:
+
+    * List all the overrides in the config set.
+    * Add overrides for a list of queues and number of workers.
+    * Remove overrides for a list of queues and number of workers.
+
+Options include:
+
+      EOS
+
+      option :list_overrides,
+        short:       '-l',
+        long:        '--list',
+        description: 'List the queue overrides'
+
+      option :add_overrides,
+        short:       '-a',
+        long:        '--add',
+        description: 'Add a queue list override'
+
+      option :delete_overrides,
+        short:       '-d',
+        long:        '--delete',
+        description:  'Delete a queue list override, be sure to add -q <queue names>'
+
+      option :queue_list,
+        short:       '-q QUEUE_LIST',
+        long:        '--queues QUEUE_LIST',
+        description: 'Specify the list of queues for the override'
+
+      option :workers,
+        short:       '-w WORKER_COUNT',
+        long:        '--workers WORKER_COUNT',
+        description: 'Specify the list of workers for the override',
+        default:     '1'
+
+      option :config_set,
+        short:       '-c CONFIG_SET_NAME',
+        long:        '--config-set CONFIG_SET_NAME',
+        description: 'Specify the config set being operated on'
+
+      option :host,
+        short:       '-h HOST',
+        description: 'The host where redis is located',
+        default:     'localhost'
+
+      option :port,
+        short:       '-p PORT',
+        description: 'The host which redis is listening on',
+        default:     6379
+
+      option :db,
+        short:       '-n DB',
+        description: 'The redis DB where the action should occur',
+        default:     0
+
+      def exec_behavior
+        if config[:list_overrides]
+          msg = "Current Overrides:\n"
+          msg += overrides.map(&stringify).join("\n")
+          write(msg)
+        elsif add_overrides_params_provided?
+          Qtrix.add_override *to_args(config_set, queue_list, workers)
+          write("Added #{workers} overrides for #{queue_list}")
+        elsif delete_overrides_params_provided?
+          Qtrix.remove_override *to_args(config_set, queue_list, workers)
+          write("Deleted #{workers} overrides for #{queue_list}")
+        end
+      end
+
+      private
+      def config_set
+        config[:config_set]
+      end
+
+      def stringify
+        lambda {|override| string_value_of(override)}
+      end
+
+      def overrides
+        Qtrix.overrides config[:config_set]
+      end
+
+      def string_value_of(override)
+        "  #{hostname_of(override)}: #{override.queues.join(',')}"
+      end
+
+      def hostname_of(override)
+        override.host || 'Unclaimed'
+      end
+
+      def add_overrides_params_provided?
+        config[:add_overrides] && queues_and_workers?
+      end
+
+      def delete_overrides_params_provided?
+        config[:delete_overrides] && queues_and_workers?
+      end
+
+      def queues_and_workers?
+        config[:queue_list] && config[:workers]
+      end
+
+      def queue_list
+        config[:queue_list].split(",").map(&:strip)
+      end
+
+      def workers
+        config[:workers].to_i
+      end
+    end
+  end
+end

data/lib/qtrix/cli/queues.rb

@@ -0,0 +1,97 @@
+module Qtrix
+  module CLI
+    class Queues < Base
+      banner <<-EOS
+
+Usage: bundle exec qtrix queues [options]
+
+Allows observance and manipulation of queue priority within a
+configuration set.  With this you can:
+
+    * View the current queue priority in the global worker pool.
+    * Specify the weightings for all queues inline or via yaml
+
+Options include:
+
+      EOS
+
+      option :queue_weights,
+        short:       '-w',
+        long:        '--weights QUEUE_WEIGHT_LIST',
+        description: 'Specifies the queue-weight mappings as queue:weight,queue:weight,...'
+
+      option :queue_weights_yaml,
+        short:       '-y',
+        long:        '--yaml PATH_TO_YAML',
+        description: 'Path to a yaml containing a hash of queue names to weights'
+
+      option :desired_distribution,
+        short:       '-l',
+        long:        '--list',
+        description: 'Lists the queues by their desired distribution'
+
+      option :config_set,
+        short:       '-c CONFIG_SET_NAME',
+        long:        '--config-set CONFIG_SET_NAME',
+        description: 'Specify the config set being operated on'
+
+      option :host,
+        short:       '-h HOST',
+        description: 'The host where redis is located',
+        default:     'localhost'
+
+      option :port,
+        short:       '-p PORT',
+        description: 'The host which redis is listening on',
+        default:     6379
+
+      option :db,
+        short:       '-n DB',
+        description: 'The redis DB where the action should occur',
+        default:     0
+
+      def exec_behavior
+        if config[:desired_distribution]
+          msg = "Queues:\n"
+          msg += Qtrix.desired_distribution.map(&stringify).join("\n")
+          write(msg)
+        elsif queue_weights
+          map_queue_weights queue_weights
+        elsif queue_weights_yaml
+          map_queue_weights queue_weights_yaml
+        end
+      end
+
+      private
+      def map_queue_weights(weight_map)
+        Qtrix.map_queue_weights *to_args(config_set, weight_map)
+        write("OK")
+      end
+
+      def stringify
+        lambda {|queue| "  #{queue.name} (#{queue.weight})"}
+      end
+
+      def string_value_of(queue)
+        "  #{q.name} (#{q.weight})"
+      end
+
+      def queue_weights
+        if config[:queue_weights]
+          tuple_list = config[:queue_weights].split(",").map{|kv| kv.split(":")}
+          Hash[tuple_list]
+        end
+      end
+
+      def queue_weights_yaml
+        if config[:queue_weights_yaml]
+          YAML.load(File.read(config[:queue_weights_yaml]))
+        end
+      end
+
+      def config_set
+        config[:config_set]
+      end
+    end
+  end
+end

data/lib/qtrix/matrix.rb

@@ -0,0 +1,83 @@
+require 'bigdecimal'
+require 'qtrix/matrix/common'
+require 'qtrix/matrix/model'
+require 'qtrix/matrix/queue_picker'
+require 'qtrix/matrix/reader'
+require 'qtrix/matrix/row_builder'
+require 'qtrix/matrix/queue_prioritizer'
+require 'qtrix/matrix/analyzer'
+
+module Qtrix
+  ##
+  # Represents the matrix of queues to workers across the global worker pool,
+  # and is used to assign queues to pools based on the following goals:
+  #
+  # 1.  Maintain a desired distribution of worker resources to queues based
+  #     on their weight within the system as a whole.
+  # 2.  Ensure that every queue is at the head of at least 1 workers queue
+  #     list.
+  #
+  # Given a queue's weight, we calculate its resource percentage as:
+  #
+  # weight_of(queue) / weight_of(all_queues)
+  #
+  # To generate the list of queues for a worker, we take a list of queues
+  # sorted by current priority.  The current priority is calculated in one of
+  # two ways.  If the queue has not been at the head of the list, it is
+  # calculated as:
+  #
+  # resource_percentage_of(queue) * 1000
+  #
+  # This inflated value ensures that we will assign all queues to the head of
+  # at least one worker's queue list.
+  #
+  # If a queue has not been assigned to the head of a worker's queue list,
+  # the following algortihm is used.  Each entry in the matrix has a value,
+  # which is generally the likelihood that the queue would be reached given
+  # the weight of jobs to its left in a queue list.  Mathematically, this is:
+  #
+  # 1.0 - resource_percentage_of(previous_queues_in_list)
+  #
+  # Thus the closer to the head of a list a queue's entry is, the higher a
+  # value it receives for that entry.  The priority is then calculated as:
+  #
+  # resource_percentage_of(queue) / (1.0 + value_of(entries_for(queue))
+  #
+  # This ensures that the higher a queue appears in a list, the lower its
+  # priority for the next generated list.
+
+  module Matrix
+    include Qtrix::Namespacing
+    include Common
+
+    class << self
+      ##
+      # Obtain lists of queues for a number of worker processes
+      # on a server identified by the hostname.  This works
+      # within the current namespace only.
+      def queues_for!(*args)
+        namespace, hostname, workers = extract_args(2, *args)
+        QueuePicker.new(namespace, Reader, hostname, workers).pick!
+      end
+
+      ##
+      # Returns all of the queues in the table.
+      def fetch(namespace=:current)
+        Reader.fetch(namespace)
+      end
+
+      ##
+      # Fetches a matrix of simple queue names for each entry.
+      def to_table(namespace=:current)
+        Reader.to_table(namespace)
+      end
+
+      ##
+      # Clears the matrix so its rebuilt again when rows are requested.
+      def clear!(namespace=:current)
+        redis(namespace).del(REDIS_KEY)
+      end
+    end
+  end
+end
+