qtrix 0.0.1

data/lib/qtrix/matrix/analyzer.rb

@@ -0,0 +1,46 @@
+module Qtrix
+  module Matrix
+    ##
+    # Utility class to examining the distribution of queues
+    # within a matrix.  Its operations result in a hash of
+    # queue names mapped to arrays containing counts that
+    # the queue appeared in that column index of the matrix.
+    module Analyzer
+      # Breaks down any old matrix
+      def self.breakdown(matrix)
+        result_hash_for(matrix).tap do |result|
+          matrix.each do |row|
+            row.each_with_index do |queue, column|
+              result[queue][column] += 1
+            end
+          end
+
+          def result.to_s
+            self.map{|queue, pos| "#{queue}: #{pos.join(',')}"}.join("\n")
+          end
+
+          def result.dump
+            puts self
+          end
+        end
+      end
+
+      ##
+      # Maps the specified queue weights, generates a matrix
+      # with the specified number of rows, then breaks it down
+      # as above.
+      def self.analyze!(rows, queue_weights={})
+        Qtrix::Matrix.clear!
+        Qtrix::Queue.clear!
+        Qtrix::Queue.map_queue_weights(queue_weights)
+        Qtrix::Matrix.queues_for!(`hostname`, rows)
+        breakdown(Qtrix::Matrix.to_table)
+      end
+
+      private
+      def self.result_hash_for(matrix)
+        Hash.new{|h, k| h[k] = [0] * matrix.first.size}
+      end
+    end
+  end
+end

data/lib/qtrix/matrix/common.rb

@@ -0,0 +1,22 @@
+require 'bigdecimal'
+
+module Qtrix
+  module Matrix
+    module Common
+      REDIS_KEY = :matrix
+      def self.included(base)
+        base.send(:extend, self)
+      end
+
+      def pack(item)
+        # Marshal is fast but not human readable, might want to
+        # go for json or yaml. This is fast at least.
+        Marshal.dump(item)
+      end
+
+      def unpack(item)
+        Marshal.restore(item)
+      end
+    end
+  end
+end

data/lib/qtrix/matrix/model.rb

@@ -0,0 +1,16 @@
+require 'bigdecimal'
+require 'qtrix/matrix/common'
+
+module Qtrix
+  module Matrix
+    ##
+    # An entry (or cell) in the matrix, contains a single queue and its value
+    # relative to the other entries to the left in the same row.
+    Entry = Struct.new(:queue, :resource_percentage, :value)
+
+    ##
+    # A row in the matrix, contains the hostname the row is for and the entries
+    # of queues within the row.
+    Row = Struct.new(:hostname, :entries)
+  end
+end

data/lib/qtrix/matrix/queue_picker.rb

@@ -0,0 +1,52 @@
+require 'bigdecimal'
+
+module Qtrix
+  module Matrix
+    ##
+    # Responsible for picking a number of queue lists from the matrix
+    # for a specific host.  Will return already picked lists if they
+    # exist.  Will generate new queue lists if they are needed and
+    # prune old lists as they are no longer needed, maintaining a row
+    # in the matrix for the number of workers for the host.
+    class QueuePicker
+      include Namespacing
+      include Common
+      attr_reader :namespace, :reader, :hostname, :workers
+
+      def initialize(*args)
+        @namespace, @reader, @hostname, @workers = extract_args(3, *args)
+      end
+
+      def pick!
+        delta = workers - rows_for_host.size
+        new_queues = []
+        if delta > 0
+          generate(delta)
+        elsif delta < 0
+          prune(delta)
+        end
+        rows_for_host.map(&to_queues)
+      end
+
+      private
+      def to_queues
+        lambda {|row| row.entries.map(&:queue)}
+      end
+
+      def rows_for_host
+        # TODO fix me.
+        reader.rows_for_host(hostname, namespace)
+      end
+
+      def generate(count)
+        RowBuilder.new(namespace, hostname, count).build
+      end
+
+      def prune(count)
+        count.abs.times.each do
+          redis(namespace).lrem(REDIS_KEY, -2, pack(rows_for_host.pop))
+        end
+      end
+    end
+  end
+end

data/lib/qtrix/matrix/queue_prioritizer.rb

@@ -0,0 +1,70 @@
+require 'bigdecimal'
+require 'qtrix/matrix/common'
+
+module Qtrix
+  module Matrix
+    ##
+    # Maintains current prioritization of queues based on the
+    # state of all rows in the matrix.
+
+    class QueuePrioritizer
+      attr_reader :desired_distribution, :heads, :all_entries
+
+      def initialize(desired_distribution, heads, all_entries)
+        @desired_distribution = desired_distribution
+        @heads = heads
+        @all_entries = all_entries
+        @new_head_picked = false
+      end
+
+      def current_priority_queue
+        # Not a true priority queue, since we are sorting after all elements
+        # are inserted
+        queue = queue_priority_tuples_from desired_distribution
+        prioritized_queue = queue.sort &by_priority_of_tuple
+        to_simple_queues_from prioritized_queue
+      end
+
+      def queue_priority_tuples_from(dist)
+        dist.map{|queue| [queue, current_priority_of(queue)]}
+      end
+
+      def by_priority_of_tuple(context={})
+        lambda {|i, j| j[1] <=> i[1]}
+      end
+
+      def to_simple_queues_from(prioritized_queue)
+        prioritized_queue.map {|tuple| tuple[0]}
+      end
+
+      def current_priority_of(queue)
+        if has_appeared_at_head_of_row?(queue.name) || @new_head_picked
+          normal_priority_for queue
+        else
+          @new_head_picked = true
+          starting_priority_for queue
+        end
+      end
+
+      def has_appeared_at_head_of_row?(queue_name)
+        heads.include?(queue_name)
+      end
+
+      def normal_priority_for(queue)
+        queue.resource_percentage / (1 + sum_of(entries_for(queue)))
+      end
+
+      def starting_priority_for(queue)
+        queue.resource_percentage * 1000
+      end
+
+      def sum_of(entries)
+        entries.inject(0) {|memo, e| memo += e.value}
+      end
+
+      def entries_for(queue)
+         all_entries.select{|entry| entry.queue == queue.name}
+      end
+    end
+  end
+end

data/lib/qtrix/matrix/reader.rb

@@ -0,0 +1,25 @@
+require 'bigdecimal'
+
+module Qtrix
+  module Matrix
+    ##
+    # Class responsible for reading & returning the persistent state of
+    # the matrix.
+    class Reader
+      include Namespacing
+      include Common
+
+      def self.fetch(namespace=:current)
+        redis(namespace).lrange(REDIS_KEY, 0, -1).map{|dump| unpack(dump)}
+      end
+
+      def self.to_table(namespace=:current)
+        fetch(namespace).map{|row| row.entries.map(&:queue)}
+      end
+
+      def self.rows_for_host(hostname, namespace=:current)
+        fetch(namespace).select{|row| row.hostname == hostname}
+      end
+    end
+  end
+end

data/lib/qtrix/matrix/row_builder.rb

@@ -0,0 +1,72 @@
+require 'bigdecimal'
+require 'qtrix/matrix/common'
+
+module Qtrix
+  module Matrix
+    ##
+    # Carries out the construction of rows within the matrix for a number of
+    # workers for a specific hostname.
+    class RowBuilder
+      include Qtrix::Namespacing
+      include Matrix::Common
+      attr_reader :namespace, :hostname, :workers, :matrix,
+                  :desired_distribution, :heads, :all_entries
+
+      def initialize(*args)
+        @namespace, @hostname, @workers = extract_args(2, *args)
+        @matrix = Qtrix::Matrix.fetch(namespace)
+        @desired_distribution = Qtrix.desired_distribution(namespace)
+        @heads = matrix.map{|row| row.entries.first.queue}
+        @all_entries = matrix.map(&:entries).flatten
+      end
+
+      def build
+        [].tap do |result|
+          (1..workers).each do
+            queues_for_row = queue_prioritizer.current_priority_queue
+            build_row_for! hostname, queues_for_row
+            result << queues_for_row.map(&:name)
+          end
+        end
+      end
+
+      private
+      def queue_prioritizer
+        QueuePrioritizer.new(desired_distribution, heads, all_entries)
+      end
+
+      def build_row_for!(hostname, queues)
+        row = Row.new(hostname, [])
+        queues.each do |queue|
+          build_entry(row, queue, next_val_for(row))
+        end
+        heads << row.entries[0].queue
+        store(row)
+        true
+      end
+
+      def build_entry(row, queue, entry_val)
+        entry = Entry.new(
+          queue.name,
+          queue.resource_percentage,
+          entry_val
+        )
+        all_entries << entry
+        row.entries << entry
+      end
+
+      def next_val_for(row)
+        raw_result = 1.0 - sum_of_resource_percentages_for(row.entries)
+        BigDecimal.new(raw_result, 4)
+      end
+
+      def sum_of_resource_percentages_for(entries)
+        entries.inject(0) {|memo, entry| memo += entry.resource_percentage}
+      end
+
+      def store(row)
+        redis(namespace).rpush(REDIS_KEY, pack(row))
+      end
+    end
+  end
+end

data/lib/qtrix/namespacing.rb

@@ -0,0 +1,198 @@
+require 'singleton'
+require 'redis-namespace'
+
+module Qtrix
+  ##
+  # Provides support for namespacing the various redis keys so we can have multiple
+  # configuration sets and a pointer to the current namespace or configuration set.
+  #
+  # This will allow for us to set up different configurations for different scenarios
+  # and to switch between them easily.
+  #
+  # Scenarios might be things like:
+  #
+  # - day vs night configuration
+  # - weekday vs weekend configuration
+  # - common flood handling distributions.
+  #
+  # Most interaction should be through the mixin and not the manager singelton here.
+  # Example:
+  #
+  # class Foo
+  #   include Qtrix::Namespacing
+  #   @redis_namespace = :foo  # or [:current, :foo]
+  #
+  #   def some_method
+  #     redis.keys  # constrained to :qtrix:foo:*
+  #   end
+  # end
+  module Namespacing
+    def self.included(base)
+      # make redis_key and current_namespaced available to both class and
+      # instance methods
+      base.instance_exec do
+        extend Namespacing
+      end
+    end
+
+    ##
+    # Returns a redis client namespaced to the #redis_namespace defined in the
+    # object/class, or to the id within the parent option.  An id of :current
+    # will be evaluated to the current namespace.  By default, this
+    # a root (qtrix:) namespaced client.  Examples:
+    def redis(*namespaces)
+      all_namespaces = redis_namespace + namespaces
+      Manager.instance.redis(*all_namespaces)
+    end
+
+    ##
+    # Returns the redis namespace as defined in the instance or class
+    # @redis_namespace variable.
+    def redis_namespace
+      namespaces = Array(
+        self.instance_variable_get(:@redis_namespace) ||
+        self.class.instance_variable_get(:@redis_namespace)
+      )
+    end
+
+    ##
+    # Extracts the namespace, if any, from the arg list.
+    def extract_args(arg_count, *args)
+      if arg_count == args.size
+        [:current] + args
+      else
+        args
+      end
+    end
+
+    ##
+    # Manages namespaces.  Uses Redis::Namespace to impose the namespacing
+    # on calls to redis, and maintains the known config namespaces within
+    # redis.  Should not be working directly with this too much, it should
+    # be transparen when mixing in the Qtrix::Namespacing module.
+    class Manager
+      include Singleton
+      NAMESPACING_KEY = :namespacing
+      DEFAULT_NAMESPACE = :default
+      attr_reader :connection_config
+
+      def connection_config(opts={})
+        @connection_config ||= opts
+      end
+
+      def redis(*namespaces)
+        namespaced_client = Redis::Namespace.new(:qtrix, redis: client)
+        namespaced_redis({redis: namespaced_client}, *evaluate(namespaces.uniq))
+      end
+
+      def add_namespace(namespace)
+        validate namespace
+        namespacing_redis.sadd(:namespaces, namespace)
+      end
+
+      def remove_namespace!(namespace)
+        raise "Cannot remove default namespace" if namespace == :default
+        raise "Cannot remove current namespace" if namespace == current_namespace
+        namespacing_redis.srem(:namespaces, namespace)
+        Qtrix::Override.clear!(namespace)
+        Qtrix::Queue.clear!(namespace)
+        Qtrix::Matrix.clear!(namespace)
+      end
+
+      def namespaces
+        if default_namespace_does_not_exist?
+          namespacing_redis.sadd(:namespaces, DEFAULT_NAMESPACE)
+        end
+        namespacing_redis.smembers(:namespaces).map{|ns| unpack(ns)}
+      end
+
+      def change_current_namespace(namespace)
+        unless namespaces.include? namespace
+          raise "Unknown namespace: #{namespace}"
+        end
+        if not_ready?(namespace)
+          raise "#{namespace} is empty"
+        end
+        namespacing_redis.set(:current_namespace, namespace)
+      end
+
+      def current_namespace
+        if no_current_namespace?
+          self.change_current_namespace DEFAULT_NAMESPACE
+        end
+        unpack(namespacing_redis.get(:current_namespace))
+      end
+
+      private
+      def not_ready?(namespace)
+        namespace != :default &&
+        Qtrix.desired_distribution(namespace).empty?
+      end
+
+      def evaluate(namespaces)
+        ensure_a_namespace_in(namespaces)
+        namespaces.map do |namespace|
+          namespace == :current ? current_namespace : namespace
+        end
+      end
+
+      def ensure_a_namespace_in(namespaces)
+        if namespaces.compact.empty?
+          namespaces << :current
+        end
+      end
+
+      def client
+        @client ||= Redis.connect(connection_config)
+      end
+
+      def namespacing_redis
+        @namespacing_redis ||= redis(:namespacing)
+      end
+
+      def namespaced_redis(ctx, *namespaces)
+        current, *others = namespaces
+        if others.empty?
+          if current
+            Redis::Namespace.new(current, redis: ctx[:redis])
+          else
+            ctx[:redis]
+          end
+        else
+          next_ctx = {redis: Redis::Namespace.new(current, redis: ctx[:redis])}
+          namespaced_redis(next_ctx, *others)
+        end
+      end
+
+      def validate(namespace)
+        raise "cannot be nil" if namespace.nil?
+        unless only_letters_numbers_and_underscores? namespace
+          raise "must contain alphanumerics and underscores"
+        end
+        raise "#{namespace} already exists" if namespacing_redis.sismember(:namespaces, namespace)
+      end
+
+      def only_letters_numbers_and_underscores?(namespace)
+        namespace =~ /^[\w\d_]+$/
+      end
+
+      def default_namespace_does_not_exist?
+        !namespacing_redis.smembers(:namespaces).include? DEFAULT_NAMESPACE
+      end
+
+      def no_current_namespace?
+        namespacing_redis.get(:current_namespace).nil?
+      end
+
+      def unpack(value)
+        value.nil? ? nil : value.to_sym
+      end
+
+      def ensure_default_exists
+        unless namespaces.include?(DEFAULT_NAMESPACE)
+          self.add_namespace(DEFAULT_NAMESPACE)
+        end
+      end
+    end
+  end
+end