redis-repeater 0.1.0 → 0.2.0

data/bin/redis-repeater

@@ -1,9 +1,34 @@
 #!/usr/bin/env ruby
 
+require 'yaml'
 require File.dirname(__FILE__) + '/../lib/redis_repeater'
+require 'pidly'
 
-if ARGV.empty?
-  puts "Please specify a config directory"
+class RedisRepeaterController < Pidly::Control
+
+  before_start do
+    if ARGV.size != 2
+      puts 'redis-repeater start /path/to/config.yml'
+      exit
+    else
+      configuration = YAML::load File.open(ARGV[1], 'r')
+      @repeater = RedisRepeater::Repeater.new(configuration)
+      puts self.pid
+    end
+  end
+
+  start do
+    @repeater.run_forever
+  end
+
+end
+
+@daemon = RedisRepeaterController.spawn :name => 'redis_repeater', :verbose => true
+
+# Handle the command
+commands = %w{start stop status restart clean! kill}
+if commands.include?(ARGV.first)
+  @daemon.send ARGV.first
 else
-  RedisRepeater::start(ARGV[0])
+  puts "No such command, try one of: #{commands.join(', ')}"
 end

data/lib/redis_repeater.rb

@@ -1,75 +1,14 @@
-require File.dirname(__FILE__) + '/redis_repeater/scheduler.rb'
-require File.dirname(__FILE__) + '/redis_repeater/scheduler_job.rb'
-require File.dirname(__FILE__) + '/redis_repeater/transfer_scheduler_job.rb'
-
-require 'yaml'
+require 'bundler/setup'
+require 'redis'
 require 'logger'
-require 'fileutils'
-
 require 'eventmachine'
-require 'redis'
-
-module RedisRepeater
- 
-  DefaultRedisHost = 'localhost'
-  DefaultRedisPort = 6379
-  LogDefaultFilename = File.dirname(__FILE__) + '/../log/redis_repeater.log'
 
-  def self.start(config_path)
+require File.dirname(__FILE__) + '/redis_repeater/configuration_error'
+require File.dirname(__FILE__) + '/redis_repeater/repeater'
 
-    # allow a directory OR a yml file
-    if File.directory?(config_path)
-      config = YAML::load File.open(File.join(config_path, 'config.yml'))
-      queues = YAML::load File.open(File.join(config_path, 'queues.yml'))
-    else
-      config = YAML::load File.open(config_path)
-      queues = config['queues']
-    end
-
-    # Connect to redis
-    redis_from = redis_configure(config, 'origin')
-    redis_to = redis_configure(config, 'destination')
-
-    # replace 'magic' resque:queues key with all resque queues, but don't overwrite otherwise configured queues
-    # warning: only resolves queue names on startup, if new queue is created it will not pick it up.
-    if queues.has_key?("resque:queues")
-      redis_from.smembers("resque:queues").each do |queue|
-        queues["resque:queue:#{queue}"] = queues["resque:queues"] unless queues.has_key?("resque:queue:#{queue}")
-      end
-      queues.delete("resque:queues")
-    end
-    
-    # Logger
-    if config.has_key?('log')
-      log_filename = config['log']
-      FileUtils.mkdir_p(File.dirname(log_filename))
-      logger = Logger.new(log_filename)
-    else
-      logger = Logger.new(STDOUT)
-    end
-
-    # Load the queues into the scheduler
-    scheduler = Scheduler.new(logger)
-    EventMachine::run do
-      queues.each do |name, timeout|
-        puts "#{name} - #{timeout}"
-        scheduler << TransferSchedulerJob.new(name, timeout, logger, redis_from, redis_to, config['counter'])
-      end
-    end
-
-    # Perform forever
-    scheduler.perform # TODO make configurable
-
-  end
+module RedisRepeater
 
-  def self.redis_configure(config, name)
-    host = DefaultRedisHost
-    port = DefaultRedisPort
-    if config.has_key?(name)
-      host = config[name]['host'] if config[name].has_key?('host')
-      port = config[name]['port'] if config[name].has_key?('port')
-    end
-    Redis.new(:host => host, :port => port)
-  end
+  autoload :VERSION, File.dirname(__FILE__) + '/redis_repeater/version'
+  autoload :TransferSchedulerJob, File.dirname(__FILE__) + '/redis_repeater/transfer_scheduler_job'
 
 end

data/lib/redis_repeater/configuration_error.rb

@@ -0,0 +1,7 @@
+module RedisRepeater
+
+  class ConfigurationError < StandardError
+
+  end
+
+end

data/lib/redis_repeater/repeater.rb

@@ -0,0 +1,66 @@
+module RedisRepeater
+
+  class Repeater
+
+    attr_reader :servers, :repeats, :logger
+
+    def initialize(configuration, logger = nil)
+      raise ConfigurationError.new 'Configuration is empty' unless configuration
+      @servers = {}
+      @repeats = []
+      # Get the logger
+      @logger = logger
+      @logger ||= Logger.new(File.open(configuration['log'], 'a')) if configuration['log']
+      @logger ||= Logger.new(STDOUT)
+      # Load the configuration and start the server
+      load_hash_configuration configuration
+    end
+
+    def run_forever
+      raise ConfigurationError.new('No repeat jobs specified in configuration') if @repeats.empty?
+      # start the scheduler and run it forever
+      EventMachine::run do
+        @repeats.each do |job|
+          EventMachine::add_periodic_timer(job.timeout) { job.perform }
+        end
+      end
+      @logger.info "\nRepeating #{@repeats.count} #{@repeats.count == 1 ? 'queue' : 'queues'} forever!"
+    end
+
+    private
+
+    def load_hash_configuration(hash)
+      hash['servers'] && hash['servers'].each do |name, value|
+        options = {}
+        value.each { |k, v| options[k.to_sym] = v }
+        @servers[name] = Redis.new(options)
+      end
+      hash['repeats'] && hash['repeats'].each do |repeat|
+        options = {}
+        options[:source] = find_server_by_name repeat['source']
+        # Handle 'magic' queues
+        if repeat['queue'] == 'resque:queues' || repeat['queue'] == 'resque:queues:*'
+          options[:source].smembers('resque:queues').each do |queue|
+            repeat = repeat.dup
+            repeat['queue'] = "resque:queue:#{queue}"
+            hash['repeats'] << repeat
+          end          
+          next
+        end
+        options[:destinations] = repeat['destinations'].map { |d| { :server => find_server_by_name(d['server']), :queue => d['queue'] || repeat['queue'] } }
+        options[:queue] = repeat['queue']
+        options[:timeout] = repeat['timeout'] || 0
+        options[:maintain_count] = !!repeat['maintain_count'] # default false
+        @repeats << TransferSchedulerJob.new(self, options)
+      end
+    end
+
+    def find_server_by_name(name)
+      server = @servers[name]
+      raise ConfigurationError.new("No such server found: #{name}") if server.nil?
+      server
+    end
+
+  end
+
+end

data/lib/redis_repeater/transfer_scheduler_job.rb

@@ -1,25 +1,30 @@
 module RedisRepeater
 
-  # Move jobs from One Redis queue to the same queue on a different machine
+  class TransferSchedulerJob
 
-  class TransferSchedulerJob < SchedulerJob
+    attr_reader :source, :destinations, :queue, :timeout, :maintain_count
 
-    def initialize(name, timeout, logger, redis_from, redis_to, maintain_count)
-      @redis_from = redis_from
-      @redis_to = redis_to
-      @maintain_count = maintain_count
-      super(name, timeout, logger)
+    def initialize(repeater, config)
+      @source = config[:source]
+      @destinations = config[:destinations]
+      @queue = config[:queue]
+      @timeout = config[:timeout]
+      @maintain_count = config[:maintain_count]
+      @logger = repeater.logger
     end
 
     def perform
+      # Transport everything we can
       count = 0
-      while (item = @redis_from.lpop @name)
-        # work the same direction resque does
-        @redis_to.rpush(@name, item)
+      while (item = @source.lpop @queue)
+        @destinations.each do |destination|
+          destination[:server].rpush destination[:queue], item
+        end
         count += 1
       end
-      @redis_from.incrby("redis-repeater:#{@name}:count", count) if @maintain_count
-      @logger.debug "Transported #{count} items for #{@name}"
+      # Keep the count and log a debug message
+      @source.incrby("redis-repeater:#{@queue}:count", count) if @maintain_count
+      @logger.debug "Transported #{count} items from #{@queue} to #{@destinations.count} #{@destinations.count == 1 ? 'destination' : 'destinations'}"
     end
 
   end

data/lib/redis_repeater/version.rb

@@ -1,5 +1,5 @@
 module RedisRepeater
 
-  VERSION = [0, 1, 0]
+  VERSION = [0, 2, 0]
 
 end

data/spec/spec_helper.rb

@@ -1,39 +1,7 @@
 require File.dirname(__FILE__) + '/../lib/redis_repeater'
 
-SERVER_PORT = 6391
-CLIENT_PORT = 6392
-QN = 'queue-1234'
+SERVER_ONE = { 'hostname' => 'localhost', 'port' => 6379 }
+SERVER_TWO = { 'hostname' => 'localhost', 'port' => 6380 }
 
-SERVERS = ['client', 'server']
-
-def start_servers
-  SERVERS.each do |s|
-    print "starting a #{s}.. "
-    `redis-server #{File.dirname(__FILE__) + "/support/#{s}.conf"}`
-    puts 'DONE'
-  end
-end
-
-def stop_servers
-  SERVERS.each do |s|
-    pid = `cat #{s}.pidfile`.chop
-    print "stopping #{s} (PID:#{pid}).. "
-    `kill #{pid}`
-    puts 'DONE'
-  end
-end
-
-def start_repeater(type = :default)
-  print 'starting the repeater.. '
-  # TODO deamonize
-  @rthread = Thread.start do
-    RedisRepeater.start(File.dirname(__FILE__) + "/support/config/#{type}")
-  end
-  puts 'DONE'
-end
-
-def stop_repeater
-  print 'stopping the repeater.. '
-  @rthread.kill
-  puts 'DONE'
-end
+REDIS_ONE = Redis.new SERVER_ONE
+TEST_FILENAME = 'spec/test.log'

metadata

@@ -2,7 +2,7 @@
 name: redis-repeater
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - John Crepezzi
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-14 00:00:00 -04:00
+date: 2011-04-20 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -46,6 +46,17 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: pidly
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
 description: Automatically move events from one redis queue to the same queue on another server
 email: john@crepezzi.com
 executables: 
@@ -55,8 +66,8 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-- lib/redis_repeater/scheduler.rb
-- lib/redis_repeater/scheduler_job.rb
+- lib/redis_repeater/configuration_error.rb
+- lib/redis_repeater/repeater.rb
 - lib/redis_repeater/transfer_scheduler_job.rb
 - lib/redis_repeater/version.rb
 - lib/redis_repeater.rb

data/lib/redis_repeater/scheduler.rb

@@ -1,19 +0,0 @@
-module RedisRepeater
-
-  class Scheduler
-
-    def initialize(logger = nil)
-      @logger = logger
-      @logger.info 'Running on schedule forever...'
-    end
-
-    # When adding, we move through the current queue, and
-    # find the first one that is performing after we want to
-    def <<(job)
-      scheduler = self
-      EventMachine::add_timer(job.timeout) { job.perform; scheduler << job }
-    end
-
-  end
- 
-end

data/lib/redis_repeater/scheduler_job.rb

@@ -1,24 +0,0 @@
-module RedisRepeater
-  
-  class SchedulerJob
-
-    def initialize(name, timeout, logger)
-      @name = name
-      @timeout = timeout
-      @logger = logger
-      @next_allowed_performance = Time.now + timeout
-    end
-
-    def perform
-      puts "Performing #{self.name}"
-    end
-
-    def adjust_next_allowed_performance
-      @next_allowed_performance = Time.now + timeout
-    end
-
-    attr_reader :name, :timeout, :next_allowed_performance
-
-  end
- 
-end