ryespy 0.7.0 → 1.0.0

data/lib/ryespy/listener/imap.rb

@@ -0,0 +1,79 @@
+require 'net/imap'
+
+
+module Ryespy
+  module Listener
+    class IMAP < Base
+      
+      REDIS_KEY_PREFIX  = 'imap'.freeze
+      SIDEKIQ_JOB_CLASS = 'RyespyIMAPJob'.freeze
+      
+      def initialize(opts = {})
+        @imap_config = {
+          :host      => opts[:host],
+          :port      => opts[:port],
+          :ssl       => opts[:ssl],
+          :username  => opts[:username],
+          :password  => opts[:password],
+        }
+        
+        super(opts)
+      end
+      
+      def close
+        @imap.disconnect
+      end
+      
+      def check(mailbox)
+        @logger.debug { "mailbox: #{mailbox}" }
+        
+        @logger.debug { "redis_key: #{redis_key(mailbox)}" }
+        
+        last_seen_uid = @redis.get(redis_key(mailbox)).to_i
+        
+        unseen_uids = get_unseen_uids(mailbox, last_seen_uid)
+        
+        @logger.debug { "unseen_uids: #{unseen_uids}" }
+        
+        unseen_uids.each do |uid|
+          @redis.set(redis_key(mailbox), uid)
+          
+          @notifiers.each { |n| n.notify(SIDEKIQ_JOB_CLASS, [mailbox, uid]) }
+        end
+        
+        @logger.info { "#{mailbox} has #{unseen_uids.count} new emails" }
+      end
+      
+      private
+      
+      def connect_service
+        @imap = Net::IMAP.new(@imap_config[:host], {
+          :port => @imap_config[:port],
+          :ssl  => @imap_config[:ssl],
+        })
+        
+        @imap.login(@imap_config[:username], @imap_config[:password])
+      end
+      
+      def redis_key(mailbox)
+        [
+          REDIS_KEY_PREFIX,
+          @imap_config[:host],
+          @imap_config[:port],
+          @imap_config[:username],
+          mailbox,
+        ].join(':')
+      end
+      
+      def get_unseen_uids(mailbox, last_seen_uid = nil)
+        @imap.select(mailbox)
+        
+        uids = @imap.uid_search("#{last_seen_uid + 1}:*")
+        
+        # filter as IMAP search gets fun with edge cases
+        uids.find_all { |uid| uid > last_seen_uid }
+      end
+      
+    end
+  end
+end

data/lib/ryespy/listener/rax_cf.rb

@@ -0,0 +1,53 @@
+require 'fog'
+
+require_relative 'fogable'
+
+
+module Ryespy
+  module Listener
+    class RaxCF < Base
+      
+      include Listener::Fogable
+      
+      REDIS_KEY_PREFIX  = 'rax_cf'.freeze
+      SIDEKIQ_JOB_CLASS = 'RyespyRaxCFJob'.freeze
+      
+      def initialize(opts = {})
+        @config = {
+          :auth_url  => Fog::Rackspace.const_get(
+            "#{opts[:endpoint].upcase}_AUTH_ENDPOINT"
+          ),
+          :region    => opts[:region].downcase.to_sym,
+          :username  => opts[:username],
+          :api_key   => opts[:api_key],
+          :directory => opts[:container],
+        }
+        
+        super(opts)
+      end
+      
+      private
+      
+      def connect_service
+        @fog_storage = Fog::Storage.new({
+          :provider           => 'Rackspace',
+          :rackspace_auth_url => @config[:auth_url],
+          :rackspace_region   => @config[:region],
+          :rackspace_username => @config[:username],
+          :rackspace_api_key  => @config[:api_key],
+        })
+      end
+      
+      def redis_key
+        # CF container (directory) is unique across an account (region?).
+        [
+          REDIS_KEY_PREFIX,
+          @config[:username],
+          @config[:directory],
+          @config[:region],
+        ].join(':')
+      end
+      
+    end
+  end
+end

data/lib/ryespy/notifier/sidekiq.rb

@@ -0,0 +1,69 @@
+require 'logger'
+require 'redis'
+require 'redis-namespace'
+require 'json'
+require 'securerandom'
+
+
+module Ryespy
+  module Notifier
+    class Sidekiq
+      
+      SIDEKIQ_QUEUE       = 'ryespy'.freeze
+      SIDEKIQ_KEY_QUEUES  = 'queues'.freeze
+      SIDEKIQ_KEY_QUEUE_X = "queue:#{SIDEKIQ_QUEUE}".freeze
+      
+      def initialize(opts = {})
+        @redis_config = {
+          :url       => opts[:url],
+          :namespace => opts[:namespace],
+        }
+        
+        @logger = opts[:logger] || Logger.new(nil)
+        
+        connect_redis
+        
+        if block_given?
+          yield self
+          
+          close
+        end
+      end
+      
+      def close
+        @redis.quit
+      end
+      
+      def notify(job_class, args)
+        @redis.sadd(SIDEKIQ_KEY_QUEUES, SIDEKIQ_QUEUE)
+        
+        sidekiq_job_payload = sidekiq_job(job_class, args)
+        
+        @logger.debug { "Setting Redis Key #{SIDEKIQ_KEY_QUEUE_X} Payload #{sidekiq_job_payload.to_json}" }
+        
+        @redis.rpush(SIDEKIQ_KEY_QUEUE_X, sidekiq_job_payload.to_json)
+      end
+      
+      private
+      
+      def connect_redis
+        @redis = Redis::Namespace.new(@redis_config[:namespace],
+          :redis => Redis.connect(:url => @redis_config[:url])
+        )
+      end
+      
+      def sidekiq_job(job_class, args)
+        {
+          # resque
+          :class => job_class,
+          :args  => args,
+          # sidekiq (extra)
+          :queue => SIDEKIQ_QUEUE,
+          :retry => true,
+          :jid   => SecureRandom.hex(12),
+        }
+      end
+      
+    end
+  end
+end

data/lib/ryespy/version.rb

@@ -1,5 +1,5 @@
 module Ryespy
   
-  VERSION = "0.7.0"
+  VERSION = '1.0.0'.freeze
   
 end

data/ryespy.gemspec

@@ -8,11 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = Ryespy::VERSION
   spec.authors       = ["tiredpixel"]
   spec.email         = ["tp@tiredpixel.com"]
-  spec.description   = %q{Ryespy provides a simple executable for listening to
-    IMAP mailboxes or FTP folders, keeps track of what it's seen using Redis,
-    and notifies Redis in a way in which Resque and Sidekiq can process using
-    workers.}
-  spec.summary       = %q{Ryespy listens to IMAP and FTP and queues in Redis (Sidekiq/Resque).}
+  spec.description   = %q{Redis Sidekiq/Resque IMAP, FTP, Amazon S3, Google Cloud Storage, Rackspace Cloud Files listener.}
+  spec.summary       = %q{Redis Sidekiq/Resque IMAP, FTP, Amazon S3, Google Cloud Storage, Rackspace Cloud Files listener.}
   spec.homepage      = "https://github.com/tiredpixel/ryespy"
   spec.license       = "MIT"
   
@@ -21,8 +18,12 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
   
-  spec.add_dependency "redis", "~> 3.0.4"
+  spec.add_dependency "redis", "~> 3.0"
+  spec.add_dependency "redis-namespace", "~> 1.4"
   
-  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "bundler", "~> 1.3", "!= 1.5.0"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "fog", "~> 1.19" # conditional dependency
+  spec.add_development_dependency "mocha", "~> 0.14"
+  spec.add_development_dependency "sidekiq-spy", "~> 0.3"
 end

data/test/helper.rb

@@ -0,0 +1,41 @@
+require 'minitest/autorun'
+require 'mocha/setup'
+require 'securerandom'
+require 'json'
+require 'redis'
+require 'redis-namespace'
+
+
+module Ryespy
+  module Test
+    
+    def self.config
+      @config ||= {
+        :redis => {
+          :url       => ENV['REDIS_URL'], # defaults
+          :namespace => 'ryespy:test',
+        },
+      }
+    end
+    
+    module Redis
+      
+      def self.setup
+        ::Redis.current = ::Redis::Namespace.new(self.namespace,
+          :redis => ::Redis.connect(:url => Ryespy::Test.config[:redis][:url])
+        )
+      end
+      
+      def self.namespace
+        "#{Ryespy::Test.config[:redis][:namespace]}:#{SecureRandom.hex}"
+      end
+      
+      def self.flush_namespace(redis)
+        # Redis::Namespace means only namespaced keys removed
+        redis.keys('*').each { |k| redis.del(k) }
+      end
+      
+    end
+    
+  end
+end

data/test/ryespy/app_test.rb

@@ -0,0 +1,348 @@
+require 'logger'
+require 'net/imap'
+
+
+require_relative '../helper'
+
+require_relative '../../lib/ryespy'
+
+
+def start_and_stop_app(app)
+  app_thread = Thread.new { app.start }
+  
+  sleep 1 # patience, patience; give app time to start
+  
+  app.stop
+  
+  app_thread.join(2)
+  
+  Thread.kill(app_thread)
+end
+
+
+describe Ryespy::App do
+  
+  describe "#initialize" do
+    before do
+      @app = Ryespy::App.new
+    end
+    
+    it "defaults running to false" do
+      @app.running.must_equal false
+    end
+  end
+  
+  describe "#configure" do
+    before do
+      @app = Ryespy::App.new
+      
+      @config = @app.config
+    end
+    
+    describe "main" do
+      before do
+        @app.configure do |c|
+          c.log_level           = 'ERROR'
+          c.listener            = 'imap'
+          c.polling_interval    = 13
+          c.redis_url           = 'redis://127.0.0.1:6379/1'
+          c.redis_ns_ryespy     = 'WithMyLittleEye!'
+          c.redis_ns_notifiers  = 'LaLaLiLi-'
+          c.notifiers           = [{ :sidekiq => 'redis://127.0.0.1:6379/2' }]
+        end
+      end
+      
+      it "configures log_level" do
+        @config.log_level.must_equal 'ERROR'
+      end
+      
+      it "sets logger level" do
+        @app.instance_variable_get(:@logger).level.must_equal Logger::ERROR
+      end
+      
+      it "configures listener" do
+        @config.listener.must_equal 'imap'
+      end
+      
+      it "configures polling_interval" do
+        @config.polling_interval.must_equal 13
+      end
+      
+      it "configures redis_url" do
+        @config.redis_url.must_equal 'redis://127.0.0.1:6379/1'
+      end
+      
+      it "configures redis_ns_ryespy" do
+        @config.redis_ns_ryespy.must_equal 'WithMyLittleEye!'
+      end
+      
+      it "configures redis_ns_notifiers" do
+        @config.redis_ns_notifiers.must_equal 'LaLaLiLi-'
+      end
+      
+      it "configures notifiers" do
+        @config.notifiers.must_equal [{ :sidekiq => 'redis://127.0.0.1:6379/2' }]
+      end
+    end
+    
+    describe "listener IMAP" do
+      before do
+        @app.configure do |c|
+          c.imap_host     = 'imap.example.com'
+          c.imap_port     = 143
+          c.imap_ssl      = false
+          c.imap_username = 'lucy.westenra@example.com'
+          c.imap_password = 'white'
+          c.imap_filters  = 'BoxA,Sent Messages'
+        end
+      end
+      
+      it "configures imap_host" do
+        @config.imap_host.must_equal 'imap.example.com'
+      end
+      
+      it "configures imap_port" do
+        @config.imap_port.must_equal 143
+      end
+      
+      it "configures imap_ssl" do
+        @config.imap_ssl.must_equal false
+      end
+      
+      it "configures imap_username" do
+        @config.imap_username.must_equal 'lucy.westenra@example.com'
+      end
+      
+      it "configures imap_password" do
+        @config.imap_password.must_equal 'white'
+      end
+      
+      it "configures imap_filters" do
+        @config.imap_filters.must_equal 'BoxA,Sent Messages'
+      end
+    end
+    
+    describe "listener FTP" do
+      before do
+        @app.configure do |c|
+          c.ftp_host     = 'ftp.example.org'
+          c.ftp_port     = 2121
+          c.ftp_passive  = true
+          c.ftp_username = 'madam.mina@example.com'
+          c.ftp_password = 'black'
+          c.ftp_filters  = ['BoxA', 'Sent Messages']
+        end
+      end
+      
+      it "configures ftp_host" do
+        @config.ftp_host.must_equal 'ftp.example.org'
+      end
+      
+      it "configures ftp_port" do
+        @config.ftp_port.must_equal 2121
+      end
+      
+      it "configures ftp_passive" do
+        @config.ftp_passive.must_equal true
+      end
+      
+      it "configures ftp_username" do
+        @config.ftp_username.must_equal 'madam.mina@example.com'
+      end
+      
+      it "configures ftp_password" do
+        @config.ftp_password.must_equal 'black'
+      end
+      
+      it "configures ftp_filters" do
+        @config.ftp_filters.must_equal ['BoxA', 'Sent Messages']
+      end
+    end
+    
+    describe "listener amzn-s3" do
+      before do
+        @app.configure do |c|
+          c.amzn_s3_access_key = 'r.m.renfield'
+          c.amzn_s3_secret_key = 'master'
+          c.amzn_s3_bucket     = 'i-can-wait'
+          c.amzn_s3_filters    = ['flies/', 'spiders/']
+        end
+      end
+      
+      it "configures amzn_s3_access_key" do
+        @config.amzn_s3_access_key.must_equal 'r.m.renfield'
+      end
+      
+      it "configures amzn_s3_secret_key" do
+        @config.amzn_s3_secret_key.must_equal 'master'
+      end
+      
+      it "configures amzn_s3_bucket" do
+        @config.amzn_s3_bucket.must_equal 'i-can-wait'
+      end
+      
+      it "configures amzn_s3_filters" do
+        @config.amzn_s3_filters.must_equal ["flies/", "spiders/"]
+      end
+    end
+    
+    describe "listener goog-cs" do
+      before do
+        @app.configure do |c|
+          c.goog_cs_access_key = 'r.m.renfield'
+          c.goog_cs_secret_key = 'master'
+          c.goog_cs_bucket     = 'i-can-wait'
+          c.goog_cs_filters    = ['flies/', 'spiders/']
+        end
+      end
+      
+      it "configures goog_cs_access_key" do
+        @config.goog_cs_access_key.must_equal 'r.m.renfield'
+      end
+      
+      it "configures goog_cs_secret_key" do
+        @config.goog_cs_secret_key.must_equal 'master'
+      end
+      
+      it "configures goog_cs_bucket" do
+        @config.goog_cs_bucket.must_equal 'i-can-wait'
+      end
+      
+      it "configures goog_cs_filters" do
+        @config.goog_cs_filters.must_equal ["flies/", "spiders/"]
+      end
+    end
+    
+    describe "listener rax-cf" do
+      before do
+        @app.configure do |c|
+          c.rax_cf_endpoint  = 'uk'
+          c.rax_cf_region    = 'lon'
+          c.rax_cf_username  = 'van.helsing'
+          c.rax_cf_api_key   = 'M.D., D.Ph., D.Litt., etc.'
+          c.rax_cf_container = 'the-milk-that-is-spilt-cries-not-out-afterwards'
+          c.rax_cf_filters   = ['abraham/', 'van/']
+        end
+      end
+      
+      it "configures rax_cf_endpoint" do
+        @config.rax_cf_endpoint.must_equal 'uk'
+      end
+      
+      it "configures rax_cf_region" do
+        @config.rax_cf_region.must_equal 'lon'
+      end
+      
+      it "configures rax_cf_username" do
+        @config.rax_cf_username.must_equal 'van.helsing'
+      end
+      
+      it "configures rax_cf_api_key" do
+        @config.rax_cf_api_key.must_equal 'M.D., D.Ph., D.Litt., etc.'
+      end
+      
+      it "configures rax_cf_container" do
+        @config.rax_cf_container.must_equal 'the-milk-that-is-spilt-cries-not-out-afterwards'
+      end
+      
+      it "configures rax_cf_filters" do
+        @config.rax_cf_filters.must_equal ["abraham/", "van/"]
+      end
+    end
+  end
+  
+  describe "#notifiers" do
+    before do
+      @app = Ryespy::App.new
+      
+      @app.configure do |c|
+        c.notifiers = { :sidekiq => ['redis://127.0.0.1:6379/11'] }
+      end
+      
+      @app.instance_variable_set(:@notifiers, nil)
+    end
+    
+    it "creates notifiers when empty" do
+      @app.notifiers.map(&:class).must_equal [Ryespy::Notifier::Sidekiq]
+    end
+    
+    it "returns notifiers when extant" do
+      @notifiers = stub
+      
+      @app.instance_variable_set(:@notifiers, @notifiers)
+      
+      @app.notifiers.must_equal @notifiers
+    end
+  end
+  
+  describe "#start" do
+    before do
+      Net::IMAP.stubs(:new).returns(stub(
+        :login      => nil,
+        :select     => nil,
+        :uid_search => [],
+        :disconnect => nil
+      ))
+      
+      @app = Ryespy::App.new(true)
+      
+      @app.instance_variable_set(:@logger, Logger.new(nil))
+      
+      @app.configure do |c|
+        c.listener         = :imap
+        c.polling_interval = 10
+      end
+    end
+    
+    it "sets status running within 1s" do
+      thread_app = Thread.new { @app.start }
+      
+      sleep 1 # patience, patience; give app time to start
+      
+      @app.running.must_equal true
+      
+      Thread.kill(thread_app)
+    end
+    
+    it "stops running within 1s" do
+      thread_app = Thread.new { @app.start }
+      
+      sleep 1 # patience, patience; give app time to start
+      
+      @app.stop; t0 = Time.now
+      
+      thread_app.join(2)
+      
+      Thread.kill(thread_app)
+      
+      assert_operator (Time.now - t0), :<=, 1
+    end
+    
+    it "calls #setup hook" do
+      @app.expects(:setup)
+      
+      start_and_stop_app(@app)
+    end
+    
+    it "calls #cleanup hook" do
+      @app.expects(:cleanup)
+      
+      start_and_stop_app(@app)
+    end
+  end
+  
+  describe "#stop" do
+    before do
+      @app = Ryespy::App.new
+      
+      @app.instance_variable_set(:@running, true)
+    end
+    
+    it "sets status not-running" do
+      @app.stop
+      
+      @app.running.must_equal false
+    end
+  end
+  
+end