ryespy 0.7.0 → 1.0.0

data/lib/ryespy.rb

@@ -1,53 +1,8 @@
-require 'logger'
-
 require_relative 'ryespy/version'
-require_relative 'ryespy/config'
-require_relative 'ryespy/redis_conn'
-
-require_relative 'ryespy/listeners/imap'
-require_relative 'ryespy/listeners/ftp'
 
-require_relative 'ryespy/notifiers/sidekiq'
+require_relative 'ryespy/app'
 
+require_relative 'ryespy/listener/base'
+# ryespy/listener/X dynamically required in ryespy/app.rb
 
-module Ryespy
-  
-  extend self
-  
-  def config
-    @config ||= Ryespy::Config.new
-  end
-  
-  def configure
-    yield config
-    
-    Ryespy.logger.debug { "Configured #{Ryespy.config.to_s}" }
-  end
-  
-  def logger
-    unless @logger
-      @logger = Logger.new($stdout)
-      
-      @logger.level = Logger.const_get(Ryespy.config.log_level)
-    end
-    
-    @logger
-  end
-  
-  def redis
-    @redis ||= Ryespy::RedisConn.new(Ryespy.config.redis_url).redis
-  end
-  
-  def notifiers
-    unless @notifiers
-      @notifiers = []
-      
-      Ryespy.config.notifiers[:sidekiq].each do |notifier_instance|
-        @notifiers << Ryespy::Notifier::Sidekiq.new(notifier_instance)
-      end
-    end
-    
-    @notifiers
-  end
-  
-end
+require_relative 'ryespy/notifier/sidekiq'

data/lib/ryespy/app.rb

@@ -0,0 +1,159 @@
+require 'logger'
+require 'ostruct'
+require 'redis'
+require 'redis-namespace'
+
+# listener dynamically required in App#setup
+
+
+module Ryespy
+  class App
+    
+    def self.config_defaults
+      {
+        :log_level          => :INFO,
+        :polling_interval   => 60,
+        :redis_ns_ryespy    => 'ryespy',
+        :redis_ns_notifiers => 'resque',
+        :imap => {
+          :port    => 993,
+          :ssl     => true,
+          :filters => ['INBOX'], # mailboxes
+        },
+        :ftp => {
+          :port    => 21,
+          :passive => false,
+          :filters => ['/'], # dirs
+        },
+        :amzn_s3 => {
+          :filters => [''], # prefixes
+        },
+        :goog_cs => {
+          :filters => [''], # prefixes
+        },
+        :rax_cf => {
+          :endpoint => :us,
+          :region   => :dfw,
+          :filters  => [''], # prefixes
+        },
+      }
+    end
+    
+    attr_reader :config
+    attr_reader :running
+    
+    def initialize(eternal = false, opts = {})
+      @eternal = eternal
+      
+      @logger = opts[:logger] || Logger.new(nil)
+      
+      @config = OpenStruct.new(self.class.config_defaults)
+      
+      @running = false
+      @threads = {}
+    end
+    
+    def configure
+      yield @config
+      
+      @logger.level = Logger.const_get(@config.log_level)
+      
+      Redis.current = Redis::Namespace.new(@config.redis_ns_ryespy,
+        :redis => Redis.connect(:url => @config.redis_url)
+      )
+      
+      @logger.debug { "Configured #{@config.to_s}" }
+    end
+    
+    def notifiers
+      unless @notifiers
+        @notifiers = []
+        
+        @config.notifiers[:sidekiq].each do |notifier_url|
+          @notifiers << Notifier::Sidekiq.new(
+            :url       => notifier_url,
+            :namespace => @config.redis_ns_notifiers,
+            :logger    => @logger
+          )
+        end
+      end
+      
+      @notifiers
+    end
+    
+    def start
+      begin
+        @running = true
+        
+        setup
+        
+        @threads[:refresh] ||= Thread.new do
+          refresh_loop # refresh frequently
+        end
+        
+        @threads.values.each(&:join)
+      ensure
+        cleanup
+      end
+    end
+    
+    def stop
+      @running = false
+      
+      @threads.values.each { |t| t.run if t.status == 'sleep' }
+    end
+    
+    private
+    
+    def setup
+      require_relative "listener/#{@config.listener}"
+    end
+    
+    def cleanup
+    end
+    
+    def refresh_loop
+      while @running do
+        begin
+          check_all
+        rescue StandardError => e
+          @logger.error { e.to_s }
+          
+          raise if @config.log_level == :DEBUG
+        end
+        
+        if !@eternal
+          stop
+          
+          break
+        end
+        
+        @logger.debug { "Snoring for #{@config.polling_interval} s" }
+        
+        sleep @config.polling_interval # sleep awhile (snore)
+      end
+    end
+    
+    def check_all
+      listener_class_map = {
+        :imap    => :IMAP,
+        :ftp     => :FTP,
+        :amzn_s3 => :AmznS3,
+        :goog_cs => :GoogCS,
+        :rax_cf  => :RaxCF,
+      }
+      
+      listener_config = @config[@config.listener].merge({
+        :notifiers => notifiers,
+        :logger    => @logger,
+      })
+      
+      listener_class = Listener.const_get(listener_class_map[@config.listener])
+      
+      listener_class.new(listener_config) do |listener|
+        listener_config[:filters].each { |f| listener.check(f) }
+      end
+    end
+    
+  end
+end

data/lib/ryespy/listener/amzn_s3.rb

@@ -0,0 +1,37 @@
+require 'fog'
+
+require_relative 'fogable'
+
+
+module Ryespy
+  module Listener
+    class AmznS3 < Base
+      
+      include Listener::Fogable
+      
+      REDIS_KEY_PREFIX  = 'amzn_s3'.freeze
+      SIDEKIQ_JOB_CLASS = 'RyespyAmznS3Job'.freeze
+      
+      def initialize(opts = {})
+        @config = {
+          :access_key => opts[:access_key],
+          :secret_key => opts[:secret_key],
+          :directory  => opts[:bucket],
+        }
+        
+        super(opts)
+      end
+      
+      private
+      
+      def connect_service
+        @fog_storage = Fog::Storage.new({
+          :provider              => 'AWS',
+          :aws_access_key_id     => @config[:access_key],
+          :aws_secret_access_key => @config[:secret_key],
+        })
+      end
+      
+    end
+  end
+end

data/lib/ryespy/listener/base.rb

@@ -0,0 +1,34 @@
+require 'logger'
+require 'redis'
+
+
+module Ryespy
+  module Listener
+    class Base
+      
+      def initialize(opts = {})
+        @notifiers = opts[:notifiers] || []
+        @logger    = opts[:logger] || Logger.new(nil)
+        
+        @redis = Redis.current
+        
+        connect_service
+        
+        if block_given?
+          yield self
+          
+          close
+        end
+      end
+      
+      def close
+      end
+      
+      private
+      
+      def connect_service
+      end
+      
+    end
+  end
+end

data/lib/ryespy/listener/fogable.rb

@@ -0,0 +1,59 @@
+module Ryespy
+  module Listener
+    module Fogable
+      
+      def check(prefix)
+        @logger.debug { "prefix: #{prefix}" }
+        
+        @logger.debug { "redis_key: #{redis_key}" }
+        
+        seen_files = @redis.hgetall(redis_key)
+        
+        unseen_files = get_unseen_files(prefix, seen_files)
+        
+        @logger.debug { "unseen_files: #{unseen_files}" }
+        
+        unseen_files.each do |filename, checksum|
+          @redis.hset(redis_key, filename, checksum)
+          
+          # prefix is not included as it is part of key, and list operations
+          # return files (virtually) recursively. Constructing Redis key in this
+          # way means a file matching multiple prefixes will only notify once.
+          @notifiers.each do |notifier|
+            notifier.notify(self.class::SIDEKIQ_JOB_CLASS, [filename])
+          end
+        end
+        
+        @logger.info { "#{prefix}* has #{unseen_files.count} new files" }
+      end
+      
+      private
+      
+      def redis_key
+        [
+          self.class::REDIS_KEY_PREFIX,
+          @config[:directory],
+        ].join(':')
+      end
+      
+      def get_unseen_files(prefix, seen_files)
+        files = {}
+        
+        @fog_storage.directories.get(@config[:directory],
+          :prefix => prefix
+        ).files.each do |file|
+          if file.content_type == 'application/directory' || file.content_length == 0
+            next # virtual dirs or 0-length file
+          end
+          
+          if seen_files[file.key] != file.etag # etag is server-side checksum
+            files[file.key] = file.etag
+          end
+        end
+        
+        files
+      end
+      
+    end
+  end
+end

data/lib/ryespy/listener/ftp.rb

@@ -0,0 +1,92 @@
+require 'net/ftp'
+
+
+module Ryespy
+  module Listener
+    class FTP < Base
+      
+      REDIS_KEY_PREFIX  = 'ftp'.freeze
+      SIDEKIQ_JOB_CLASS = 'RyespyFTPJob'.freeze
+      
+      def initialize(opts = {})
+        @ftp_config = {
+          :host     => opts[:host],
+          :port     => opts[:port],
+          :passive  => opts[:passive],
+          :username => opts[:username],
+          :password => opts[:password],
+        }
+        
+        super(opts)
+      end
+      
+      def close
+        @ftp.close
+      end
+      
+      def check(dir)
+        @logger.debug { "dir: #{dir}" }
+        
+        @logger.debug { "redis_key: #{redis_key(dir)}" }
+        
+        seen_files = @redis.hgetall(redis_key(dir))
+        
+        unseen_files = get_unseen_files(dir, seen_files)
+        
+        @logger.debug { "unseen_files: #{unseen_files}" }
+        
+        unseen_files.each do |filename, checksum|
+          @redis.hset(redis_key(dir), filename, checksum)
+          
+          @notifiers.each { |n| n.notify(SIDEKIQ_JOB_CLASS, [dir, filename]) }
+        end
+        
+        @logger.info { "#{dir} has #{unseen_files.count} new files" }
+      end
+      
+      private
+      
+      def connect_service
+        @ftp = Net::FTP.new
+        
+        @ftp.connect(@ftp_config[:host], @ftp_config[:port])
+        
+        @ftp.passive = @ftp_config[:passive]
+        
+        @ftp.login(@ftp_config[:username], @ftp_config[:password])
+      end
+      
+      def redis_key(dir)
+        [
+          REDIS_KEY_PREFIX,
+          @ftp_config[:host],
+          @ftp_config[:port],
+          @ftp_config[:username],
+          dir,
+        ].join(':')
+      end
+      
+      def get_unseen_files(dir, seen_files)
+        @ftp.chdir(dir)
+        
+        files = {}
+        
+        @ftp.nlst.each do |file|
+          mtime = @ftp.mtime(file).to_i
+          size = @ftp.size(file) rescue nil # ignore non-file error
+          
+          if size # exclude directories
+            checksum = "#{mtime},#{size}".freeze
+            
+            if seen_files[file] != checksum
+              files[file] = checksum
+            end
+          end
+        end
+        
+        files
+      end
+      
+    end
+  end
+end

data/lib/ryespy/listener/goog_cs.rb

@@ -0,0 +1,37 @@
+require 'fog'
+
+require_relative 'fogable'
+
+
+module Ryespy
+  module Listener
+    class GoogCS < Base
+      
+      include Listener::Fogable
+      
+      REDIS_KEY_PREFIX  = 'goog_cs'.freeze
+      SIDEKIQ_JOB_CLASS = 'RyespyGoogCSJob'.freeze
+      
+      def initialize(opts = {})
+        @config = {
+          :access_key => opts[:access_key],
+          :secret_key => opts[:secret_key],
+          :directory  => opts[:bucket],
+        }
+        
+        super(opts)
+      end
+      
+      private
+      
+      def connect_service
+        @fog_storage = Fog::Storage.new({
+          :provider                         => 'Google',
+          :google_storage_access_key_id     => @config[:access_key],
+          :google_storage_secret_access_key => @config[:secret_key],
+        })
+      end
+      
+    end
+  end
+end