ryespy 0.7.0 → 1.0.0

data/test/ryespy/notifier/sidekiq_test.rb

@@ -0,0 +1,44 @@
+require_relative '../../helper'
+
+require_relative '../../../lib/ryespy/notifier/sidekiq'
+
+
+describe Ryespy::Notifier::Sidekiq do
+  
+  describe "#notify" do
+    before do
+      @sidekiq = Ryespy::Notifier::Sidekiq.new(
+        :namespace => Ryespy::Test::Redis::namespace
+      )
+      
+      @redis = @sidekiq.instance_variable_get(:@redis)
+    end
+    
+    after do
+      @sidekiq.close
+      
+      Ryespy::Test::Redis::flush_namespace(@redis)
+    end
+    
+    it "writes to queues set" do
+      @sidekiq.notify(nil, nil)
+      
+      @redis.smembers('queues').must_equal ['ryespy']
+    end
+    
+    it "writes to queue list" do
+      SecureRandom.stubs(:hex).returns('9c964160d25fdf24c6549e6d')
+      
+      @sidekiq.notify('PlanetClass', { :brain => 'marvin' })
+      
+      @redis.lrange('queue:ryespy', 0, -1).must_equal([{
+        'class' => 'PlanetClass',
+        'args'  => { 'brain' => 'marvin' },
+        'queue' => 'ryespy',
+        'retry' => true,
+        'jid'   => '9c964160d25fdf24c6549e6d',
+      }.to_json])
+    end
+  end
+  
+end

data/test/ryespy/version_test.rb

@@ -1,4 +1,4 @@
-require 'minitest/autorun'
+require_relative '../helper'
 
 require_relative '../../lib/ryespy/version'
 

metadata

@@ -1,62 +1,121 @@
 --- !ruby/object:Gem::Specification
 name: ryespy
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 1.0.0
 platform: ruby
 authors:
 - tiredpixel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-29 00:00:00.000000000 Z
+date: 2014-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.4
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.4
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: |-
-  Ryespy provides a simple executable for listening to
-      IMAP mailboxes or FTP folders, keeps track of what it's seen using Redis,
-      and notifies Redis in a way in which Resque and Sidekiq can process using
-      workers.
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: sidekiq-spy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+description: Redis Sidekiq/Resque IMAP, FTP, Amazon S3, Google Cloud Storage, Rackspace
+  Cloud Files listener.
 email:
 - tp@tiredpixel.com
 executables:
@@ -64,26 +123,37 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .ruby-gemset
-- .ruby-version
-- .travis.yml
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/ryespy
 - lib/ryespy.rb
-- lib/ryespy/config.rb
-- lib/ryespy/listeners/ftp.rb
-- lib/ryespy/listeners/imap.rb
-- lib/ryespy/notifiers/sidekiq.rb
-- lib/ryespy/redis_conn.rb
+- lib/ryespy/app.rb
+- lib/ryespy/listener/amzn_s3.rb
+- lib/ryespy/listener/base.rb
+- lib/ryespy/listener/fogable.rb
+- lib/ryespy/listener/ftp.rb
+- lib/ryespy/listener/goog_cs.rb
+- lib/ryespy/listener/imap.rb
+- lib/ryespy/listener/rax_cf.rb
+- lib/ryespy/notifier/sidekiq.rb
 - lib/ryespy/version.rb
 - ryespy.gemspec
-- test/ryespy/config_test.rb
+- test/helper.rb
+- test/ryespy/app_test.rb
+- test/ryespy/listener/amzn_s3_test.rb
+- test/ryespy/listener/ftp_test.rb
+- test/ryespy/listener/goog_cs_test.rb
+- test/ryespy/listener/imap_test.rb
+- test/ryespy/listener/rax_cf_test.rb
+- test/ryespy/notifier/sidekiq_test.rb
 - test/ryespy/version_test.rb
-- test/ryespy_test.rb
 homepage: https://github.com/tiredpixel/ryespy
 licenses:
 - MIT
@@ -94,21 +164,28 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
-summary: Ryespy listens to IMAP and FTP and queues in Redis (Sidekiq/Resque).
+summary: Redis Sidekiq/Resque IMAP, FTP, Amazon S3, Google Cloud Storage, Rackspace
+  Cloud Files listener.
 test_files:
-- test/ryespy/config_test.rb
+- test/helper.rb
+- test/ryespy/app_test.rb
+- test/ryespy/listener/amzn_s3_test.rb
+- test/ryespy/listener/ftp_test.rb
+- test/ryespy/listener/goog_cs_test.rb
+- test/ryespy/listener/imap_test.rb
+- test/ryespy/listener/rax_cf_test.rb
+- test/ryespy/notifier/sidekiq_test.rb
 - test/ryespy/version_test.rb
-- test/ryespy_test.rb

data/lib/ryespy/config.rb

@@ -1,83 +0,0 @@
-module Ryespy
-  class Config
-    
-    attr_accessor :log_level
-    attr_accessor :listener
-    attr_accessor :polling_interval
-    attr_accessor :redis_url
-    attr_accessor :redis_ns_ryespy
-    attr_accessor :redis_ns_notifiers
-    attr_accessor :notifiers
-    
-    attr_accessor :imap_host
-    attr_accessor :imap_port
-    attr_accessor :imap_ssl
-    attr_accessor :imap_username
-    attr_accessor :imap_password
-    attr_accessor :imap_mailboxes
-    
-    attr_accessor :ftp_host
-    attr_accessor :ftp_passive
-    attr_accessor :ftp_username
-    attr_accessor :ftp_password
-    attr_accessor :ftp_dirs
-    
-    def initialize
-      @log_level          = 'INFO'
-      @polling_interval   = 60
-      @redis_ns_ryespy    = 'ryespy:'
-      @redis_ns_notifiers = 'resque:'
-      @notifiers          = {
-        :sidekiq => [],
-      }
-      
-      @imap_port      = 993
-      @imap_ssl       = true
-      @imap_mailboxes = ['INBOX']
-      
-      @ftp_passive    = false
-      @ftp_dirs       = ['/']
-    end
-    
-    def to_s
-      params = [
-        :log_level,
-        :listener,
-        :polling_interval,
-        :redis_url,
-        :redis_ns_ryespy,
-        :redis_ns_notifiers,
-        :notifiers,
-      ]
-      
-      params.concat case @listener.to_s
-      when 'imap'
-        [
-          :imap_host,
-          :imap_port,
-          :imap_ssl,
-          :imap_username,
-          :imap_password,
-        ]
-      when 'ftp'
-        [
-          :ftp_host,
-          :ftp_passive,
-          :ftp_username,
-          :ftp_dirs,
-        ]
-      else
-        []
-      end
-      
-      params.collect! { |s| [s, instance_variable_get("@#{s}")] }
-      
-      Hash[params].to_s
-    end
-    
-    def redis_prefix_ryespy
-      "#{Ryespy.config.redis_ns_ryespy}#{Ryespy.config.listener}:"
-    end
-    
-  end
-end

data/lib/ryespy/listeners/ftp.rb

@@ -1,86 +0,0 @@
-require 'net/ftp'
-
-
-module Ryespy
-  module Listener
-    class FTP
-      
-      def initialize
-        begin
-          @ftp = Net::FTP.new(Ryespy.config.ftp_host)
-          
-          @ftp.passive = Ryespy.config.ftp_passive
-          
-          @ftp.login(Ryespy.config.ftp_username, Ryespy.config.ftp_password)
-        rescue Errno::ECONNREFUSED, Net::FTPError => e
-          Ryespy.logger.error { e.to_s }
-          
-          return
-        end
-        
-        if block_given?
-          yield self
-          
-          close
-        end
-      end
-      
-      def close
-        @ftp.close
-      end
-      
-      def check(params)
-        begin
-          @ftp.chdir(params[:dir])
-          
-          objects = {}
-          
-          @ftp.nlst.each do |fd|
-            mtime = @ftp.mtime(fd).to_i
-            size = @ftp.size(fd) rescue nil # ignore non-file error
-            
-            if size # exclude directories
-              checksum = "#{mtime},#{size}"
-              
-              if params[:seen_files][fd] != checksum
-                objects[fd] = checksum
-              end
-            end
-          end
-          
-          objects
-        rescue Net::FTPError => e
-          Ryespy.logger.error { e.to_s }
-          
-          return
-        end
-      end
-      
-      def check_all
-        Ryespy.config.ftp_dirs.each do |dir|
-          Ryespy.logger.debug { "dir:#{dir}" }
-          
-          redis_key = "#{Ryespy.config.redis_prefix_ryespy}#{Ryespy.config.ftp_host}:#{Ryespy.config.ftp_username}:#{dir}"
-          
-          Ryespy.logger.debug { "redis_key:#{redis_key}" }
-          
-          new_items = check({
-            :dir        => dir,
-            :seen_files => Ryespy.redis.hgetall(redis_key),
-          })
-          
-          Ryespy.logger.debug { "new_items:#{new_items}" }
-          
-          new_items.each do |filename, checksum|
-            Ryespy.redis.hset(redis_key, filename, checksum)
-            
-            Ryespy.notifiers.each { |n| n.notify('RyespyFTPJob', [dir, filename]) }
-          end
-          
-          Ryespy.logger.info { "#{dir} has #{new_items.count} new files" }
-        end
-      end
-      
-    end
-  end
-end

data/lib/ryespy/listeners/imap.rb

@@ -1,74 +0,0 @@
-require 'net/imap'
-
-
-module Ryespy
-  module Listener
-    class IMAP
-      
-      def initialize
-        begin
-          @imap = Net::IMAP.new(Ryespy.config.imap_host, {
-            :port => Ryespy.config.imap_port,
-            :ssl  => Ryespy.config.imap_ssl,
-          })
-          
-          @imap.login(Ryespy.config.imap_username, Ryespy.config.imap_password)
-        rescue Errno::ECONNREFUSED, Net::IMAP::Error => e
-          Ryespy.logger.error { e.to_s }
-          
-          return
-        end
-        
-        if block_given?
-          yield self
-          
-          close
-        end
-      end
-      
-      def close
-        @imap.disconnect
-      end
-      
-      def check(params)
-        begin
-          @imap.select(params[:mailbox])
-          
-          uids = @imap.uid_search("#{params[:last_seen_uid] + 1}:*")
-          
-          uids.find_all { |uid| uid > params[:last_seen_uid] } # IMAP search gets fun with edge cases
-        rescue Net::IMAP::Error => e
-          Ryespy.logger.error { e.to_s }
-          
-          return
-        end
-      end
-      
-      def check_all
-        Ryespy.config.imap_mailboxes.each do |mailbox|
-          Ryespy.logger.debug { "mailbox:#{mailbox}" }
-          
-          redis_key = "#{Ryespy.config.redis_prefix_ryespy}#{Ryespy.config.imap_host},#{Ryespy.config.imap_port}:#{Ryespy.config.imap_username}:#{mailbox}"
-          
-          Ryespy.logger.debug { "redis_key:#{redis_key}" }
-          
-          new_items = check({
-            :mailbox       => mailbox,
-            :last_seen_uid => Ryespy.redis.get(redis_key).to_i,
-          })
-          
-          Ryespy.logger.debug { "new_items:#{new_items}" }
-          
-          new_items.each do |uid|
-            Ryespy.redis.set(redis_key, uid)
-            
-            Ryespy.notifiers.each { |n| n.notify('RyespyIMAPJob', [mailbox, uid]) }
-          end
-          
-          Ryespy.logger.info { "#{mailbox} has #{new_items.count} new emails" }
-        end
-      end
-      
-    end
-  end
-end