mail_room 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 836563b572db61eacd01d2363273d998863e9831
-  data.tar.gz: de0cab71b636b3c7ffe0f08cd021896e403940c3
+  metadata.gz: 7e839c7e9c2857c75d399aca74fe627438432c17
+  data.tar.gz: 97d5cbe394fbd50ee36814785ce03af51381c1ae
 SHA512:
-  metadata.gz: 9648c21785544fab432cedeca09c6c28ffd7d993f7ff90c30c1bb2d893e00517687f233118ae945be43fe1c0123ce7f0b2a0addf78c7b5fb22876fae81718df9
-  data.tar.gz: 503a0ac11cf7e0dddb4299c608ba5d87a5820419afb08ed8a65aeed6e04c6e56a54133f61239f193c63a8f11384fe91540b16ad7dac2a81cbbad4044e794ef1d
+  metadata.gz: 5dfcca4fe8a7323d5207b232a48351bb78cd4df53f8301aa719c198186d08ceb09ca3ce7dd3178af2d80f513f330a52488ba9bad5a6aa079aa9a0b19f649e002
+  data.tar.gz: e2ec188a14291548bc1891459e3ce18066109daa8566d584714d26fc2be3bb3fc17a54b17e1925c5758d785d1daa4e8292735a7db17cba04f045c6801b035a02

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## mail_room 0.6.0 ##
+
+* Add redis Arbitration to reduce multiple deliveries of the same message when running multiple MailRoom instances on the same inbox
+
+    *Douwe Maan <@DouweM>*
+
 ## mail_room 0.5.2 ##
 
 * Fix Sidekiq delivery method for non-UTF8 email

data/README.md

@@ -203,6 +203,34 @@ If you're seeing the error `Please log in via your web browser: https://support.
 
 I suggest running with either upstart or init.d. Check out this wiki page for some example scripts for both: https://github.com/tpitale/mail_room/wiki/Init-Scripts-for-Running-mail_room
 
+## Arbitration ##
+
+When running multiple instances of MailRoom against a single mailbox, to try to prevent delivery of the same message multiple times, we can configure Arbitration using Redis.
+
+```yaml
+:mailboxes:
+  -
+    :email: "user1@gmail.com"
+    :password: "password"
+    :name: "inbox"
+    :delivery_method: postback
+    :delivery_options:
+      :delivery_url: "http://localhost:3000/inbox"
+      :delivery_token: "abcdefg"
+     
+    :arbitration_method: redis
+    :arbitration_options:
+      # The Redis server to connect with. Defaults to redis://localhost:6379.
+      :redis_url: redis://redis.example.com:6379
+      # The Redis namespace to house the Redis keys under. Optional. 
+      :namespace: mail_room
+
+```
+
+**Note:** This will likely never be a _perfect_ system for preventing multiple deliveries of the same message, so I would advise checking the unique `message_id` if you are running in this situation.
+
+**Note:** There are other scenarios for preventing duplication of messages at scale that _may_ be more appropriate in your particular setup. One such example is using multiple inboxes in reply-by-email situations. Another is to use labels and configure a different `SEARCH` command for each instance of MailRoom.
+
 ## Contributing ##
 
 1. Fork it

data/lib/mail_room.rb

@@ -2,16 +2,7 @@ require 'net/imap'
 require 'optparse'
 require 'yaml'
 
-# The MailRoom namespace
 module MailRoom
-  # The MailRoom Delivery namespace holds any defined delivery methods
-  #   including:
-  # 
-  #   * postback (default)
-  #   * letter_opener
-  #   * logger
-  #   * noop
-  module Delivery; end
 end
 
 require "mail_room/version"

data/lib/mail_room/arbitration.rb

@@ -0,0 +1,16 @@
+module MailRoom
+  module Arbitration
+    def [](name)
+      require_relative("./arbitration/#{name}")
+
+      case name
+      when "redis"
+        Arbitration::Redis
+      else
+        Arbitration::Noop
+      end
+    end
+
+    module_function :[]
+  end
+end

data/lib/mail_room/arbitration/noop.rb

@@ -0,0 +1,18 @@
+module MailRoom
+  module Arbitration
+    class Noop
+      Options = Class.new do
+        def initialize(*)
+          super()
+        end
+      end
+
+      def initialize(*)
+      end
+
+      def deliver?(*)
+        true
+      end
+    end
+  end
+end

data/lib/mail_room/arbitration/redis.rb

@@ -0,0 +1,60 @@
+require "redis"
+
+module MailRoom
+  module Arbitration
+    class Redis
+      Options = Struct.new(:redis_url, :namespace) do
+        def initialize(mailbox)
+          redis_url = mailbox.arbitration_options[:redis_url] || "redis://localhost:6379"
+          namespace = mailbox.arbitration_options[:namespace]
+
+          super(redis_url, namespace)
+        end
+      end
+
+      # Expire after 10 minutes so Redis doesn't get filled up with outdated data.
+      EXPIRATION = 600
+
+      attr_accessor :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      def deliver?(uid)
+        key = "delivered:#{uid}"
+
+        incr = nil
+        redis.multi do |client|
+          # At this point, `incr` is a future, which will get its value after 
+          # the MULTI command returns.
+          incr = client.incr(key)
+
+          client.expire(key, EXPIRATION)
+        end
+
+        # If INCR returns 1, that means the key didn't exist before, which means
+        # we are the first mail_room to try to deliver this message, so we get to.
+        # If we get any other value, another mail_room already (tried to) deliver
+        # the message, so we don't have to anymore.
+        incr.value == 1
+      end
+
+      private
+
+      def redis
+        @redis ||= begin
+          redis = ::Redis.new(url: options.redis_url)
+
+          namespace = options.namespace
+          if namespace
+            require 'redis/namespace'
+            ::Redis::Namespace.new(namespace, redis: redis)
+          else
+            redis
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mail_room/delivery.rb

@@ -0,0 +1,24 @@
+module MailRoom
+  module Delivery
+    def [](name)
+      require_relative("./delivery/#{name}")
+
+      case name
+      when "postback"
+        Delivery::Postback
+      when "logger"
+        Delivery::Logger
+      when "letter_opener"
+        Delivery::LetterOpener
+      when "sidekiq"
+        Delivery::Sidekiq
+      when "que"
+        Delivery::Que
+      else
+        Delivery::Noop
+      end
+    end
+
+    module_function :[]
+  end
+end

data/lib/mail_room/mailbox.rb

@@ -1,3 +1,6 @@
+require "mail_room/delivery"
+require "mail_room/arbitration"
+
 module MailRoom
   # Mailbox Configuration fields
   MAILBOX_FIELDS = [
@@ -15,7 +18,9 @@ module MailRoom
     :delivery_url, # for postback
     :delivery_token, # for postback
     :location, # for letter_opener
-    :delivery_options
+    :delivery_options,
+    :arbitration_method,
+    :arbitration_options
   ]
 
   # Holds configuration for each of the email accounts we wish to monitor
@@ -30,45 +35,52 @@ module MailRoom
       :ssl => true,
       :start_tls => false,
       :delete_after_delivery => false,
-      :delivery_options => {}
+      :delivery_options => {},
+      :arbitration_method => 'noop',
+      :arbitration_options => {}
     }
 
     # Store the configuration and require the appropriate delivery method
     # @param attributes [Hash] configuration options
     def initialize(attributes={})
       super(*DEFAULTS.merge(attributes).values_at(*members))
-
-      require_relative("./delivery/#{(delivery_method)}")
     end
 
-    # move to a mailbox deliverer class?
     def delivery_klass
-      case delivery_method
-      when "noop"
-        Delivery::Noop
-      when "logger"
-        Delivery::Logger
-      when "letter_opener"
-        Delivery::LetterOpener
-      when "sidekiq"
-        Delivery::Sidekiq
-      when "que"
-        Delivery::Que
-      else
-        Delivery::Postback
-      end
+      Delivery[delivery_method]
+    end
+
+    def arbitration_klass
+      Arbitration[arbitration_method]
+    end
+
+    def delivery
+      @delivery ||= delivery_klass.new(parsed_delivery_options)
+    end
+
+    def arbitrator
+      @arbitrator ||= arbitration_klass.new(parsed_arbitration_options)
+    end
+
+    def deliver?(uid)
+      arbitrator.deliver?(uid)
     end
 
     # deliver the imap email message
     # @param message [Net::IMAP::FetchData]
     def deliver(message)
-      message = message.attr['RFC822']
-      return true unless message
-      
-      delivery_klass.new(parsed_delivery_options).deliver(message)
+      body = message.attr['RFC822']
+      return true unless body
+
+      delivery.deliver(body)
     end
 
     private
+
+    def parsed_arbitration_options
+      arbitration_klass::Options.new(self)
+    end
+
     def parsed_delivery_options
       delivery_klass::Options.new(self)
     end

data/lib/mail_room/mailbox_handler.rb

@@ -14,14 +14,12 @@ module MailRoom
     def process
       # return if idling? || !running?
 
-      new_messages.each do |msg|
-        # puts msg.attr['RFC822']
-
+      new_messages.each do |message|
         # loop over delivery methods and deliver each
-        delivered = @mailbox.deliver(msg)
+        delivered = @mailbox.deliver(message)
 
         if delivered && @mailbox.delete_after_delivery
-          @imap.store(msg.seqno, "+FLAGS", [Net::IMAP::DELETED])
+          @imap.store(message.seqno, "+FLAGS", [Net::IMAP::DELETED])
         end
       end
 
@@ -45,17 +43,17 @@ module MailRoom
     # search for all new (unseen) message ids
     # @return [Array<Integer>] message ids
     def new_message_ids
-      @imap.search(@mailbox.search_command)
+      @imap.uid_search(@mailbox.search_command).select { |uid| @mailbox.deliver?(uid) }
     end
 
     # @private
     # fetch the email for all given ids in RFC822 format
     # @param ids [Array<Integer>] list of message ids
     # @return [Array<Net::IMAP::FetchData>] the net/imap messages for the given ids
-    def messages_for_ids(ids)
-      return [] if ids.empty?
+    def messages_for_ids(uids)
+      return [] if uids.empty?
 
-      @imap.fetch(ids, "RFC822")
+      @imap.uid_fetch(uids, "RFC822")
     end
   end
 end

data/lib/mail_room/version.rb

@@ -1,4 +1,4 @@
 module MailRoom
   # Current version of MailRoom gem
-  VERSION = "0.5.2"
+  VERSION = "0.6.0"
 end

data/mail_room.gemspec

@@ -22,12 +22,13 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "mocha"
   gem.add_development_dependency "bourne"
   gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "fakeredis"
 
   # for testing delivery methods
   gem.add_development_dependency "faraday"
   gem.add_development_dependency "mail"
   gem.add_development_dependency "letter_opener"
-  gem.add_development_dependency "redis"
+  gem.add_development_dependency "redis-namespace"
   gem.add_development_dependency "pg"
   gem.add_development_dependency "charlock_holmes"
 end

data/spec/lib/arbitration/redis_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+require 'mail_room/arbitration/redis'
+
+describe MailRoom::Arbitration::Redis do
+  let(:mailbox) { 
+    MailRoom::Mailbox.new(
+      arbitration_options: {
+        namespace: "mail_room"
+      }
+    ) 
+  }
+  let(:options) { described_class::Options.new(mailbox) }
+  subject       { described_class.new(options) }
+
+  # Private, but we don't care.
+  let(:redis) { subject.send(:redis) }
+
+  describe '#deliver?' do
+    context "when called the first time" do
+      it "returns true" do
+        expect(subject.deliver?(123)).to be_truthy
+      end
+
+      it "increments the delivered flag" do
+        subject.deliver?(123)
+
+        expect(redis.get("delivered:123")).to eq("1")
+      end
+
+      it "sets an expiration on the delivered flag" do
+        subject.deliver?(123)
+
+        expect(redis.ttl("delivered:123")).to be > 0
+      end
+    end
+
+    context "when called the second time" do
+      before do
+        subject.deliver?(123)
+      end
+
+      it "returns false" do
+        expect(subject.deliver?(123)).to be_falsey
+      end
+
+      it "increments the delivered flag" do
+        subject.deliver?(123)
+
+        expect(redis.get("delivered:123")).to eq("2")
+      end
+    end
+
+    context "when called for another uid" do
+      before do
+        subject.deliver?(123)
+      end
+
+      it "returns true" do
+        expect(subject.deliver?(234)).to be_truthy
+      end
+    end
+  end
+end

data/spec/lib/mailbox_handler_spec.rb

@@ -6,31 +6,52 @@ describe MailRoom::MailboxHandler do
     let(:mailbox) {MailRoom::Mailbox.new}
 
     it 'fetches and delivers all new messages from ids' do
-      imap.stubs(:search).returns([1,2])
-      imap.stubs(:fetch).returns(['message1', 'message2'])
+      imap.stubs(:uid_search).returns([1,2])
+      message1 = stub(:attr => {'RFC822' => 'message1'})
+      message2 = stub(:attr => {'RFC822' => 'message2'})
+      imap.stubs(:uid_fetch).returns([message1, message2])
       mailbox.stubs(:deliver)
 
       handler = MailRoom::MailboxHandler.new(mailbox, imap)
       handler.process
 
-      imap.should have_received(:search).with('UNSEEN')
-      imap.should have_received(:fetch).with([1,2], 'RFC822')
-      mailbox.should have_received(:deliver).with('message1')
-      mailbox.should have_received(:deliver).with('message2')
+      imap.should have_received(:uid_search).with('UNSEEN')
+      imap.should have_received(:uid_fetch).with([1,2], 'RFC822')
+      mailbox.should have_received(:deliver).with(message1)
+      mailbox.should have_received(:deliver).with(message2)
+    end
+
+    it "fetches and delivers all new messages from ids that haven't been processed yet" do
+      imap.stubs(:uid_search).returns([1,2])
+      message1 = stub(:attr => {'RFC822' => 'message1'})
+      message2 = stub(:attr => {'RFC822' => 'message2'})
+      imap.stubs(:uid_fetch).returns([message2])
+      mailbox.stubs(:deliver)
+      mailbox.stubs(:deliver?).with(1).returns(false)
+      mailbox.stubs(:deliver?).with(2).returns(true)
+
+      handler = MailRoom::MailboxHandler.new(mailbox, imap)
+      handler.process
+
+      imap.should have_received(:uid_search).with('UNSEEN')
+      imap.should have_received(:uid_fetch).with([2], 'RFC822')
+
+      mailbox.should have_received(:deliver).with(message1).never
+      mailbox.should have_received(:deliver).with(message2)
     end
 
     it 'returns no messages if there are no ids' do
-      imap.stubs(:search).returns([])
-      imap.stubs(:fetch)
+      imap.stubs(:uid_search).returns([])
+      imap.stubs(:uid_fetch)
       mailbox.search_command = 'NEW'
       mailbox.stubs(:deliver)
 
       handler = MailRoom::MailboxHandler.new(mailbox, imap)
       handler.process
 
-      imap.should have_received(:search).with('NEW')
-      imap.should have_received(:fetch).never
+      imap.should have_received(:uid_search).with('NEW')
+      imap.should have_received(:uid_fetch).never
       mailbox.should have_received(:deliver).never
     end
   end
-end
+end

data/spec/lib/mailbox_spec.rb

@@ -2,11 +2,39 @@ require 'spec_helper'
 
 describe MailRoom::Mailbox do
   describe "#deliver" do
+    context "with arbitration_method of noop" do
+      it 'arbitrates with a Noop instance' do
+        mailbox = MailRoom::Mailbox.new({:arbitration_method => 'noop'})
+        noop = stub(:deliver?)
+        MailRoom::Arbitration['noop'].stubs(:new => noop)
+
+        uid = 123
+
+        mailbox.deliver?(uid)
+
+        noop.should have_received(:deliver?).with(uid)
+      end
+    end
+
+    context "with arbitration_method of redis" do
+      it 'arbitrates with a Redis instance' do
+        mailbox = MailRoom::Mailbox.new({:arbitration_method => 'redis'})
+        redis = stub(:deliver?)
+        MailRoom::Arbitration['redis'].stubs(:new => redis)
+
+        uid = 123
+        
+        mailbox.deliver?(uid)
+
+        redis.should have_received(:deliver?).with(uid)
+      end
+    end
+
     context "with delivery_method of noop" do
       it 'delivers with a Noop instance' do
         mailbox = MailRoom::Mailbox.new({:delivery_method => 'noop'})
         noop = stub(:deliver)
-        MailRoom::Delivery::Noop.stubs(:new => noop)
+        MailRoom::Delivery['noop'].stubs(:new => noop)
 
         mailbox.deliver(stub(:attr => {'RFC822' => 'a message'}))
 
@@ -18,7 +46,7 @@ describe MailRoom::Mailbox do
       it 'delivers with a Logger instance' do
         mailbox = MailRoom::Mailbox.new({:delivery_method => 'logger'})
         logger = stub(:deliver)
-        MailRoom::Delivery::Logger.stubs(:new => logger)
+        MailRoom::Delivery['logger'].stubs(:new => logger)
 
         mailbox.deliver(stub(:attr => {'RFC822' => 'a message'}))
 
@@ -30,7 +58,7 @@ describe MailRoom::Mailbox do
       it 'delivers with a Postback instance' do
         mailbox = MailRoom::Mailbox.new({:delivery_method => 'postback'})
         postback = stub(:deliver)
-        MailRoom::Delivery::Postback.stubs(:new => postback)
+        MailRoom::Delivery['postback'].stubs(:new => postback)
 
         mailbox.deliver(stub(:attr => {'RFC822' => 'a message'}))
 
@@ -42,7 +70,7 @@ describe MailRoom::Mailbox do
       it 'delivers with a LetterOpener instance' do
         mailbox = MailRoom::Mailbox.new({:delivery_method => 'letter_opener'})
         letter_opener = stub(:deliver)
-        MailRoom::Delivery::LetterOpener.stubs(:new => letter_opener)
+        MailRoom::Delivery['letter_opener'].stubs(:new => letter_opener)
 
         mailbox.deliver(stub(:attr => {'RFC822' => 'a message'}))
 
@@ -54,7 +82,7 @@ describe MailRoom::Mailbox do
       it "doesn't deliver the message" do
         mailbox = MailRoom::Mailbox.new({:delivery_method => 'noop'})
         noop = stub(:deliver)
-        MailRoom::Delivery::Noop.stubs(:new => noop)
+        MailRoom::Delivery['noop'].stubs(:new => noop)
 
         mailbox.deliver(stub(:attr => {'FLAGS' => [:Seen, :Recent]}))
 

data/spec/spec_helper.rb

@@ -6,6 +6,7 @@ require 'bundler/setup'
 require 'rspec'
 require 'mocha/api'
 require 'bourne'
+require 'fakeredis/rspec'
 
 require File.expand_path('../../lib/mail_room', __FILE__)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mail_room
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Tony Pitale
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-24 00:00:00.000000000 Z
+date: 2015-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -123,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redis
+  name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -183,9 +197,13 @@ files:
 - Rakefile
 - bin/mail_room
 - lib/mail_room.rb
+- lib/mail_room/arbitration.rb
+- lib/mail_room/arbitration/noop.rb
+- lib/mail_room/arbitration/redis.rb
 - lib/mail_room/cli.rb
 - lib/mail_room/configuration.rb
 - lib/mail_room/coordinator.rb
+- lib/mail_room/delivery.rb
 - lib/mail_room/delivery/letter_opener.rb
 - lib/mail_room/delivery/logger.rb
 - lib/mail_room/delivery/noop.rb
@@ -198,6 +216,7 @@ files:
 - lib/mail_room/version.rb
 - mail_room.gemspec
 - spec/fixtures/test_config.yml
+- spec/lib/arbitration/redis_spec.rb
 - spec/lib/cli_spec.rb
 - spec/lib/configuration_spec.rb
 - spec/lib/coordinator_spec.rb
@@ -235,6 +254,7 @@ summary: mail_room will proxy email (gmail) from IMAP to a callback URL, logger,
   letter_opener
 test_files:
 - spec/fixtures/test_config.yml
+- spec/lib/arbitration/redis_spec.rb
 - spec/lib/cli_spec.rb
 - spec/lib/configuration_spec.rb
 - spec/lib/coordinator_spec.rb