inbox-sync 0.3.0 → 0.4.0

data/README.md

@@ -95,6 +95,10 @@ The (optional) folder on the source to create and archive (move) source inbox me
 
 A logger to use.  Defaults to ruby's `Logger` on `STDOUT`.
 
+### `max_threads`
+
+This is the number of threads to sync mail items with.  Each time a sync is run, the number of mail items that need to sync'd will be broken into `max_threads` groups and each group will sync its mail items in its own thread.  The default is `1` which means each message will be sync'd sequentially.
+
 ## Running
 
 InboxSync provides a `Runner` class that will loop indefinitely, running syncs every `:interval` seconds.  Stick it in a daemon, a rake task, a CLI, or whatever depending on how you want to invoke it.  Here is an example using it in a basic ruby script:

data/lib/inbox-sync/config.rb

@@ -16,6 +16,7 @@ module InboxSync
     opt :archive_folder, :default => 'Archived'
     opt :logger, Logger, :required => true, :default => STDOUT
     opt :filters, :default => [], :required => true
+    opt :max_threads, :default => 1, :required => true
 
     def filter(*args, &block)
       filters << Filter.new(*args, &block)

data/lib/inbox-sync/runner.rb

@@ -17,6 +17,7 @@ module InboxSync
       @logger = opts[:logger] || Logger.new(STDOUT)
       @shutdown = false
       @running_syncs_thread = nil
+      @sync_groups = []
 
       Signal.trap('SIGINT',  lambda{ raise Interrupt, 'SIGINT'  })
       Signal.trap('SIGQUIT', lambda{ raise Interrupt, 'SIGQUIT' })
@@ -93,7 +94,10 @@ module InboxSync
       thread_log "starting syncs..."
 
       begin
-        run_syncs
+        @syncs.each do |sync|
+          run_threaded_sync(sync) if !self.shutdown?
+        end
+        GC.start
       rescue Exception => err
         thread_log_error(err, :error)
       end
@@ -101,22 +105,30 @@ module InboxSync
       thread_log "...syncs finished"
     end
 
-    def run_syncs
-      @syncs.each do |sync|
-        begin
-          sync.setup
-          sync.run(self)
-        rescue Exception => err
-          run_sync_handle_error(sync, err)
+    def run_threaded_sync(sync)
+      begin
+        sync.setup
+        @sync_groups = sync.mail_item_groups
+        thread_log "starting #{@sync_groups.size} sync threads..."
+        @sync_groups.each do |group|
+          thread_log "starting #{group.id} sync thread", :debug
+          group.run(self)
+          sleep(1)
         end
+      rescue Exception => err
+        run_sync_handle_error(sync, err)
+      end
 
-        begin
-          sync.teardown
-        rescue Exception => err
-          run_sync_handle_error(sync, err)
+      begin
+        @sync_groups.each do |group|
+          thread_log "waiting for #{group.id} sync thread to join...", :debug
+          group.join
         end
+        thread_log "... all #{@sync_groups.size} sync threads are finished"
+        sync.teardown
+      rescue Exception => err
+        run_sync_handle_error(sync, err)
       end
-      GC.start
     end
 
     def run_sync_handle_error(sync, err)

data/lib/inbox-sync/sync/mail_item_group.rb

@@ -0,0 +1,46 @@
+module InboxSync
+  class Sync; end
+
+  class Sync::MailItemGroup
+
+    attr_reader :thread, :items
+
+    def initialize(sync)
+      @sync = sync
+      @thread = nil
+      @items = []
+    end
+
+    def id
+      "#<#{self.class}:#{'0x%x' % (self.object_id << 1)}>"
+    end
+
+    def add(mail_item)
+      @items << mail_item
+    end
+
+    def run(runner=nil)
+      @runner = runner
+      @thread = Thread.new { run_items_thread }
+    end
+
+    def join
+      @thread.join
+      @runner = nil
+    end
+
+    protected
+
+    def run_items_thread
+      @items.each do |item|
+        if @runner && @runner.shutdown?
+          @sync.logger.debug "* the runner has been shutdown - aborting #{id} sync thread"
+          break
+        end
+        @sync.run(item)
+      end
+    end
+
+  end
+
+end

data/lib/inbox-sync/sync.rb

@@ -3,6 +3,7 @@ require 'net/smtp'
 
 require 'inbox-sync/config'
 require 'inbox-sync/filter_actions'
+require 'inbox-sync/sync/mail_item_group'
 require 'inbox-sync/notice/sync_mail_item_error'
 
 module InboxSync
@@ -53,25 +54,36 @@ module InboxSync
       logger.info "=== #{config_log_detail(@config.source)} sync finished. ==="
     end
 
-    def run(runner=nil)
-      return if runner && runner.shutdown?
-      each_source_mail_item(runner) do |mail_item|
-        begin
-          logger.debug "** #{mail_item.inspect}"
-          response = send_to_dest(mail_item)
-          dest_uid = parse_append_response_uid(response)
-          apply_dest_filters(dest_uid)
-        rescue Exception => err
-          log_error(err)
-          notify(Notice::SyncMailItemError.new(@notify_smtp, @config.notify, {
-            :error => err,
-            :mail_item => mail_item,
-            :sync => self
-          }))
-        ensure
-          archive_on_source(mail_item)
-          mail_item = nil
-        end
+    # this splits the mail_items list into `@config.max_threads` lists
+    # this spreads mails evenly across the groups with earlier items
+    # appearing earliest in each list
+
+    def mail_item_groups
+      num_groups = @config.max_threads
+      groups = []
+      num_groups.times { groups << MailItemGroup.new(self) }
+      get_mail_items.each_with_index do |item, i|
+        groups[i % num_groups].add(item)
+      end
+      groups
+    end
+
+    def run(mail_item)
+      begin
+        logger.debug "** #{mail_item.inspect}"
+        response = send_to_dest(mail_item)
+        dest_uid = parse_append_response_uid(response)
+        apply_dest_filters(dest_uid)
+      rescue Exception => err
+        log_error(err)
+        notify(Notice::SyncMailItemError.new(@notify_smtp, @config.notify, {
+          :error => err,
+          :mail_item => mail_item,
+          :sync => self
+        }))
+      ensure
+        archive_on_source(mail_item)
+        mail_item = nil
       end
     end
 
@@ -98,21 +110,6 @@ module InboxSync
       true
     end
 
-    def each_source_mail_item(runner=nil)
-      logger.info "* find: #{config_log_detail(@config.source)}, #{@config.source.inbox.inspect}..."
-      items = MailItem.find(@source_imap)
-      logger.info "* ...found #{items.size} mail items"
-
-      items.each do |mail_item|
-        if runner && runner.shutdown?
-          logger.info "* the runner has been shutdown - aborting the sync"
-          break
-        end
-        yield mail_item
-      end
-      items = nil
-    end
-
     # Send a mail item to the destination:
     # The idea here is that destinations may not accept corrupted or invalid
     # mail items.  If appending the original mail item in any way fails,
@@ -223,6 +220,13 @@ module InboxSync
       end
     end
 
+    def get_mail_items
+      logger.info "* find: #{config_log_detail(@config.source)}, #{@config.source.inbox.inspect}..."
+      items = MailItem.find(@source_imap)
+      logger.info "* ...found #{items.size} mail items"
+      items
+    end
+
     def logout_imap(imap, config)
       logger.debug "* LOGOUT: #{config_log_detail(config)}"
       imap.logout

data/lib/inbox-sync/version.rb

@@ -1,3 +1,3 @@
 module InboxSync
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/test/config_test.rb

@@ -42,6 +42,11 @@ module InboxSync
       :required => true
     }
 
+    should have_option :max_threads, {
+      :default => 1,
+      :required => true
+    }
+
     should have_instance_methods :validate!, :filter
     should have_instance_methods :contains, :like, :includes, :inc
     should have_instance_methods :starts_with, :ends_with, :sw, :ew

data/test/sync/login_test.rb

@@ -45,66 +45,4 @@ module InboxSync
 
   end
 
-  class LoggedInTests < ConfiguredSyncTests
-    desc 'and is logged in and ready to run'
-    before do
-      reset_source_inbox(@sync)
-      reset_source_archive(@sync)
-
-      @sync_archive_folder = @sync.config.archive_folder
-      @mail_item = MailItem.find(@sync.source_imap).first
-    end
-
-    after do
-      @sync.config.archive_folder = @sync_archive_folder
-      @sync.source_imap.select(@sync.config.source.inbox)
-
-      reset_source_archive(@sync)
-    end
-
-    should "run without any errors" do
-      assert_nothing_raised do
-        subject.run
-      end
-    end
-
-  end
-
-  # class ArchiveTests < RunTests
-  #   should "remove the message from the inbox" do
-  #     @sync.archive_from_source(@mail_item)
-
-  #     assert_not_included @mail_item.uid, @sync.source_imap.uid_search(['ALL'])
-  #     assert_empty @sync.source_imap.uid_search(['ALL'])
-  #   end
-
-  #   should "move it to the archive folder" do
-  #     @sync.archive_from_source(@mail_item)
-  #     @sync.source_imap.select(@sync.config.archive_folder)
-
-  #     assert_equal 1, @sync.source_imap.uid_search(['ALL']).size
-  #   end
-
-  #   should "not archive if no archive folder configured" do
-  #     @sync.config.archive_folder = nil
-  #     @sync.archive_from_source(@mail_item)
-  #     @sync.source_imap.select(@sync.config.source.inbox)
-
-  #     assert_not_included @mail_item.uid, @sync.source_imap.uid_search(['ALL'])
-  #   end
-
-  # end
-
-  # class AppendTests < LoggedInTests
-  #   should "append a source msg onto the dest" do
-  #     assert_nothing_raised do
-  #       @sync.append_to_dest(test_mail_item)
-  #     end
-  #   end
-
-  #   should "parse the destination UID when appending mail items" do
-  #     assert_match /\A\d+\Z/, @sync.append_to_dest(test_mail_item)
-  #   end
-  # end
-
 end

data/test/sync/mail_item_group_test.rb

@@ -0,0 +1,19 @@
+require 'assert'
+
+require 'inbox-sync/sync/mail_item_group'
+
+module InboxSync
+
+  class SyncMailItemGroupTests < Assert::Context
+    desc "a runner mail item group"
+    before do
+      @group = Sync::MailItemGroup.new('dummy sync')
+    end
+    subject { @group }
+
+    should have_readers :thread, :items
+    should have_instance_methods :id, :add, :run, :join
+
+  end
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: inbox-sync
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 15
   prerelease: 
   segments: 
   - 0
-  - 3
+  - 4
   - 0
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Kelly Redding
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-15 00:00:00 Z
+date: 2012-06-16 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   type: :development
@@ -91,6 +91,7 @@ files:
 - lib/inbox-sync/notice/sync_mail_item_error.rb
 - lib/inbox-sync/runner.rb
 - lib/inbox-sync/sync.rb
+- lib/inbox-sync/sync/mail_item_group.rb
 - lib/inbox-sync/version.rb
 - log/.gitkeep
 - test/config/filter_test.rb
@@ -102,6 +103,7 @@ files:
 - test/runner_test.rb
 - test/sync/basic_test.rb
 - test/sync/login_test.rb
+- test/sync/mail_item_group_test.rb
 homepage: http://github.com/kellyredding/inbox-sync
 licenses: []
 
@@ -145,3 +147,4 @@ test_files:
 - test/runner_test.rb
 - test/sync/basic_test.rb
 - test/sync/login_test.rb
+- test/sync/mail_item_group_test.rb