inbox-sync 0.1.0

data/lib/inbox-sync/runner.rb

@@ -0,0 +1,123 @@
+require 'inbox-sync/notice/run_sync_error'
+
+module InboxSync
+
+  class Runner
+
+    attr_reader :syncs, :interval, :logger
+
+    def initialize(*args)
+      opts, syncs = [
+        args.last.kind_of?(Hash) ? args.pop : {},
+        args.flatten
+      ]
+
+      @syncs = syncs || []
+      @interval = opts[:interval].kind_of?(Fixnum) ? opts[:interval] : -1
+      @logger = opts[:logger] || Logger.new(STDOUT)
+      @shutdown = false
+      @running_syncs_thread = nil
+
+      Signal.trap('SIGINT', lambda{ self.stop })
+      Signal.trap('SIGQUIT', lambda{ self.stop })
+    end
+
+    def start
+      startup
+      loop do
+        break if @shutdown
+        loop_run
+      end
+      shutdown
+    end
+
+    def stop
+      main_log "Stop signal - waiting for any running syncs to finish."
+      @shutdown = true
+    end
+
+    protected
+
+    def startup
+      main_log "Starting up the runner."
+    end
+
+    def shutdown
+      main_log "Shutting down the runner"
+    end
+
+    def loop_run
+      main_log "Starting syncs in fresh thread."
+
+      @running_syncs_thread = Thread.new do
+        thread_log "starting syncs..."
+
+        begin
+          run_syncs
+        rescue Exception => err
+          thread_log_error(err, :error)
+        end
+
+        thread_log "...syncs finished"
+      end
+
+      if @interval < 0
+        main_log "run-once interval - signaling stop"
+        stop
+        @interval = 0
+      end
+
+      main_log "Sleeping for #{@interval} seconds."
+      sleep(@interval)
+      main_log "Woke from sleep - waiting for running syncs thread to join..."
+      @running_syncs_thread.join
+      main_log "... running sycs thread joined."
+    end
+
+    def run_syncs
+      @syncs.each do |sync|
+        begin
+          sync.setup
+          sync.run
+        rescue Exception => err
+          run_sync_handle_error(sync, err)
+        end
+
+        begin
+          sync.teardown
+        rescue Exception => err
+          run_sync_handle_error(sync, err)
+        end
+      end
+      GC.start
+    end
+
+    def run_sync_handle_error(sync, err)
+      thread_log_error(err)
+      notice = Notice::RunSyncError.new(sync.notify_smtp, sync.config.notify, {
+        :error => err,
+        :sync => sync
+      })
+      sync.notify(notice)
+    end
+
+    def thread_log_error(err, level=:warn)
+      thread_log "#{err.message} (#{err.class.name})", level
+      err.backtrace.each { |bt| thread_log bt.to_s, level }
+    end
+
+    def main_log(msg, level=:info)
+      log "[MAIN]: #{msg}", level
+    end
+
+    def thread_log(msg, level=:info)
+      log "[THREAD]: #{msg}", level
+    end
+
+    def log(msg, level)
+      @logger.send(level, msg)
+    end
+
+  end
+
+end

data/lib/inbox-sync/sync.rb

@@ -0,0 +1,249 @@
+require 'net/imap'
+require 'net/smtp'
+
+require 'inbox-sync/config'
+require 'inbox-sync/notice/sync_mail_item_error'
+
+module InboxSync
+
+  class Sync
+
+    attr_reader :config, :source_imap, :notify_smtp
+
+    def initialize(configs={})
+      @config = InboxSync::Config.new(configs)
+      @source_imap = nil
+      @notify_smtp = nil
+      @logged_in   = false
+    end
+
+    def logger
+      @config.logger
+    end
+
+    def uid
+      "#{@config.source.login.user}:#{@config.source.host}"
+    end
+
+    def name
+      "#{@config.source.login.user} (#{@config.source.host})"
+    end
+
+    def logged_in?
+      !!@logged_in
+    end
+
+    def configure(&config_block)
+      @config.instance_eval(&config_block) if config_block
+      self
+    end
+
+    def setup
+      logger.info "=== #{config_log_detail(@config.source)} sync started. ==="
+
+      @notify_smtp ||= setup_smtp(:notify, @config.notify)
+      @config.validate!
+      login if !logged_in?
+    end
+
+    def teardown
+      logout if logged_in?
+      @source_imap = @notify_smtp = nil
+      logger.info "=== #{config_log_detail(@config.source)} sync finished. ==="
+    end
+
+    def run
+      each_source_mail_item do |mail_item|
+        begin
+          response = send_to_dest(mail_item)
+          dest_uid = parse_append_response_uid(response)
+          logger.debug "** dest uid: #{dest_uid.inspect}"
+        rescue Exception => err
+          log_error(err)
+          notify(Notice::SyncMailItemError.new(@notify_smtp, @config.notify, {
+            :error => err,
+            :mail_item => mail_item,
+            :sync => self
+          }))
+        ensure
+          archive_on_source(mail_item)
+          mail_item = nil
+        end
+      end
+    end
+
+    def notify(notice)
+      logger.info "** sending '#{notice.subject}' to #{notice.to.inspect}"
+      begin
+        notice.send
+      rescue Exception => err
+        log_error(err)
+      end
+    end
+
+    protected
+
+    def login
+      @source_imap = login_imap(:source, @config.source)
+      @logged_in = true
+      true
+    end
+
+    def logout
+      logout_imap(@source_imap, @config.source)
+      @logged_in = false
+      true
+    end
+
+    def each_source_mail_item
+      items = MailItem.find(@source_imap)
+      logger.info "* found #{items.size} mails"
+
+      items.each do |mail_item|
+        logger.debug "** #{mail_item.inspect}"
+        yield mail_item
+      end
+      items = nil
+    end
+
+    # Send a mail item to the destination:
+    # The idea here is that destinations may not accept corrupted or invalid
+    # mail items.  If appending the original mail item in any way fails,
+    # create a stripped down version of the mail item and try to append that.
+    # If appending the stripped down version still fails, error on up
+    # and it will be archived and notified.
+
+    def send_to_dest(mail_item)
+      begin
+        append_to_dest(mail_item)
+      rescue Exception => err
+        log_error(err)
+
+        logger.debug "** trying to append a the stripped down version"
+        append_to_dest(mail_item.stripped, 'stripped down ')
+      end
+    end
+
+    def append_to_dest(mail_item, desc='')
+      logger.info "** Appending #{desc}#{mail_item.uid} to dest #{@config.dest.inbox}..."
+
+      inbox  = @config.dest.inbox
+      mail_s = mail_item.meta['RFC822']
+      flags  = []
+      date   = mail_item.meta['INTERNALDATE']
+
+      using_dest_imap do |imap|
+        imap.append(inbox, mail_s, flags, date)
+      end
+    end
+
+    def archive_on_source(mail_item)
+      folder = @config.archive_folder
+      if !folder.nil? && !folder.empty?
+        logger.debug "** Archiving #{mail_item.uid.inspect}"
+
+        begin
+          @source_imap.select(folder)
+        rescue Net::IMAP::NoResponseError => err
+          logger.debug "* Creating #{folder.inspect} archive folder"
+          @source_imap.create(folder)
+        ensure
+          @source_imap.select(@config.source.inbox)
+        end
+
+        mark_as_seen(@source_imap, mail_item.uid)
+
+        logger.debug "** Copying #{mail_item.uid.inspect} to #{folder.inspect}"
+        @source_imap.uid_copy(mail_item.uid, folder)
+      end
+
+      mark_as_deleted(@source_imap, mail_item.uid)
+
+      expunge_imap(@source_imap, @config.source)
+
+      @source_imap.expunge
+    end
+
+    def using_dest_imap
+      dest_imap = login_imap(:dest, @config.dest)
+      result = yield dest_imap
+      logout_imap(dest_imap, @config.dest)
+      result
+    end
+
+    def login_imap(named, config)
+      logger.debug "* LOGIN: #{config_log_detail(config)}"
+      begin
+        named_imap = Net::IMAP.new(config.host, config.port, config.ssl)
+      rescue Errno::ECONNREFUSED => err
+        raise Errno::ECONNREFUSED, "#{named} imap {:host => #{config.host}, :port => #{config.port}, :ssl => #{config.ssl}}: #{err.message}"
+      end
+
+      begin
+        named_imap.login(config.login.user, config.login.pw)
+      rescue Net::IMAP::NoResponseError => err
+        raise Net::IMAP::NoResponseError, "#{named} imap #{config.login.to_hash.inspect}: #{err.message}"
+      end
+
+      logger.debug "* SELECT #{config.inbox.inspect}: #{config_log_detail(config)}"
+      begin
+        named_imap.select(config.inbox)
+      rescue Net::IMAP::NoResponseError => err
+        raise Net::IMAP::NoResponseError, "#{named} imap: #{err.message}"
+      end
+
+      expunge_imap(named_imap, config)
+
+      named_imap
+    end
+
+    def expunge_imap(imap, config)
+      if config.expunge
+        logger.debug "* EXPUNGE #{config.inbox.inspect}: #{config_log_detail(config)}"
+        imap.expunge
+      end
+    end
+
+    def logout_imap(imap, config)
+      logger.debug "* LOGOUT: #{config_log_detail(config)}"
+      imap.logout
+    end
+
+    def setup_smtp(named, config)
+      logger.debug "* SMTP: #{config_log_detail(config)}"
+
+      named_smtp = Net::SMTP.new(config.host, config.port)
+      named_smtp.enable_starttls if config.tls
+
+      named_smtp
+    end
+
+    def config_log_detail(config)
+      "host=#{config.host.inspect}, user=#{config.login.user.inspect}"
+    end
+
+    def log_error(err)
+      logger.warn "#{err.message} (#{err.class.name})"
+      err.backtrace.each { |bt| logger.warn bt.to_s }
+    end
+
+    # Given a response like this:
+    # #<struct Net::IMAP::TaggedResponse tag="RUBY0012", name="OK", data=#<struct Net::IMAP::ResponseText code=#<struct Net::IMAP::ResponseCode name="APPENDUID", data="6 9">, text=" (Success)">, raw_data="RUBY0012 OK [APPENDUID 6 9] (Success)\r\n">
+    # (here '9' is the UID)
+    def parse_append_response_uid(response)
+      response.data.code.data.split(/\s+/).last
+    end
+
+    def mark_as_seen(imap, uid)
+      logger.debug "** Marking #{uid.inspect} as :Seen"
+      imap.uid_store(uid, "+FLAGS", [:Seen])
+    end
+
+    def mark_as_deleted(imap, uid)
+      logger.debug "** Marking #{uid.inspect} as :Deleted"
+      imap.uid_store(uid, "+FLAGS", [:Deleted])
+    end
+
+  end
+
+end

data/lib/inbox-sync/version.rb

@@ -0,0 +1,3 @@
+module InboxSync
+  VERSION = "0.1.0"
+end

data/lib/inbox-sync.rb

@@ -0,0 +1,16 @@
+require 'inbox-sync/mail_item'
+require 'inbox-sync/config'
+require 'inbox-sync/sync'
+require 'inbox-sync/runner'
+
+module InboxSync
+
+  def self.new(settings={})
+    Sync.new(settings)
+  end
+
+  def self.run(*args)
+    Runner.new(*args).start
+  end
+
+end

data/test/config_test.rb

@@ -0,0 +1,302 @@
+require 'assert'
+require 'ns-options/assert_macros'
+
+require 'inbox-sync/config'
+
+module InboxSync
+
+  class ConfigTests < Assert::Context
+    include NsOptions::AssertMacros
+
+    before do
+      @config = InboxSync::Config.new
+    end
+    subject { @config }
+
+    should have_option :source, InboxSync::Config::IMAPConfig, {
+      :default => {},
+      :required => true
+    }
+
+    should have_option :dest,   InboxSync::Config::IMAPConfig, {
+      :default => {},
+      :required => true
+    }
+
+    should have_option :notify, InboxSync::Config::SMTPConfig, {
+      :default => {},
+      :required => true
+    }
+
+    should have_option :archive_folder, {
+      :default => "Archived"
+    }
+
+    should have_option :logger, Logger, {
+      :required => true,
+      :default => STDOUT
+    }
+
+    should have_instance_method  :validate!
+
+    should "complain if missing :source config" do
+      assert_raises ArgumentError do
+        subject.source = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :dest config" do
+      assert_raises ArgumentError do
+        subject.dest = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :notify config" do
+      assert_raises ArgumentError do
+        subject.notify = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :archive_folder config" do
+      assert_raises ArgumentError do
+        subject.archive_folder = nil
+        subject.validate!
+      end
+    end
+
+    should "validate its source" do
+      assert_raises ArgumentError do
+        subject.source.host = nil
+        subject.validate!
+      end
+    end
+
+    should "validate its dest" do
+      assert_raises ArgumentError do
+        subject.dest.host = nil
+        subject.validate!
+      end
+    end
+
+    should "validate its notify" do
+      assert_raises ArgumentError do
+        subject.notify.host = nil
+        subject.validate!
+      end
+    end
+
+  end
+
+  class CredentialsTests < ConfigTests
+    subject { @config.source.login }
+
+    should have_option :user, :required => true
+    should have_option :pw, :required => true
+
+    should have_instance_method :validate!
+
+    should "complain if missing :user config" do
+      assert_raises ArgumentError do
+        subject.user = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :pw config" do
+      assert_raises ArgumentError do
+        subject.pw = nil
+        subject.validate!
+      end
+    end
+
+    should "be built from set of args" do
+      cred = InboxSync::Config::Credentials.new 'me', 'secret'
+
+      assert_equal 'me',     cred.user
+      assert_equal 'secret', cred.pw
+    end
+
+  end
+
+  class IMAPConfigTests < ConfigTests
+    subject { @config.source }
+
+    should have_option :host, :required => true
+
+    should have_option :port, {
+      :default => 143,
+      :required => true
+    }
+
+    should have_option :ssl, NsOptions::Boolean, {
+      :default => false,
+      :required => true
+    }
+
+    should have_option :login, InboxSync::Config::Credentials, {
+      :default => {},
+      :required => true
+    }
+
+    should have_option :inbox, {
+      :default => "INBOX",
+      :required => true
+    }
+
+    should have_option :expunge, NsOptions::Boolean, {
+      :default => true,
+      :required => true
+    }
+
+    should "complain if missing :host config" do
+      assert_raises ArgumentError do
+        subject.host = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :port config" do
+      assert_raises ArgumentError do
+        subject.port = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :ssl config" do
+      assert_raises ArgumentError do
+        subject.ssl = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :login config" do
+      assert_raises ArgumentError do
+        subject.login = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :inbox config" do
+      assert_raises ArgumentError do
+        subject.inbox = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :host config" do
+      assert_raises ArgumentError do
+        subject.expunge = nil
+        subject.validate!
+      end
+    end
+
+  end
+
+  class SMTPConfigTests < ConfigTests
+    subject { @config.notify }
+
+    should have_option :host, :required => true
+
+    should have_option :port, {
+      :default => 25,
+      :required => true
+    }
+
+    should have_option :tls,  NsOptions::Boolean, {
+      :default => false,
+      :required => true
+    }
+
+    should have_option :helo, :required => true
+
+    should have_option :login, InboxSync::Config::Credentials, {
+      :default => {},
+      :required => true
+    }
+
+    should have_option :authtype, {
+      :default => :login,
+      :required => true
+    }
+
+    should have_option :from_addr, :required => true
+
+    should have_option :to_addr, :required => true
+
+    should have_instance_method  :validate!
+
+    should "complain if missing :host config" do
+      assert_raises ArgumentError do
+        subject.host = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :port config" do
+      assert_raises ArgumentError do
+        subject.port = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :tls config" do
+      assert_raises ArgumentError do
+        subject.tls = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :helo config" do
+      assert_raises ArgumentError do
+        subject.helo = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :login config" do
+      assert_raises ArgumentError do
+        subject.login = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :authtype config" do
+      assert_raises ArgumentError do
+        subject.authtype = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :from_addr config" do
+      assert_raises ArgumentError do
+        subject.from_addr = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if missing :to_addr config" do
+      assert_raises ArgumentError do
+        subject.to_addr = nil
+        subject.validate!
+      end
+    end
+
+    should "complain if empty addrs configs" do
+      assert_raises ArgumentError do
+        subject.from_addr = ""
+        subject.validate!
+      end
+
+      assert_raises ArgumentError do
+        subject.to_addr = ""
+        subject.validate!
+      end
+    end
+
+
+  end
+
+end