larch 1.1.0.dev.20100120 → 1.1.0.dev.20100206

data/HISTORY

@@ -18,6 +18,9 @@ Version 1.1.0 (git)
   * Folders are now copied recursively by default.
   * Progress information is now displayed regularly while scanning large
     mailboxes.
+  * Added --delete option to delete messages from the source after copying them
+    to the destination, or if they already exist at the destination.
+  * Added --expunge option to expunge deleted messages from the source.
   * Added --sync-flags option to synchronize message flags (like Seen, Flagged,
     etc.) from the source server to the destination server for messages that
     already exist on the destination.

data/README.rdoc

@@ -10,7 +10,7 @@ accounts.
 
 *Author*::    Ryan Grove (mailto:ryan@wonko.com)
 *Version*::   1.1.0 (git)
-*Copyright*:: Copyright (c) 2009 Ryan Grove. All rights reserved.
+*Copyright*:: Copyright (c) 2010 Ryan Grove. All rights reserved.
 *License*::   GPL 2.0 (http://opensource.org/licenses/gpl-2.0.php)
 *Website*::   http://github.com/rgrove/larch
 
@@ -28,7 +28,7 @@ Latest development version:
 
   larch [config section] [options]
   larch --from <uri> --to <uri> [options]
-  
+
   Server Options:
            --from, -f <s>:   URI of the source IMAP server
     --from-folder, -F <s>:   Source folder to copy from (default: INBOX)
@@ -39,13 +39,16 @@ Latest development version:
         --to-pass, -P <s>:   Destination server password (default: prompt)
         --to-user, -U <s>:   Destination server username (default: prompt)
 
-  Sync Options:
+  Copy Options:
                 --all, -a:   Copy all folders recursively
      --all-subscribed, -s:   Copy all subscribed folders recursively
+             --delete, -d:   Delete messages from the source after copying
+                             them, or if they already exist at the destination
            --exclude <s+>:   List of mailbox names/patterns that shouldn't be
                              copied
        --exclude-file <s>:   Filename containing mailbox names/patterns that
                              shouldn't be copied
+            --expunge, -x:   Expunge deleted messages from the source
          --sync-flags, -S:   Sync message flags from the source to the
                              destination for messages that already exist at the
                              destination
@@ -270,18 +273,16 @@ fairly well-known trick. That said, as with anything tricky, there are caveats.
 
 ==== No hierarchical folders
 
-Similar to Gmail, Yahoo!'s IMAP gateway doesn't allow hierarchical (nested)
-folders. If you try to copy a folder hierarchy to Yahoo!, it will work, but
-you'll end up with a set of folders named "folder" and "folder.subfolder"
-rather than seeing "subfolder" as an actual subfolder of "folder".
+Similar to Gmail, Yahoo! Mail doesn't allow hierarchical (nested) folders. If
+you try to copy a folder hierarchy to Yahoo!, it will work, but you'll end up
+with a set of folders named "folder" and "folder.subfolder" rather than seeing
+"subfolder" as an actual subfolder of "folder".
 
 ==== No custom flags
 
-Yahoo! Mail IMAP doesn't appear to support custom message flags, such as the
-labels and junk/not junk flags used by Thunderbird. When transferring messages
-with custom flags to a Yahoo! Mail IMAP account, the custom flags will be lost.
-Also, Larch's <tt>sync-flags</tt> option will not work correctly if you have
-messages with custom flags.
+Yahoo! Mail IMAP doesn't support custom message flags, such as the tags and
+junk/not junk flags used by Thunderbird. When transferring messages with custom
+flags to a Yahoo! Mail IMAP account, the custom flags will be lost.
 
 ==== Here there be dragons
 
@@ -336,7 +337,7 @@ Gray II).
 
 == License
 
-Copyright (c) 2009 Ryan Grove <ryan@wonko.com>
+Copyright (c) 2010 Ryan Grove <ryan@wonko.com>
 
 Licensed under the GNU General Public License version 2.0.
 

data/bin/larch

@@ -12,7 +12,7 @@ module Larch
   options = Trollop.options do
     version "Larch #{APP_VERSION}\n" << APP_COPYRIGHT
     banner <<-EOS
-Larch syncs messages from one IMAP server to another. Awesomely.
+Larch copies messages from one IMAP server to another. Awesomely.
 
 Usage:
   larch [config section] [options]
@@ -29,11 +29,13 @@ EOS
     opt :to_pass,     "Destination server password (default: prompt)", :short => '-P', :type => :string
     opt :to_user,     "Destination server username (default: prompt)", :short => '-U', :type => :string
 
-    text "\nSync Options:"
+    text "\nCopy Options:"
     opt :all,              "Copy all folders recursively", :short => '-a'
     opt :all_subscribed,   "Copy all subscribed folders recursively", :short => '-s'
+    opt :delete,           "Delete messages from the source after copying them, or if they already exist at the destination", :short => '-d'
     opt :exclude,          "List of mailbox names/patterns that shouldn't be copied", :short => :none, :type => :strings, :multi => true
     opt :exclude_file,     "Filename containing mailbox names/patterns that shouldn't be copied", :short => :none, :type => :string
+    opt :expunge,          "Expunge deleted messages from the source", :short => '-x'
     opt :sync_flags,       "Sync message flags from the source to the destination for messages that already exist at the destination", :short => '-S'
 
     text "\nGeneral Options:"

data/lib/larch.rb

@@ -38,9 +38,10 @@ module Larch
       Net::IMAP.debug = true if @log.level == :insane
 
       # Stats
-      @copied = 0
-      @failed = 0
-      @total  = 0
+      @copied  = 0
+      @deleted = 0
+      @failed  = 0
+      @total   = 0
     end
 
     # Recursively copies all messages in all folders from the source to the
@@ -49,9 +50,10 @@ module Larch
       raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
       raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
 
-      @copied = 0
-      @failed = 0
-      @total  = 0
+      @copied  = 0
+      @deleted = 0
+      @failed  = 0
+      @total   = 0
 
       imap_from.each_mailbox do |mailbox_from|
         next if excluded?(mailbox_from.name)
@@ -82,9 +84,10 @@ module Larch
       raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
       raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
 
-      @copied = 0
-      @failed = 0
-      @total  = 0
+      @copied  = 0
+      @deleted = 0
+      @failed  = 0
+      @total   = 0
 
       mailbox_from = imap_from.mailbox(imap_from.uri_mailbox || 'INBOX')
       mailbox_to   = imap_to.mailbox(imap_to.uri_mailbox || 'INBOX')
@@ -144,7 +147,7 @@ module Larch
     end
 
     def summary
-      @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@total - @copied - @failed} untouched out of #{@total} total"
+      @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@deleted} deleted out of #{@total} total"
     end
 
 
@@ -182,19 +185,26 @@ module Larch
 
       mailbox_from.each_db_message do |from_db_message|
         guid = from_db_message.guid
+        uid  = from_db_message.uid
 
         if mailbox_to.has_guid?(guid)
-          next unless @config['sync_flags']
-
           begin
-            to_db_message = mailbox_to.fetch_db_message(guid)
+            if @config['sync_flags']
+              to_db_message = mailbox_to.fetch_db_message(guid)
+
+              if to_db_message.flags != from_db_message.flags
+                new_flags = from_db_message.flags_str
+                new_flags = '(none)' if new_flags.empty?
 
-            if to_db_message.flags != from_db_message.flags
-              new_flags = from_db_message.flags_str
-              new_flags = '(none)' if new_flags.empty?
+                @log.info "[>] syncing flags: uid #{uid}: #{new_flags}"
+                mailbox_to.set_flags(guid, from_db_message.flags)
+              end
+            end
 
-              @log.info "syncing flags: UID #{to_db_message.uid}: #{new_flags}"
-              mailbox_to.set_flags(guid, from_db_message.flags)
+            if @config['delete'] && !from_db_message.flags.include?(:Deleted)
+              @log.info "[<] deleting uid #{uid} (already exists at destination)"
+              mailbox_from.set_flags(guid, [:Deleted], true)
+              @deleted += 1
             end
           rescue Larch::IMAP::Error => e
             @log.error e.message
@@ -213,17 +223,32 @@ module Larch
             from = '?'
           end
 
-          @log.info "copying: #{from} - #{msg.envelope.subject}"
+          @log.info "[>] copying uid #{uid}: #{from} - #{msg.envelope.subject}"
 
           mailbox_to << msg
           @copied += 1
 
+          if @config['delete']
+            @log.info "[<] deleting uid #{uid}"
+            mailbox_from.set_flags(guid, [:Deleted], true)
+            @deleted += 1
+          end
+
         rescue Larch::IMAP::Error => e
           @failed += 1
           @log.error e.message
           next
         end
       end
+
+      if @config['expunge']
+        begin
+          @log.debug "[<] expunging deleted messages"
+          mailbox_from.expunge
+        rescue Larch::IMAP::Error => e
+          @log.error e.message
+        end
+      end
     end
 
     def db_maintenance

data/lib/larch/config.rb

@@ -8,9 +8,11 @@ class Config
     'all-subscribed'   => false,
     'config'           => File.join('~', '.larch', 'config.yaml'),
     'database'         => File.join('~', '.larch', 'larch.db'),
+    'delete'           => false,
     'dry-run'          => false,
     'exclude'          => [],
     'exclude-file'     => nil,
+    'expunge'          => false,
     'from'             => nil,
     'from-folder'      => nil, # actually INBOX; see validate()
     'from-pass'        => nil,

data/lib/larch/imap.rb

@@ -198,7 +198,7 @@ class IMAP
 
       raise unless (retries += 1) <= @options[:max_retries]
 
-      warn "#{e.class.name}: #{e.message} (reconnecting)"
+      warning "#{e.class.name}: #{e.message} (reconnecting)"
 
       reset
       sleep 1 * retries
@@ -211,7 +211,7 @@ class IMAP
 
       raise unless (retries += 1) <= @options[:max_retries]
 
-      warn "#{e.class.name}: #{e.message} (will retry)"
+      warning "#{e.class.name}: #{e.message} (will retry)"
 
       sleep 1 * retries
       retry
@@ -292,7 +292,7 @@ class IMAP
       # verification errors.
       raise if e.is_a?(OpenSSL::SSL::SSLError) && e.message =~ /certificate verify failed/
 
-      warn "#{e.class.name}: #{e.message} (will retry)"
+      warning "#{e.class.name}: #{e.message} (will retry)"
 
       reset
       sleep 1 * retries

data/lib/larch/imap/mailbox.rb

@@ -74,21 +74,8 @@ class Mailbox
         raise Larch::IMAP::Error, "mailbox cannot contain messages: #{@name}"
       end
 
-      flags = message.flags.dup
-
-      # Don't set any flags that aren't supported on the destination mailbox.
-      flags.delete_if do |flag|
-        # The \Recent flag is read-only, so we shouldn't try to set it.
-        next true if flag == :Recent
-
-        unless @flags.include?(flag) || @perm_flags.include?(:*) || @perm_flags.include?(flag)
-          info "flag not supported on destination: #{flag}"
-          true
-        end
-      end
-
       debug "appending message: #{message.guid}"
-      @imap.conn.append(@name_utf7, message.rfc822, flags, message.internaldate) unless @imap.options[:dry_run]
+      @imap.conn.append(@name_utf7, message.rfc822, get_supported_flags(message.flags), message.internaldate) unless @imap.options[:dry_run]
     end
 
     true
@@ -114,6 +101,19 @@ class Mailbox
     mailboxes.each {|mb| yield mb }
   end
 
+  # Expunges this mailbox, permanently removing all messages with the \Deleted
+  # flag.
+  def expunge
+    return false unless imap_select
+
+    @imap.safely do
+      debug "expunging deleted messages"
+
+      @last_scan = nil
+      @imap.conn.expunge unless @imap.options[:dry_run]
+    end
+  end
+
   # Returns a Larch::IMAP::Message struct representing the message with the
   # specified Larch _guid_, or +nil+ if the specified guid was not found in this
   # mailbox.
@@ -121,7 +121,7 @@ class Mailbox
     scan
 
     unless db_message = fetch_db_message(guid)
-      warn "message not found in local db: #{guid}"
+      warning "message not found in local db: #{guid}"
       return nil
     end
 
@@ -135,7 +135,7 @@ class Mailbox
           data.attr['FLAGS'], Time.parse(data.attr['INTERNALDATE']))
     end
 
-    warn "message not found on server: #{guid}"
+    warning "message not found on server: #{guid}"
     return nil
   end
   alias [] fetch
@@ -242,16 +242,28 @@ class Mailbox
     return
   end
 
-  # Sets the IMAP flags for the message specified by _guid_, replacing any
-  # existing flags (except <code>:Recent</code>). _flags_ should be an array of
-  # symbols. Returns +true+ on success, +false+ on failure.
-  def set_flags(guid, flags)
+  # Sets the IMAP flags for the message specified by _guid_. _flags_ should be
+  # an array of symbols for standard flags, strings for custom flags.
+  #
+  # If _merge_ is +true+, the specified flags will be merged with the message's
+  # existing flags. Otherwise, all existing flags will be cleared and replaced
+  # with the specified flags.
+  #
+  # Note that the :Recent flag cannot be manually set or removed.
+  #
+  # Returns +true+ on success, +false+ on failure.
+  def set_flags(guid, flags, merge = false)
     raise ArgumentError, "flags must be an Array" unless flags.is_a?(Array)
 
-    db_message = fetch_db_message(guid)
-    return false if db_message.nil? || !imap_select
+    return false unless db_message = fetch_db_message(guid)
+
+    merged_flags    = merge ? (db_message.flags + flags).uniq : flags
+    supported_flags = get_supported_flags(merged_flags)
 
-    @imap.safely { @imap.conn.uid_store(db_message.uid, 'FLAGS.SILENT', flags) } unless @imap.options[:dry_run]
+    return true if db_message.flags == supported_flags
+
+    return false if !imap_select
+    @imap.safely { @imap.conn.uid_store(db_message.uid, 'FLAGS.SILENT', supported_flags) } unless @imap.options[:dry_run]
 
     true
   end
@@ -317,6 +329,24 @@ class Mailbox
     end
   end
 
+  # Returns only the flags from the specified _flags_ array that can be set in
+  # this mailbox. Emits a warning message for any unsupported flags.
+  def get_supported_flags(flags)
+    supported_flags = flags.dup
+
+    supported_flags.delete_if do |flag|
+      # The \Recent flag is read-only, so we shouldn't try to set it.
+      next true if flag == :Recent
+
+      unless @flags.include?(flag) || @perm_flags.include?(:*) || @perm_flags.include?(flag)
+        warning "flag not supported on destination: #{flag}"
+        true
+      end
+    end
+
+    supported_flags
+  end
+
   # Fetches the latest flags from the server for the specified range of message
   # UIDs.
   def fetch_flags(flag_range)
@@ -563,7 +593,7 @@ class Mailbox
           raise unless e.message == 'Some messages could not be FETCHed (Failure)'
 
           # Workaround for stupid Gmail shenanigans.
-          warn "Gmail error: '#{e.message}'; continuing anyway"
+          warning "Gmail error: '#{e.message}'; continuing anyway"
         end
 
         next data

data/lib/larch/logger.rb

@@ -4,12 +4,13 @@ class Logger
   attr_reader :level, :output
 
   LEVELS = {
-    :fatal  => 0,
-    :error  => 1,
-    :warn   => 2,
-    :info   => 3,
-    :debug  => 4,
-    :insane => 5
+    :fatal   => 0,
+    :error   => 1,
+    :warn    => 2,
+    :warning => 2,
+    :info    => 3,
+    :debug   => 4,
+    :insane  => 5
   }
 
   def initialize(level = :info, output = $stdout)

data/lib/larch/version.rb

@@ -1,9 +1,9 @@
 module Larch
   APP_NAME      = 'Larch'
-  APP_VERSION   = '1.1.0.dev.20100120'
+  APP_VERSION   = '1.1.0.dev.20100206'
   APP_AUTHOR    = 'Ryan Grove'
   APP_EMAIL     = 'ryan@wonko.com'
   APP_URL       = 'http://github.com/rgrove/larch/'
-  APP_COPYRIGHT = 'Copyright (c) 2009 Ryan Grove <ryan@wonko.com>. All ' <<
+  APP_COPYRIGHT = 'Copyright (c) 2010 Ryan Grove <ryan@wonko.com>. All ' <<
                   'rights reserved.'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: larch
 version: !ruby/object:Gem::Version 
-  version: 1.1.0.dev.20100120
+  version: 1.1.0.dev.20100206
 platform: ruby
 authors: 
 - Ryan Grove
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-20 00:00:00 -08:00
+date: 2010-02-06 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -104,6 +104,6 @@ rubyforge_project:
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
-summary: Larch syncs messages from one IMAP server to another. Awesomely.
+summary: Larch copies messages from one IMAP server to another. Awesomely.
 test_files: []