larch 1.1.0.dev.20091106 → 1.1.0.dev.20091123

data/HISTORY

@@ -17,8 +17,13 @@ Version 1.1.0 (git)
     mailboxes.
   * Added short versions of common command-line options.
   * The --fast-scan option has been removed.
+  * The --to-folder option can now be used in conjunction with --all or
+    --all-subscribed to copy messages from multiple source folders to a single
+    destination folder.
   * Fixed encoding issues when creating mailboxes and getting mailbox lists.
   * Fixed incorrect case-sensitive treatment of the 'INBOX' folder name.
+  * Fixed a bug in which Larch would try to copy flags that weren't supported on
+    the destination server.
 
 Version 1.0.2 (2009-08-05)
   * Fixed a bug that caused Larch to try to set the read-only \Recent flag on

data/bin/larch

@@ -21,11 +21,11 @@ Usage:
 Server Options:
 EOS
     opt :from,        "URI of the source IMAP server.", :short => '-f', :type => :string
-    opt :from_folder, "Source folder to copy from", :short => '-F', :default => Config::DEFAULT['from-folder']
+    opt :from_folder, "Source folder to copy from (default: INBOX)", :short => '-F', :default => Config::DEFAULT['from-folder'], :type => :string
     opt :from_pass,   "Source server password (default: prompt)", :short => '-p', :type => :string
     opt :from_user,   "Source server username (default: prompt)", :short => '-u', :type => :string
     opt :to,          "URI of the destination IMAP server.", :short => '-t', :type => :string
-    opt :to_folder,   "Destination folder to copy to", :short => '-T', :default => Config::DEFAULT['to-folder']
+    opt :to_folder,   "Destination folder to copy to (default: INBOX)", :short => '-T', :default => Config::DEFAULT['to-folder'], :type => :string
     opt :to_pass,     "Destination server password (default: prompt)", :short => '-P', :type => :string
     opt :to_user,     "Destination server username (default: prompt)", :short => '-U', :type => :string
 
@@ -46,16 +46,13 @@ EOS
     opt :verbosity,        "Output verbosity: debug, info, warn, error, or fatal", :short => '-V', :default => Config::DEFAULT['verbosity']
   end
 
-  # Load config.
-  config = Config.new(ARGV.shift || 'default', options[:config], options)
-  
   if options[:config_given]
     Trollop.die :config, ": file not found: #{options[:config]}" unless File.exist?(options[:config])
   end
 
-  # Validate config.
+  # Load config.
   begin
-    config.validate
+    config = Config.new(ARGV.shift || 'default', options[:config], options)
   rescue Config::Error => e
     abort "Config error: #{e}"
   end
@@ -65,13 +62,12 @@ EOS
   uri_to   = URI(config.to)
 
   # Use --from-folder and --to-folder unless folders were specified in the URIs.
-  uri_from.path = uri_from.path.empty? ? '/' + CGI.escape(config.from_folder.gsub(/^\//, '')) : uri_from.path
-  uri_to.path   = uri_to.path.empty?   ? '/' + CGI.escape(config.to_folder.gsub(/^\//, ''))   : uri_to.path
+  uri_from.path = uri_from.path.empty? ? '/' + CGI.escape(config.from_folder.gsub(/^\//, '')) : uri_from.path if config.from_folder
+  uri_to.path   = uri_to.path.empty?   ? '/' + CGI.escape(config.to_folder.gsub(/^\//, ''))   : uri_to.path if config.to_folder
 
   # --all and --all-subscribed options override folders
   if config.all || config.all_subscribed
     uri_from.path = ''
-    uri_to.path   = ''
   end
 
   # Usernames and passwords specified as arguments override those in the URIs

data/lib/larch/config.rb

@@ -12,7 +12,7 @@ class Config
     'exclude'          => [],
     'exclude-file'     => nil,
     'from'             => nil,
-    'from-folder'      => 'INBOX',
+    'from-folder'      => nil, # actually INBOX; see validate()
     'from-pass'        => nil,
     'from-user'        => nil,
     'max-retries'      => 3,
@@ -20,7 +20,7 @@ class Config
     'ssl-certs'        => nil,
     'ssl-verify'       => false,
     'to'               => nil,
-    'to-folder'        => 'INBOX',
+    'to-folder'        => nil, # actually INBOX; see validate()
     'to-pass'          => nil,
     'to-user'          => nil,
     'verbosity'        => 'info'
@@ -36,6 +36,7 @@ class Config
     end
 
     load_file(filename)
+    validate
   end
 
   def fetch(name)
@@ -64,6 +65,7 @@ class Config
     fetch(name)
   end
 
+  # Validates the config and resolves conflicting settings.
   def validate
     ['from', 'to'].each do |s|
       raise Error, "'#{s}' must be a valid IMAP URI (e.g. imap://example.com)" unless fetch(s) =~ IMAP::REGEX_URI
@@ -77,6 +79,22 @@ class Config
       raise Error, "exclude file not found: #{exclude_file}" unless File.file?(exclude_file)
       raise Error, "exclude file cannot be read: #{exclude_file}" unless File.readable?(exclude_file)
     end
+
+    if @cached['all'] || @cached['all-subscribed']
+      # A specific source folder wins over 'all' and 'all-subscribed'
+      if @cached['from-folder']
+        @cached['all']              = false
+        @cached['all-subscribed']   = false
+        @cached['to-folder']      ||= @cached['from-folder']
+
+      elsif @cached['all'] && @cached['all-subscribed']
+        # 'all' wins over 'all-subscribed'
+        @cached['all-subscribed'] = false
+      end
+    else
+      @cached['from-folder'] ||= 'INBOX'
+      @cached['to-folder']   ||= 'INBOX'
+    end
   end
 
   private

data/lib/larch/db/account.rb

@@ -1,6 +1,6 @@
 module Larch; module Database
 
-class Account < Sequel::Model
+class Account < Sequel::Model(:accounts)
   plugin :hook_class_methods
   one_to_many :mailboxes, :class => Larch::Database::Mailbox
 

data/lib/larch/db/mailbox.rb

@@ -1,6 +1,6 @@
 module Larch; module Database
 
-class Mailbox < Sequel::Model
+class Mailbox < Sequel::Model(:mailboxes)
   plugin :hook_class_methods
   one_to_many :messages, :class => Larch::Database::Message
 

data/lib/larch/db/message.rb

@@ -1,6 +1,6 @@
 module Larch; module Database
 
-class Message < Sequel::Model
+class Message < Sequel::Model(:messages)
 end
 
 end; end

data/lib/larch/imap/mailbox.rb

@@ -2,7 +2,7 @@ module Larch; class IMAP
 
 # Represents an IMAP mailbox.
 class Mailbox
-  attr_reader :attr, :db_mailbox, :delim, :imap, :name, :state, :subscribed
+  attr_reader :attr, :db_mailbox, :delim, :flags, :imap, :name, :perm_flags, :state, :subscribed
 
   # Maximum number of message headers to fetch with a single IMAP command.
   FETCH_BLOCK_SIZE = 1024
@@ -20,6 +20,8 @@ class Mailbox
     @name       = name
     @name_utf7  = Net::IMAP.encode_utf7(@name)
     @delim      = delim
+    @flags      = []
+    @perm_flags = []
     @subscribed = subscribed
     @attr       = attr.flatten
 
@@ -72,13 +74,20 @@ class Mailbox
         raise Larch::IMAP::Error, "mailbox cannot contain messages: #{@name}"
       end
 
-      debug "appending message: #{message.guid}"
-
-      # The \Recent flag is read-only, so we shouldn't try to set it at the
-      # destination.
       flags = message.flags.dup
-      flags.delete(:Recent)
 
+      # Don't set any flags that aren't supported on the destination mailbox.
+      flags.delete_if do |flag|
+        # The \Recent flag is read-only, so we shouldn't try to set it.
+        next true if flag == :Recent
+
+        unless @flags.include?(flag) || @perm_flags.include?(:*) || @perm_flags.include?(flag)
+          debug "flag not supported on destination: #{flag}"
+          true
+        end
+      end
+
+      debug "appending message: #{message.guid}"
       @imap.conn.append(@name_utf7, message.rfc822, flags, message.internaldate) unless @imap.options[:dry_run]
     end
 
@@ -90,7 +99,7 @@ class Mailbox
   # of each to the provided block.
   def each_guid # :yields: guid
     scan
-    @db_mailbox.messages.each {|db_message| yield db_message.guid }
+    @db_mailbox.messages_dataset.each {|db_message| yield db_message.guid }
   end
 
   # Iterates through mailboxes that are first-level children of this mailbox,
@@ -152,8 +161,8 @@ class Mailbox
   end
 
   # Same as fetch, but doesn't mark the message as seen.
-  def peek(message_id)
-    fetch(message_id, true)
+  def peek(guid)
+    fetch(guid, true)
   end
 
   # Resets the mailbox state.
@@ -211,17 +220,31 @@ class Mailbox
     if flag_range && flag_range.last - flag_range.first > 0
       info "fetching latest message flags..."
 
+      # Load the expected UIDs and their flags into a Hash for quicker lookups.
       expected_uids = {}
-      @db_mailbox.messages.each {|db_message| expected_uids[db_message.uid] = true }
+      @db_mailbox.messages_dataset.each do |db_message|
+        expected_uids[db_message.uid] = db_message.flags.split(',').map{|f| f.to_sym }
+      end
 
       imap_uid_fetch(flag_range, "(UID FLAGS)", 16384) do |fetch_data|
+        # Check the fields in the first response to ensure that everything we
+        # asked for is there.
+        check_response_fields(fetch_data.first, 'UID', 'FLAGS') unless fetch_data.empty?
+
         Larch.db.transaction do
           fetch_data.each do |data|
-            check_response_fields(data, 'UID', 'FLAGS')
-            expected_uids.delete(data.attr['UID'])
+            uid         = data.attr['UID']
+            flags       = data.attr['FLAGS']
+            local_flags = expected_uids[uid]
+
+            # If we haven't seen this message before, or if its flags have
+            # changed, update the database.
+            unless local_flags && local_flags == flags
+              @db_mailbox.messages_dataset.filter(:uid => uid).update(
+                  :flags => flags.map{|f| f.to_s }.join(','))
+            end
 
-            @db_mailbox.messages_dataset.filter(:uid => data.attr['UID']).
-                  update(:flags => data.attr['FLAGS'].map{|f| f.to_s }.join(','))
+            expected_uids.delete(uid)
           end
         end
       end
@@ -233,7 +256,7 @@ class Mailbox
         debug "removing #{expected_uids.length} deleted messages from the database..."
 
         Larch.db.transaction do
-          expected_uids.each do |uid|
+          expected_uids.each_key do |uid|
             @db_mailbox.messages_dataset.filter(:uid => uid).destroy
           end
         end
@@ -378,6 +401,7 @@ class Mailbox
 
         debug "examining mailbox"
         @imap.conn.examine(@name_utf7)
+        refresh_flags
 
         @mutex.synchronize { @state = :examined }
 
@@ -405,6 +429,7 @@ class Mailbox
 
         debug "selecting mailbox"
         @imap.conn.select(@name_utf7)
+        refresh_flags
 
         @mutex.synchronize { @state = :selected }
 
@@ -487,7 +512,18 @@ class Mailbox
     blocks.each do |block|
       data = @imap.safely do
         imap_examine
-        @imap.conn.uid_fetch(block, fields)
+
+        begin
+          data = @imap.conn.uid_fetch(block, fields)
+
+        rescue Net::IMAP::NoResponseError => e
+          raise unless e.message == 'Some messages could not be FETCHed (Failure)'
+
+          # Workaround for stupid Gmail shenanigans.
+          warn "Gmail error: '#{e.message}'; continuing anyway"
+        end
+
+        next data
       end
 
       yield data unless data.nil?
@@ -500,6 +536,15 @@ class Mailbox
     return str =~ REGEX_MESSAGE_ID ? $1 : nil
   end
 
+  # Refreshes the list of valid flags for this mailbox.
+  def refresh_flags
+    return unless @imap.conn.responses.has_key?('FLAGS') &&
+        @imap.conn.responses.has_key?('PERMANENTFLAGS')
+
+    @flags      = Array(@imap.conn.responses['FLAGS'].first)
+    @perm_flags = Array(@imap.conn.responses['PERMANENTFLAGS'].first)
+  end
+
 end
 
 end; end

data/lib/larch/imap.rb

@@ -54,9 +54,9 @@ class IMAP
 
     raise ArgumentError, "must provide a username and password" unless @uri.user && @uri.password
 
-    @conn       = nil
-    @mailboxes  = {}
-    @mutex      = Mutex.new
+    @conn      = nil
+    @mailboxes = {}
+    @mutex     = Mutex.new
 
     @db_account = Database::Account.find_or_create(
       :hostname => host,
@@ -173,6 +173,7 @@ class IMAP
            Errno::ENOTCONN,
            Errno::EPIPE,
            Errno::ETIMEDOUT,
+           IOError,
            Net::IMAP::ByeResponseError,
            OpenSSL::SSL::SSLError => e
 

data/lib/larch/version.rb

@@ -1,6 +1,6 @@
 module Larch
   APP_NAME      = 'Larch'
-  APP_VERSION   = '1.1.0.dev.20091106'
+  APP_VERSION   = '1.1.0.dev.20091123'
   APP_AUTHOR    = 'Ryan Grove'
   APP_EMAIL     = 'ryan@wonko.com'
   APP_URL       = 'http://github.com/rgrove/larch/'

data/lib/larch.rb

@@ -69,7 +69,12 @@ module Larch
         next if excluded?(mailbox_from.name)
         next if subscribed_only && !mailbox_from.subscribed?
 
-        mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim)
+        if imap_to.uri_mailbox
+          mailbox_to = imap_to.mailbox(imap_to.uri_mailbox)
+        else
+          mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim)
+        end
+
         mailbox_to.subscribe if mailbox_from.subscribed?
 
         copy_messages(mailbox_from, mailbox_to)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: larch
 version: !ruby/object:Gem::Version 
-  version: 1.1.0.dev.20091106
+  version: 1.1.0.dev.20091123
 platform: ruby
 authors: 
 - Ryan Grove
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-06 00:00:00 -08:00
+date: 2009-11-23 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -30,7 +30,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 3.3.0
+        version: 3.6.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: sqlite3-ruby