rgrove-larch 1.0.2 → 1.0.2.1

data/HISTORY

@@ -1,6 +1,14 @@
 Larch History
 ================================================================================
 
+Version 1.1.0 (git)
+  * Mailbox and message state information is now stored in a local SQLite
+    database, which allows Larch to resync and resume interrupted syncs much
+    more quickly without having to rescan all messages. As a result, SQLite 3 is
+    now a dependency.
+  * Progress information is now displayed regularly while scanning large
+    mailboxes.
+
 Version 1.0.2 (2009-08-05)
   * Fixed a bug that caused Larch to try to set the read-only \Recent flag on
     the destination server.

data/bin/larch

@@ -22,7 +22,7 @@ EOS
     opt :from, "URI of the source IMAP server.", :short => '-f', :type => :string, :required => true
     opt :to,   "URI of the destination IMAP server.", :short => '-t', :type => :string, :required => true
 
-    text "\nCopy Options:"
+    text "\nSync Options:"
     opt :all,              "Copy all folders recursively", :short => :none
     opt :all_subscribed,   "Copy all subscribed folders recursively", :short => :none
     opt :exclude,          "List of mailbox names/patterns that shouldn't be copied", :short => :none, :type => :strings, :multi => true
@@ -35,7 +35,8 @@ EOS
     opt :to_user,          "Destination server username (default: prompt)", :short => :none, :type => :string
 
     text "\nGeneral Options:"
-    opt :dry_run,          "Don't actually make any changes.", :short => '-n'
+    opt :database,         "Specify a non-default message database to use", :short => :none, :default => File.join('~', '.larch', 'larch.db')
+    opt :dry_run,          "Don't actually make any changes", :short => '-n'
     opt :fast_scan,        "Use a faster (but less accurate) method to scan mailboxes. This may result in messages being re-copied.", :short => :none
     opt :max_retries,      "Maximum number of times to retry after a recoverable error", :short => :none, :default => 3
     opt :no_create_folder, "Don't create destination folders that don't already exist", :short => :none
@@ -104,10 +105,10 @@ EOS
   uri_to.password   ||= CGI.escape(ask("Destination password (#{uri_to.host}): ") {|q| q.echo = false })
 
   # Go go go!
-  init(
-    options[:verbosity],
-    options[:exclude] ? options[:exclude].flatten : [],
-    options[:exclude_file]
+  init(options[:database],
+    :exclude      => options[:exclude] ? options[:exclude].flatten : [],
+    :exclude_file => options[:exclude_file],
+    :log_level    => options[:verbosity]
   )
 
   Net::IMAP.debug = true if @log.level == :insane

data/lib/larch/db/account.rb

@@ -0,0 +1,12 @@
+module Larch; module Database
+
+class Account < Sequel::Model
+  plugin :hook_class_methods
+  one_to_many :mailboxes, :class => Larch::Database::Mailbox
+
+  before_destroy do
+    Mailbox.filter(:account_id => id).destroy
+  end
+end
+
+end; end

data/lib/larch/db/mailbox.rb

@@ -0,0 +1,12 @@
+module Larch; module Database
+
+class Mailbox < Sequel::Model
+  plugin :hook_class_methods
+  one_to_many :messages, :class => Larch::Database::Message
+
+  before_destroy do
+    Message.filter(:mailbox_id => id).destroy
+  end
+end
+
+end; end

data/lib/larch/db/message.rb

@@ -0,0 +1,6 @@
+module Larch; module Database
+
+class Message < Sequel::Model
+end
+
+end; end

data/lib/larch/db/migrate/001_create_schema.rb

@@ -0,0 +1,42 @@
+class CreateSchema < Sequel::Migration
+  def down
+    drop_table :accounts, :mailboxes, :messages
+  end
+
+  def up
+    create_table :accounts do
+      primary_key :id
+      text :hostname, :null => false
+      text :username, :null => false
+
+      unique [:hostname, :username]
+    end
+
+    create_table :mailboxes do
+      primary_key :id
+      foreign_key :account_id, :table => :accounts
+      text :name, :null => false
+      text :delim, :null => false
+      text :attr, :null => false, :default => ''
+      integer :subscribed, :null => false, :default => 0
+      integer :uidvalidity
+      integer :uidnext
+
+      unique [:account_id, :name, :uidvalidity]
+    end
+
+    create_table :messages do
+      primary_key :id
+      foreign_key :mailbox_id, :table => :mailboxes
+      integer :uid, :null => false
+      text :guid, :null => false
+      text :message_id
+      integer :rfc822_size, :null => false
+      integer :internaldate, :null => false
+      text :flags, :null => false, :default => ''
+
+      index :guid
+      unique [:mailbox_id, :uid]
+    end
+  end
+end

data/lib/larch/imap/mailbox.rb

@@ -2,7 +2,10 @@ module Larch; class IMAP
 
 # Represents an IMAP mailbox.
 class Mailbox
-  attr_reader :attr, :delim, :imap, :name, :state
+  attr_reader :attr, :db_mailbox, :delim, :imap, :name, :state, :subscribed
+
+  # Maximum number of message headers to fetch with a single IMAP command.
+  FETCH_BLOCK_SIZE = 1024
 
   # Regex to capture a Message-Id header.
   REGEX_MESSAGE_ID = /message-id\s*:\s*(\S+)/i
@@ -15,12 +18,11 @@ class Mailbox
 
     @imap       = imap
     @name       = name
+    @name_utf7  = Net::IMAP.encode_utf7(@name)
     @delim      = delim
     @subscribed = subscribed
     @attr       = attr.flatten
 
-    @ids       = {}
-    @last_id   = 0
     @last_scan = nil
     @mutex     = Mutex.new
 
@@ -28,6 +30,23 @@ class Mailbox
     # open and read-only), or :selected (mailbox open and read-write).
     @state = :closed
 
+    # Create/update this mailbox in the database.
+    mb_data = {
+      :name       => @name,
+      :delim      => @delim,
+      :attr       => @attr.map{|a| a.to_s }.join(','),
+      :subscribed => @subscribed ? 1 : 0
+    }
+
+    @db_mailbox = imap.db_account.mailboxes_dataset.filter(:name => @name).first
+
+    if @db_mailbox
+      @db_mailbox.update(mb_data)
+    else
+      @db_mailbox = Database::Mailbox.create(mb_data)
+      imap.db_account.add_mailbox(@db_mailbox)
+    end
+
     # Create private convenience methods (debug, info, warn, etc.) to make
     # logging easier.
     Logger::LEVELS.each_key do |level|
@@ -46,68 +65,71 @@ class Mailbox
   # +false+ if the message already exists in the mailbox.
   def append(message)
     raise ArgumentError, "must provide a Larch::IMAP::Message object" unless message.is_a?(Larch::IMAP::Message)
-    return false if has_message?(message)
+    return false if has_guid?(message.guid)
 
     @imap.safely do
       unless imap_select(!!@imap.options[:create_mailbox])
         raise Larch::IMAP::Error, "mailbox cannot contain messages: #{@name}"
       end
 
-      debug "appending message: #{message.id}"
+      debug "appending message: #{message.guid}"
 
       # The \Recent flag is read-only, so we shouldn't try to set it at the
       # destination.
       flags = message.flags.dup
       flags.delete(:Recent)
 
-      @imap.conn.append(@name, message.rfc822, flags, message.internaldate) unless @imap.options[:dry_run]
+      @imap.conn.append(@name_utf7, message.rfc822, flags, message.internaldate) unless @imap.options[:dry_run]
     end
 
     true
   end
   alias << append
 
-  # Iterates through Larch message ids in this mailbox, yielding each one to the
-  # provided block.
-  def each
+  # Iterates through messages in this mailbox, yielding the Larch message guid
+  # of each to the provided block.
+  def each_guid
     scan
-    @ids.dup.each_key {|id| yield id }
+    @db_mailbox.messages.each {|db_message| yield db_message.guid }
   end
 
-  # Gets a Net::IMAP::Envelope for the specified message id.
-  def envelope(message_id)
+  # Returns a Larch::IMAP::Message struct representing the message with the
+  # specified Larch _guid_, or +nil+ if the specified guid was not found in this
+  # mailbox.
+  def fetch(guid, peek = false)
     scan
-    raise Larch::IMAP::MessageNotFoundError, "message not found: #{message_id}" unless uid = @ids[message_id]
 
-    debug "fetching envelope: #{message_id}"
-    imap_uid_fetch([uid], 'ENVELOPE').first.attr['ENVELOPE']
-  end
+    unless db_message = @db_mailbox.messages_dataset.filter(:guid => guid).first
+      warn "message not found in local db: #{guid}"
+      return nil
+    end
 
-  # Fetches a Larch::IMAP::Message struct representing the message with the
-  # specified Larch message id.
-  def fetch(message_id, peek = false)
-    scan
-    raise Larch::IMAP::MessageNotFoundError, "message not found: #{message_id}" unless uid = @ids[message_id]
+    debug "#{peek ? 'peeking at' : 'fetching'} message: #{guid}"
+
+    imap_uid_fetch([db_message.uid], [(peek ? 'BODY.PEEK[]' : 'BODY[]'), 'FLAGS', 'INTERNALDATE', 'ENVELOPE']) do |fetch_data|
+      data = fetch_data.first
+      check_response_fields(data, 'BODY[]', 'FLAGS', 'INTERNALDATE', 'ENVELOPE')
 
-    debug "#{peek ? 'peeking at' : 'fetching'} message: #{message_id}"
-    data = imap_uid_fetch([uid], [(peek ? 'BODY.PEEK[]' : 'BODY[]'), 'FLAGS', 'INTERNALDATE', 'ENVELOPE']).first
+      return Message.new(guid, data.attr['ENVELOPE'], data.attr['BODY[]'],
+          data.attr['FLAGS'], Time.parse(data.attr['INTERNALDATE']))
+    end
 
-    Message.new(message_id, data.attr['ENVELOPE'], data.attr['BODY[]'],
-        data.attr['FLAGS'], Time.parse(data.attr['INTERNALDATE']))
+    warn "message not found on server: #{guid}"
+    return nil
   end
   alias [] fetch
 
-  # Returns +true+ if a message with the specified Larch <em>message_id</em>
-  # exists in this mailbox, +false+ otherwise.
-  def has_message?(message_id)
+  # Returns +true+ if a message with the specified Larch guid exists in this
+  # mailbox, +false+ otherwise.
+  def has_guid?(guid)
     scan
-    @ids.has_key?(message_id)
+    @db_mailbox.messages_dataset.filter(:guid => guid).count > 0
   end
 
   # Gets the number of messages in this mailbox.
   def length
     scan
-    @ids.length
+    @db_mailbox.messages_dataset.count
   end
   alias size length
 
@@ -124,52 +146,149 @@ class Mailbox
   # Fetches message headers from this mailbox.
   def scan
     return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
+    first_scan = @last_scan.nil?
+    @mutex.synchronize { @last_scan = Time.now }
 
+    # Compare the mailbox's current status with its last known status.
     begin
-      return unless imap_examine
+      return unless status = imap_status('MESSAGES', 'UIDNEXT', 'UIDVALIDITY')
     rescue Error => e
       return if @imap.options[:create_mailbox]
       raise
     end
 
-    last_id = @imap.safely { @imap.conn.responses['EXISTS'].last }
-    @mutex.synchronize { @last_scan = Time.now }
-    return if last_id == @last_id
+    flag_range = nil
+    full_range = nil
 
-    range = (@last_id + 1)..last_id
-    @mutex.synchronize { @last_id = last_id }
+    if @db_mailbox.uidvalidity && @db_mailbox.uidnext &&
+        status['UIDVALIDITY'] == @db_mailbox.uidvalidity
 
-    info "fetching message headers #{range}" <<
-        (@imap.options[:fast_scan] ? ' (fast scan)' : '')
+      # The UIDVALIDITY is the same as what we saw last time we scanned at this
+      # mailbox, which means that all the existing messages in the database are
+      # still valid. We only need to request headers for new messages.
+      #
+      # If this is the first scan of this mailbox during this Larch session,
+      # then we'll also update the flags of all messages in the mailbox.
+
+      flag_range = 1...@db_mailbox.uidnext if first_scan
+      full_range = @db_mailbox.uidnext...status['UIDNEXT']
 
-    fields = if @imap.options[:fast_scan]
-      ['UID', 'RFC822.SIZE', 'INTERNALDATE']
     else
-      "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)] RFC822.SIZE INTERNALDATE)"
+
+      # The UIDVALIDITY has changed or this is the first time we've scanned this
+      # mailbox (ever). Either way, all existing messages in the database are no
+      # longer valid, so we have to throw them out and re-request everything.
+
+      @db_mailbox.remove_all_messages
+      full_range = 1...status['UIDNEXT']
     end
 
-    imap_fetch(range.begin..-1, fields).each do |data|
-      id = create_id(data)
+    @db_mailbox.update(:uidvalidity => status['UIDVALIDITY'])
+
+    return unless flag_range || full_range.last - full_range.first > 0
+
+    # Open the mailbox for read-only access.
+    return unless imap_examine
+
+    if flag_range && flag_range.last - flag_range.first > 0
+      info "fetching latest message flags..."
+
+      expected_uids = {}
+      @db_mailbox.messages.each {|db_message| expected_uids[db_message.uid] = true }
 
-      unless uid = data.attr['UID']
-        error "UID not in IMAP response for message: #{id}"
-        next
+      imap_uid_fetch(flag_range, "(UID FLAGS)", 16384) do |fetch_data|
+        Larch.db.transaction do
+          fetch_data.each do |data|
+            check_response_fields(data, 'UID', 'FLAGS')
+            expected_uids.delete(data.attr['UID'])
+
+            @db_mailbox.messages_dataset.filter(:uid => data.attr['UID']).
+                  update(:flags => data.attr['FLAGS'].map{|f| f.to_s }.join(','))
+          end
+        end
       end
 
-      if Larch.log.level == :debug && @ids.has_key?(id)
-        envelope = imap_uid_fetch([uid], 'ENVELOPE').first.attr['ENVELOPE']
-        debug "duplicate message? #{id} (Subject: #{envelope.subject})"
+      # Any UIDs that are in the database but weren't in the response have been
+      # deleted from the server, so we need to delete them from the database as
+      # well.
+      unless expected_uids.empty?
+        debug "removing #{expected_uids.length} deleted messages from the database..."
+
+        Larch.db.transaction do
+          expected_uids.each do |uid|
+            @db_mailbox.messages_dataset.filter(:uid => uid).destroy
+          end
+        end
       end
+    end
+
+    if full_range && full_range.last - full_range.first > 0
+      start    = @db_mailbox.messages_dataset.count + 1
+      total    = status['MESSAGES']
+      fetched  = 0
+      progress = 0
+
+      show_progress = total - start > FETCH_BLOCK_SIZE * 4
+
+      info "fetching message headers #{start} through #{total}..."
+
+      begin
+        last_good_uid = nil
+
+        imap_uid_fetch(full_range, "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)] RFC822.SIZE INTERNALDATE FLAGS)") do |fetch_data|
+          check_response_fields(fetch_data, 'UID', 'RFC822.SIZE', 'INTERNALDATE', 'FLAGS')
+
+          Larch.db.transaction do
+            fetch_data.each do |data|
+              uid = data.attr['UID']
+
+              message_data = {
+                :guid         => create_guid(data),
+                :uid          => uid,
+                :message_id   => parse_message_id(data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]']),
+                :rfc822_size  => data.attr['RFC822.SIZE'].to_i,
+                :internaldate => Time.parse(data.attr['INTERNALDATE']).to_i,
+                :flags        => data.attr['FLAGS'].map{|f| f.to_s }.join(',')
+              }
+
+              @db_mailbox.add_message(Database::Message.create(message_data))
+              last_good_uid = uid
+            end
+
+            @db_mailbox.update(:uidnext => last_good_uid + 1)
+          end
+
+          if show_progress
+            fetched       += fetch_data.length
+            last_progress  = progress
+            progress       = ((100 / (total - start).to_f) * fetched).round
+
+            info "#{progress}% complete" if progress > last_progress
+          end
+        end
 
-      @mutex.synchronize { @ids[id] = uid }
+      rescue => e
+        # Set this mailbox's uidnext value to the last known good UID that was
+        # stored in the database, plus 1. This will allow Larch to resume where
+        # the error occurred on the next attempt rather than having to start over.
+        @db_mailbox.update(:uidnext => last_good_uid + 1) if last_good_uid
+        raise
+      end
     end
+
+    @db_mailbox.update(:uidnext => status['UIDNEXT'])
+    return
   end
 
   # Subscribes to this mailbox.
   def subscribe(force = false)
-    return if subscribed? && !force
-    @imap.safely { @imap.conn.subscribe(@name) } unless @imap.options[:dry_run]
+    return false if subscribed? && !force
+
+    @imap.safely { @imap.conn.subscribe(@name_utf7) } unless @imap.options[:dry_run]
     @mutex.synchronize { @subscribed = true }
+    @db_mailbox.update(:subscribed => 1)
+
+    true
   end
 
   # Returns +true+ if this mailbox is subscribed, +false+ otherwise.
@@ -179,35 +298,55 @@ class Mailbox
 
   # Unsubscribes from this mailbox.
   def unsubscribe(force = false)
-    return unless subscribed? || force
-    @imap.safely { @imap.conn.unsubscribe(@name) } unless @imap.options[:dry_run]
+    return false unless subscribed? || force
+
+    @imap.safely { @imap.conn.unsubscribe(@name_utf7) } unless @imap.options[:dry_run]
     @mutex.synchronize { @subscribed = false }
+    @db_mailbox.update(:subscribed => 0)
+
+    true
   end
 
   private
 
-  # Creates an id suitable for uniquely identifying a specific message across
-  # servers (we hope).
+  # Checks the specified Net::IMAP::FetchData object and raises a
+  # Larch::IMAP::Error unless it contains all the specified _fields_.
+  #
+  # _data_ can be a single object or an Array of objects; if it's an Array, then
+  # only the first object in the Array will be checked.
+  def check_response_fields(data, *fields)
+    check_data = data.is_a?(Array) ? data.first : data
+
+    fields.each do |f|
+      raise Error, "required data not in IMAP response: #{f}" unless check_data.attr.has_key?(f)
+    end
+
+    true
+  end
+
+  # Creates a globally unique id suitable for identifying a specific message
+  # on any mail server (we hope) based on the given IMAP FETCH _data_.
   #
   # If the given message data includes a valid Message-Id header, then that will
   # be used to generate an MD5 hash. Otherwise, the hash will be generated based
   # on the message's RFC822.SIZE and INTERNALDATE.
-  def create_id(data)
-    ['RFC822.SIZE', 'INTERNALDATE'].each do |a|
-      raise Error, "required data not in IMAP response: #{a}" unless data.attr[a]
-    end
-
-    if data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]'] =~ REGEX_MESSAGE_ID
-      Digest::MD5.hexdigest($1)
+  def create_guid(data)
+    if message_id = parse_message_id(data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]'])
+      Digest::MD5.hexdigest(message_id)
     else
+      check_response_fields(data, 'RFC822.SIZE', 'INTERNALDATE')
+
       Digest::MD5.hexdigest(sprintf('%d%d', data.attr['RFC822.SIZE'],
           Time.parse(data.attr['INTERNALDATE']).to_i))
     end
   end
 
-  # Examines the mailbox. If _force_ is true, the mailbox will be examined even
+  # Examines this mailbox. If _force_ is true, the mailbox will be examined even
   # if it is already selected (which isn't necessary unless you want to ensure
   # that it's in a read-only state).
+  #
+  # Returns +false+ if this mailbox cannot be examined, which may be the case if
+  # the \Noselect attribute is set.
   def imap_examine(force = false)
     return false if @attr.include?(:Noselect)
     return true if @state == :examined || (!force && @state == :selected)
@@ -217,7 +356,7 @@ class Mailbox
         @mutex.synchronize { @state = :closed }
 
         debug "examining mailbox"
-        @imap.conn.examine(@name)
+        @imap.conn.examine(@name_utf7)
 
         @mutex.synchronize { @state = :examined }
 
@@ -229,18 +368,12 @@ class Mailbox
     return true
   end
 
-  # Fetches the specified _fields_ for the specified _set_ of message sequence
-  # ids (either a Range or an Array of ids).
-  def imap_fetch(set, fields)
-    @imap.safely do
-      imap_examine
-      @imap.conn.fetch(set, fields)
-    end
-  end
-
   # Selects the mailbox if it is not already selected. If the mailbox does not
   # exist and _create_ is +true+, it will be created. Otherwise, a
   # Larch::IMAP::Error will be raised.
+  #
+  # Returns +false+ if this mailbox cannot be selected, which may be the case if
+  # the \Noselect attribute is set.
   def imap_select(create = false)
     return false if @attr.include?(:Noselect)
     return true if @state == :selected
@@ -250,7 +383,7 @@ class Mailbox
         @mutex.synchronize { @state = :closed }
 
         debug "selecting mailbox"
-        @imap.conn.select(@name)
+        @imap.conn.select(@name_utf7)
 
         @mutex.synchronize { @state = :selected }
 
@@ -260,7 +393,7 @@ class Mailbox
         info "creating mailbox: #{@name}"
 
         begin
-          @imap.conn.create(@name) unless @imap.options[:dry_run]
+          @imap.conn.create(@name_utf7) unless @imap.options[:dry_run]
           retry
         rescue => e
           raise Error, "unable to create mailbox: #{e.message}"
@@ -271,15 +404,81 @@ class Mailbox
     return true
   end
 
-  # Fetches the specified _fields_ for the specified _set_ of UIDs (either a
-  # Range or an Array of UIDs).
-  def imap_uid_fetch(set, fields)
+  # Sends an IMAP STATUS command and returns the status of the requested
+  # attributes. Supported attributes include:
+  #
+  #   - MESSAGES
+  #   - RECENT
+  #   - UIDNEXT
+  #   - UIDVALIDITY
+  #   - UNSEEN
+  def imap_status(*attr)
     @imap.safely do
-      imap_examine
-      @imap.conn.uid_fetch(set, fields)
+      begin
+        debug "getting mailbox status"
+        @imap.conn.status(@name_utf7, attr)
+      rescue Net::IMAP::NoResponseError => e
+        raise Error, "unable to get status of mailbox: #{e.message}"
+      end
+    end
+  end
+
+  # Fetches the specified _fields_ for the specified _set_ of UIDs, which can be
+  # a number, Range, or Array of UIDs.
+  #
+  # If _set_ is a number, an Array containing a single Net::IMAP::FetchData
+  # object will be yielded to the given block.
+  #
+  # If _set_ is a Range or Array of UIDs, Arrays of up to <i>block_size</i>
+  # Net::IMAP::FetchData objects will be yielded until all requested messages
+  # have been fetched.
+  #
+  # However, if _set_ is a Range with an end value of -1, a single Array
+  # containing all requested messages will be yielded, since it's impossible to
+  # divide an infinite range into finite blocks.
+  def imap_uid_fetch(set, fields, block_size = FETCH_BLOCK_SIZE, &block) # :yields: fetch_data
+    if set.is_a?(Numeric) || (set.is_a?(Range) && set.last < 0)
+      data = @imap.safely do
+        imap_examine
+        @imap.conn.uid_fetch(set, fields)
+      end
+
+      yield data unless data.nil?
+    end
+
+    blocks = []
+    pos    = 0
+
+    if set.is_a?(Array)
+      while pos < set.length
+        blocks += set[pos, block_size]
+        pos    += block_size
+      end
+
+    elsif set.is_a?(Range)
+      pos = set.first - 1
+
+      while pos < set.last
+        blocks << ((pos + 1)..[set.last, pos += block_size].min)
+      end
+    end
+
+    blocks.each do |block|
+      data = @imap.safely do
+        imap_examine
+        @imap.conn.uid_fetch(block, fields)
+      end
+
+      yield data unless data.nil?
     end
   end
 
+  # Parses a Message-Id header out of _str_ and returns it, or +nil+ if _str_
+  # doesn't contain a valid Message-Id header.
+  def parse_message_id(str)
+    return str =~ REGEX_MESSAGE_ID ? $1 : nil
+  end
+
 end
 
 end; end

data/lib/larch/imap.rb

@@ -6,19 +6,19 @@ module Larch
 # required reading if you're doing anything with IMAP in Ruby:
 # http://sup.rubyforge.org
 class IMAP
-  attr_reader :conn, :options
+  attr_reader :conn, :db_account, :options
 
   # URI format validation regex.
   REGEX_URI = URI.regexp(['imap', 'imaps'])
 
   # Larch::IMAP::Message represents a transferable IMAP message which can be
   # passed between Larch::IMAP instances.
-  Message = Struct.new(:id, :envelope, :rfc822, :flags, :internaldate)
+  Message = Struct.new(:guid, :envelope, :rfc822, :flags, :internaldate)
 
   # Initializes a new Larch::IMAP instance that will connect to the specified
   # IMAP URI.
   #
-  # In addition to the URI, the following options may also be specified:
+  # In addition to the URI, the following options may be specified:
   #
   # [:create_mailbox]
   #   If +true+, mailboxes that don't already exist will be created if
@@ -62,9 +62,14 @@ class IMAP
 
     raise ArgumentError, "must provide a username and password" unless @uri.user && @uri.password
 
-    @conn      = nil
-    @mailboxes = {}
-    @mutex     = Mutex.new
+    @conn       = nil
+    @mailboxes  = {}
+    @mutex      = Mutex.new
+
+    @db_account = Database::Account.find_or_create(
+      :hostname => host,
+      :username => username
+    )
 
     # Create private convenience methods (debug, info, warn, etc.) to make
     # logging easier.
@@ -170,7 +175,8 @@ class IMAP
     begin
       yield
 
-    rescue Errno::ECONNRESET,
+    rescue Errno::ECONNABORTED,
+           Errno::ECONNRESET,
            Errno::ENOTCONN,
            Errno::EPIPE,
            Errno::ETIMEDOUT,
@@ -332,6 +338,11 @@ class IMAP
             subscribed.any?{|s| s.name == mb.name}, mb.attr)
       end
     end
+
+    # Remove mailboxes that no longer exist from the database.
+    @db_account.mailboxes.each do |db_mailbox|
+      db_mailbox.destroy unless @mailboxes.has_key?(db_mailbox.name)
+    end
   end
 
 end

data/lib/larch/version.rb

@@ -1,6 +1,6 @@
 module Larch
   APP_NAME      = 'Larch'
-  APP_VERSION   = '1.0.2'
+  APP_VERSION   = '1.0.2.1'
   APP_AUTHOR    = 'Ryan Grove'
   APP_EMAIL     = 'ryan@wonko.com'
   APP_URL       = 'http://github.com/rgrove/larch/'

data/lib/larch.rb

@@ -4,10 +4,14 @@ $:.uniq!
 
 require 'cgi'
 require 'digest/md5'
+require 'fileutils'
 require 'net/imap'
 require 'time'
 require 'uri'
 
+require 'sequel'
+require 'sequel/extensions/migration'
+
 require 'larch/errors'
 require 'larch/imap'
 require 'larch/imap/mailbox'
@@ -17,16 +21,23 @@ require 'larch/version'
 module Larch
 
   class << self
-    attr_reader :log, :exclude
+    attr_reader :db, :log, :exclude
 
     EXCLUDE_COMMENT = /#.*$/
     EXCLUDE_REGEX   = /^\s*\/(.*)\/\s*/
     GLOB_PATTERNS   = {'*' => '.*', '?' => '.'}
+    LIB_DIR         = File.join(File.dirname(File.expand_path(__FILE__)), 'larch')
+
+    def init(database, config = {})
+      @config = {
+        :exclude   => [],
+        :log_level => :info
+      }.merge(config)
 
-    def init(log_level = :info, exclude = [], exclude_file = nil)
-      @log = Logger.new(log_level)
+      @log = Logger.new(@config[:log_level])
+      @db  = open_db(database)
 
-      @exclude = exclude.map do |e|
+      @exclude = @config[:exclude].map do |e|
         if e =~ EXCLUDE_REGEX
           Regexp.new($1, Regexp::IGNORECASE)
         else
@@ -34,7 +45,7 @@ module Larch
         end
       end
 
-      load_exclude_file(exclude_file) if exclude_file
+      load_exclude_file(@config[:exclude_file]) if @config[:exclude_file]
 
       # Stats
       @copied = 0
@@ -59,7 +70,7 @@ module Larch
         mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim)
         mailbox_to.subscribe if mailbox_from.subscribed?
 
-        copy_messages(imap_from, mailbox_from, imap_to, mailbox_to)
+        copy_messages(imap_from, mailbox_from.name, imap_to, mailbox_to.name)
       end
 
     rescue => e
@@ -79,13 +90,12 @@ module Larch
       @failed = 0
       @total  = 0
 
-      from_name = imap_from.uri_mailbox || 'INBOX'
-      to_name   = imap_to.uri_mailbox || 'INBOX'
+      from_mb_name = imap_from.uri_mailbox || 'INBOX'
+      to_mb_name   = imap_to.uri_mailbox || 'INBOX'
 
-      return if excluded?(from_name) || excluded?(to_name)
+      return if excluded?(from_mb_name) || excluded?(to_mb_name)
 
-      copy_messages(imap_from, imap_from.mailbox(from_name), imap_to,
-          imap_to.mailbox(to_name))
+      copy_messages(imap_from, from_mb_name, imap_to, to_mb_name)
 
       imap_from.disconnect
       imap_to.disconnect
@@ -97,32 +107,70 @@ module Larch
       summary
     end
 
+    # Opens a connection to the Larch message database, creating it if
+    # necessary.
+    def open_db(database)
+      filename  = File.expand_path(database)
+      directory = File.dirname(filename)
+
+      unless File.exist?(directory)
+        FileUtils.mkdir_p(directory)
+        File.chmod(0700, directory)
+      end
+
+      begin
+        db = Sequel.connect("sqlite://#{filename}")
+        db.test_connection
+      rescue => e
+        @log.fatal "unable to open message database: #{e}"
+        abort
+      end
+
+      # Ensure that the database schema is up to date.
+      migration_dir = File.join(LIB_DIR, 'db', 'migrate')
+
+      unless Sequel::Migrator.get_current_migration_version(db) ==
+          Sequel::Migrator.latest_migration_version(migration_dir)
+        begin
+          Sequel::Migrator.apply(db, migration_dir)
+        rescue => e
+          @log.fatal "unable to migrate message database: #{e}"
+          abort
+        end
+      end
+
+      require 'larch/db/message'
+      require 'larch/db/mailbox'
+      require 'larch/db/account'
+
+      db
+    end
+
     def summary
       @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@total - @copied - @failed} untouched out of #{@total} total"
     end
 
     private
 
-    def copy_messages(imap_from, mailbox_from, imap_to, mailbox_to)
+    def copy_messages(imap_from, mb_name_from, imap_to, mb_name_to)
       raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
-      raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(IMAP::Mailbox)
       raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
-      raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(IMAP::Mailbox)
 
-      return if excluded?(mailbox_from.name) || excluded?(mailbox_to.name)
+      return if excluded?(mb_name_from) || excluded?(mb_name_to)
 
-      @log.info "copying messages from #{imap_from.host}/#{mailbox_from.name} to #{imap_to.host}/#{mailbox_to.name}"
+      @log.info "copying messages from #{imap_from.host}/#{mb_name_from} to #{imap_to.host}/#{mb_name_to}"
 
-      imap_from.connect
-      imap_to.connect
+      mailbox_from = imap_from.mailbox(mb_name_from)
 
       @total += mailbox_from.length
 
-      mailbox_from.each do |id|
-        next if mailbox_to.has_message?(id)
+      mailbox_to = imap_to.mailbox(mb_name_to)
+
+      mailbox_from.each_guid do |guid|
+        next if mailbox_to.has_guid?(guid)
 
         begin
-          msg = mailbox_from.peek(id)
+          next unless msg = mailbox_from.peek(guid)
 
           if msg.envelope.from
             env_from = msg.envelope.from.first
@@ -137,7 +185,6 @@ module Larch
           @copied += 1
 
         rescue Larch::IMAP::Error => e
-          # TODO: Keep failed message envelopes in a buffer for later output?
           @failed += 1
           @log.error e.message
           next

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rgrove-larch
 version: !ruby/object:Gem::Version 
-  version: 1.0.2
+  version: 1.0.2.1
 platform: ruby
 authors: 
 - Ryan Grove
@@ -9,8 +9,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-05-16 00:00:00 -07:00
-default_executable: 
+date: 2009-08-17 00:00:00 -07:00
+default_executable: larch
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: highline
@@ -22,6 +22,26 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.5.0
     version: 
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 3.3.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: sqlite3-ruby
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.2.5
+    version: 
 - !ruby/object:Gem::Dependency 
   name: trollop
   type: :runtime
@@ -45,12 +65,16 @@ files:
 - LICENSE
 - README.rdoc
 - bin/larch
-- lib/larch.rb
+- lib/larch/db/account.rb
+- lib/larch/db/mailbox.rb
+- lib/larch/db/message.rb
+- lib/larch/db/migrate/001_create_schema.rb
 - lib/larch/errors.rb
-- lib/larch/imap.rb
 - lib/larch/imap/mailbox.rb
+- lib/larch/imap.rb
 - lib/larch/logger.rb
 - lib/larch/version.rb
+- lib/larch.rb
 has_rdoc: false
 homepage: http://github.com/rgrove/larch/
 licenses: 
@@ -76,7 +100,7 @@ requirements: []
 rubyforge_project: 
 rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: Larch syncs messages from one IMAP server to another. Awesomely.
 test_files: []