sup 0.0.6 → 0.0.7

data/lib/sup/imap.rb

@@ -25,14 +25,16 @@ require 'time'
 ## slow for large mailboxes, and we'll just have to hope that there
 ## are no collisions. ho ho! a perfectly reasonable solution!
 
-## fuck you, imap committee. you managed to design something as shitty
-## as mbox but goddamn THIRTY YEARS LATER.
-
+## fuck you, imap committee. you managed to design something nearly as
+## shitty as mbox but goddamn THIRTY YEARS LATER.
 module Redwood
 
 class IMAP < Source
   SCAN_INTERVAL = 60 # seconds
 
+  ## upon these errors we'll try to rereconnect a few times
+  RECOVERABLE_ERRORS = [ Errno::EPIPE, Errno::ETIMEDOUT ]
+
   attr_reader_cloned :labels
   attr_accessor :username, :password
 
@@ -62,6 +64,7 @@ class IMAP < Source
     x.nil? || x.empty? ? 'INBOX' : x
   end
   def ssl?; @parsed_uri.scheme == 'imaps' end
+  def == o; o.is_a?(IMAP) && o.uri == self.uri && o.username == self.username; end
 
   def load_header id
     MBox::read_header StringIO.new(raw_header(id))
@@ -72,42 +75,87 @@ class IMAP < Source
   end
 
   def raw_header id
-    @mutex.synchronize do
-      connect
-      header, flags = get_imap_fields id, 'RFC822.HEADER', 'FLAGS'
-      header = "Status: RO\n" + header if flags.include? :Seen # fake an mbox-style read header
-      header.gsub(/\r\n/, "\n")
-    end
+    unsynchronized_scan_mailbox
+    header, flags = get_imap_fields id, 'RFC822.HEADER', 'FLAGS'
+    header = header + "Status: RO\n" if flags.include? :Seen # fake an mbox-style read header # TODO: improve source-marked-as-read reporting system
+    header.gsub(/\r\n/, "\n")
   end
+  synchronized :raw_header
 
   def raw_full_message id
-    @mutex.synchronize do
-      connect
-      get_imap_fields(id, 'RFC822').first.gsub(/\r\n/, "\n")
-    end
+    unsynchronized_scan_mailbox
+    get_imap_fields(id, 'RFC822').first.gsub(/\r\n/, "\n")
   end
+  synchronized :raw_full_message
 
   def connect
-    return false if broken?
-    return true if @imap
+    return if broken? || @imap
+    safely { } # do nothing!
+  end
+  synchronized :connect
 
-    say "Connecting to IMAP server #{host}:#{port}..."
+  def scan_mailbox
+    return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
+    last_id = safely do
+      @imap.examine mailbox
+      @imap.responses["EXISTS"].last
+    end
+    @last_scan = Time.now
+
+    return if last_id == @ids.length
+
+    Redwood::log "fetching IMAP headers #{(@ids.length + 1) .. last_id}"
+    values = safely { @imap.fetch((@ids.length + 1) .. last_id, ['RFC822.SIZE', 'INTERNALDATE']) }
+    values.each do |v|
+      id = make_id v
+      @ids << id
+      @imap_ids[id] = v.seqno
+    end
+  end
+  synchronized :scan_mailbox
+
+  def each
+    ids = 
+      @mutex.synchronize do
+        unsynchronized_scan_mailbox
+        @ids
+      end
 
-    ## ok, this is FUCKING ANNOYING.
-    ##
-    ## what imap.rb likes to do is, if an exception occurs, catch it
-    ## and re-raise it on the calling thread. seems reasonable. but
-    ## what that REALLY means is that the only way to reasonably
-    ## initialize imap is in its own thread, because otherwise, you
-    ## will never be able to catch the exception it raises on the
-    ## calling thread, and the backtrace will not make any sense at
-    ## all, and you will waste HOURS of your life on this fucking
-    ## problem.
-    ##
-    ## FUCK!!!!!!!!!
+    start = ids.index(cur_offset || start_offset) or die_from "Unknown message id #{cur_offset || start_offset}.", :suggest_rebuild => true # couldn't find the most recent email
 
+    start.upto(ids.length - 1) do |i|         
+      id = ids[i]
+      self.cur_offset = id
+      yield id, labels
+    end
+  end
+
+  def start_offset
+    unsynchronized_scan_mailbox
+    @ids.first
+  end
+  synchronized :start_offset
+
+  def end_offset
+    unsynchronized_scan_mailbox
+    @ids.last
+  end
+  synchronized :end_offset
+
+  def pct_done; 100.0 * (@ids.index(cur_offset) || 0).to_f / (@ids.length - 1).to_f; end
+
+private
+
+  def unsafe_connect
+    say "Connecting to IMAP server #{host}:#{port}..."
+
+    ## apparently imap.rb does a lot of threaded stuff internally and
+    ## if an exception occurs, it will catch it and re-raise it on the
+    ## calling thread. but i can't seem to catch that exception, so
+    ## i've resorted to initializing it in its own thread. surely
+    ## there's a better way.
     exception = nil
-    Redwood::reporting_thread do
+    ::Thread.new do
       begin
         #raise Net::IMAP::ByeResponseError, "simulated imap failure"
         @imap = Net::IMAP.new host, port, ssl?
@@ -115,7 +163,7 @@ class IMAP < Source
 
         ## although RFC1730 claims that "If an AUTHENTICATE command
         ## fails with a NO response, the client may try another", in
-        ## practice it seems like they will also send BAD responses.
+        ## practice it seems like they can also send a BAD response.
         begin
           @imap.authenticate 'CRAM-MD5', @username, @password
         rescue Net::IMAP::BadResponseError, Net::IMAP::NoResponseError => e
@@ -127,51 +175,17 @@ class IMAP < Source
             @imap.login @username, @password
           end
         end
-        scan_mailbox
         say "Successfully connected to #{@parsed_uri}."
-      rescue SocketError, Net::IMAP::Error, SourceError => e
+      rescue Exception => e
         exception = e
       ensure
         shutup
       end
     end.join
 
-    die_from exception, :while => "connecting" if exception
-  end
-
-  def each
-    @mutex.synchronize { connect or raise SourceError, broken_msg }
-
-    start = @ids.index(cur_offset || start_offset) or die_from "Unknown message id #{cur_offset || start_offset}.", :suggest_rebuild => true # couldn't find the most recent email
-
-    start.upto(@ids.length - 1) do |i|         
-      id = @ids[i]
-      self.cur_offset = id
-      yield id, labels
-    end
-  end
-
-  def start_offset
-    @mutex.synchronize { connect }
-    @ids.first
+    raise exception if exception
   end
 
-  def end_offset
-    @mutex.synchronize do
-      begin
-        connect
-        scan_mailbox
-      rescue SocketError, Net::IMAP::Error => e
-        die_from e, :while => "scanning mailbox"
-      end
-    end
-    @ids.last
-  end
-
-  def pct_done; 100.0 * (@ids.index(cur_offset) || 0).to_f / (@ids.length - 1).to_f; end
-
-private
-
   def say s
     @say_id = BufferManager.say s, @say_id if BufferManager.instantiated?
     Redwood::log s
@@ -182,22 +196,6 @@ private
     @say_id = nil
   end
 
-  def scan_mailbox
-    return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
-
-    @imap.examine mailbox
-    last_id = @imap.responses["EXISTS"].last
-    @last_scan = Time.now
-    return if last_id == @ids.length
-    Redwood::log "fetching IMAP headers #{(@ids.length + 1) .. last_id}"
-    values = @imap.fetch((@ids.length + 1) .. last_id, ['RFC822.SIZE', 'INTERNALDATE'])
-    values.each do |v|
-      id = make_id v
-      @ids << id
-      @imap_ids[id] = v.seqno
-    end
-  end
-
   def die_from e, opts={}
     @imap = nil
 
@@ -225,26 +223,38 @@ private
   end
 
   def get_imap_fields id, *fields
-    retries = 0
-    f = nil
+    raise SourceError, broken_msg if broken?
     imap_id = @imap_ids[id] or die_from "Unknown message id #{id}.", :suggest_rebuild => true
+
+    retried = false
+    results = safely { @imap.fetch imap_id, (fields + ['RFC822.SIZE', 'INTERNALDATE']).uniq }.first
+    got_id = make_id results
+    die_from "IMAP message mismatch: requested #{id}, got #{got_id}.", :suggest_rebuild => true unless got_id == id
+
+    fields.map { |f| results.attr[f] }
+  end
+
+  ## execute a block, connected if unconnected, re-connected up to 3
+  ## times if a recoverable error occurs, and properly dying if an
+  ## unrecoverable error occurs.
+  def safely
+    retries = 0
     begin
-      f = @imap.fetch imap_id, (fields + ['RFC822.SIZE', 'INTERNALDATE']).uniq
-      got_id = make_id f[0]
-      die_from "IMAP message mismatch: requested #{id}, got #{got_id}.", :suggest_rebuild => true unless id == got_id
-    rescue SocketError, Net::IMAP::Error => e
-      die_from e, :while => "communicating with IMAP server"
-    rescue Errno::EPIPE
-      if (retries += 1) <= 3
-        @imap = nil
-        connect
-        retry
+      begin
+        unsafe_connect unless @imap
+        yield
+      rescue *RECOVERABLE_ERRORS
+        if (retries += 1) <= 3
+          @imap = nil
+          retry
+        end
+        raise
       end
+    rescue Net, SocketError, Net::IMAP::Error, SystemCallError => e
+      die_from e, :while => "communicating with IMAP server"
     end
-    die_from "Null IMAP field '#{field}' for message with id #{id} imap id #{imap_id}." if f.nil?
-
-    fields.map { |field| f[0].attr[field] }
   end
+
 end
 
 Redwood::register_yaml(IMAP, %w(uri username password cur_offset usual archived id))

data/lib/sup/index.rb

@@ -3,7 +3,6 @@
 require 'thread'
 require 'fileutils'
 require 'ferret'
-#require_gem 'ferret', ">= 0.10.13"
 
 module Redwood
 
@@ -45,7 +44,7 @@ class Index
     @sources[source.id] = source
   end
 
-  def source_for name; @sources.values.find { |s| s.is_source_for? name }; end
+  def source_for uri; @sources.values.find { |s| s.is_source_for? uri }; end
   def usual_sources; @sources.values.find_all { |s| s.usual? }; end
   def sources; @sources.values; end
 
@@ -73,24 +72,29 @@ class Index
     end
   end
 
-  ## update the message by deleting and re-adding
-  def update_message m, source=nil, source_info=nil
-    docid, entry = load_entry_for_id m.id
-    if entry
-      source ||= entry[:source_id].to_i
-      source_info ||= entry[:source_info].to_i
+  ## Update the message state on disk, by deleting and re-adding it.
+  ## The message must exist in the index. docid and entry are found
+  ## unless given.
+  ##
+  ## Overwrites the labels on disk with the new labels in 'm', so that
+  ## we can actually change message state.
+  def update_message m, docid=nil, entry=nil
+    unless docid && entry
+      docid, entry = load_entry_for_id m.id
+      raise ArgumentError, "cannot find #{m.id} in the index" unless entry
     end
 
-    ## this happens sometimes. i'm not sure why. ferret bug?
-    raise "no entry and no source info for message #{m.id}: source #{source.inspect}, info #{source_info.inspect}, entry #{entry.inspect}, query #{Ferret::Search::TermQuery.new(:message_id, m.id)}, results #{@index.search(Ferret::Search::TermQuery.new(:message_id, m.id)).inspect}" unless source && source_info
+    raise "no entry and no source info for message #{m.id}" unless m.source && m.source_info
 
     raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+
     @index.delete docid
     add_message m
+    docid, entry = load_entry_for_id m.id
   end
 
   def save_index fn=File.join(@dir, "ferret")
-    # don't have to do anything apparently
+    # don't have to do anything,  apparently
   end
 
   def contains_id? id
@@ -100,8 +104,7 @@ class Index
   def size; @index.size; end
 
   ## you should probably not call this on a block that doesn't break
-  ## rather quickly because the results will probably be, as we say
-  ## in scotland, frikkin' huuuge.
+  ## rather quickly because the results can be very large.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
     return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
@@ -123,20 +126,20 @@ class Index
   end
 
   ## yield all messages in the thread containing 'm' by repeatedly
-  ## querying the index.  yields pairs of message ids and
+  ## querying the index. uields pairs of message ids and
   ## message-building lambdas, so that building an unwanted message
   ## can be skipped in the block if desired.
+  ##
+  ## stops loading any thread if a message with a :killed flag is found.
+
   SAME_SUBJECT_DATE_LIMIT = 7
   def each_message_in_thread_for m, opts={}
     messages = {}
     searched = {}
     num_queries = 0
 
-    ## temporarily disabling subject searching because it's a
-    ## significant slowdown.
-    ##
-    ## TODO: make this configurable, i guess
-    if true
+    ## todo: make subject querying configurable
+    if true # do subject queries
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
 
@@ -146,9 +149,10 @@ class Index
         sq.add_term t
       end
       q.add_query sq, :must
-      q.add_query Ferret::Search::TermQuery.new(:label, "spam"), :must_not
       q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
 
+      q = build_query :qobj => q
+
       pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
       Redwood::log "found #{pending.size} results for subject query #{q}"
     else
@@ -163,9 +167,12 @@ class Index
       q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should
       q.add_query Ferret::Search::TermQuery.new(:refs, id), :should
 
+      q = build_query :qobj => q, :load_killed => true
+
       num_queries += 1
       @index.search_each(q, :limit => :all) do |docid, score|
         break if opts[:limit] && messages.size >= opts[:limit]
+        break if @index[docid][:label].split(/\s+/).include? "killed" unless opts[:load_killed]
         mid = @index[docid][:message_id]
         unless messages.member? mid
           messages[mid] ||= lambda { build_message docid }
@@ -174,7 +181,7 @@ class Index
         end
       end
     end
-    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}"
+    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}" if num_queries > 0
     messages.each { |mid, builder| yield mid, builder }
   end
 
@@ -203,8 +210,10 @@ class Index
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
   def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
+  ## Adds a message to the index. The message cannot already exist in
+  ## the index.
   def add_message m
-    return false if contains? m
+    raise ArgumentError, "index already contains #{m.id}" if contains? m
 
     source_id = 
       if m.source.is_a? Integer
@@ -230,9 +239,9 @@ class Index
 
     @index.add_document d
     
-    ## TODO: figure out why this is sometimes triggered
-    #docid, entry = load_entry_for_id m.id
-    #raise "just added message #{m.id} but couldn't find it in a search" unless docid
+    docid, entry = load_entry_for_id m.id
+    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
+    raise "just added message #{m.id} but couldn't find it in a search" unless docid
     true
   end
 
@@ -294,6 +303,7 @@ protected
     end
         
     query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam)
+    query.add_query Ferret::Search::TermQuery.new("label", "deleted"), :must_not unless opts[:load_deleted] || labels.include?(:deleted)
     query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed)
     query
   end

data/lib/sup/label.rb

@@ -5,10 +5,10 @@ class LabelManager
 
   ## labels that have special semantics. user will be unable to
   ## add/remove these via normal label mechanisms.
-  RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent ]
+  RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent, :deleted ]
 
   ## labels which it nonetheless makes sense to search for by
-  LISTABLE_LABELS = [ :starred, :spam, :draft, :sent ]
+  LISTABLE_LABELS = [ :starred, :spam, :draft, :sent, :killed, :deleted ]
 
   ## labels that will never be displayed to the user
   HIDDEN_LABELS = [ :starred, :unread ]

data/lib/sup/mbox.rb

@@ -16,24 +16,29 @@ module MBox
     ## when scanning over large mbox files.
     while(line = f.gets)
       case line
-      when /^From:\s+(.*)$/i: header[last = "From"] = $1
-      when /^To:\s+(.*)$/i: header[last = "To"] = $1
-      when /^Cc:\s+(.*)$/i: header[last = "Cc"] = $1
-      when /^Bcc:\s+(.*)$/i: header[last = "Bcc"] = $1
-      when /^Subject:\s+(.*)$/i: header[last = "Subject"] = $1
-      when /^Date:\s+(.*)$/i: header[last = "Date"] = $1
-      when /^Message-Id:\s+<(.*)>$/i: header[last = "Message-Id"] = $1
-      when /^References:\s+(.*)$/i: header[last = "References"] = $1
-      when /^In-Reply-To:\s+(.*)$/i: header[last = "In-Reply-To"] = $1
-      when /^List-Post:\s+(.*)$/i: header[last = "List-Post"] = $1
-      when /^Reply-To:\s+(.*)$/i: header[last = "Reply-To"] = $1
-      when /^Status:\s+(.*)$/i: header[last = "Status"] = $1
-      when /^Delivered-To:\s+(.*)$/i
-        header[last = "Delivered-To"] = $1 unless header["Delivered-To"]
+      when /^(From):\s+(.*)$/i,
+        /^(To):\s+(.*)$/i,
+        /^(Cc):\s+(.*)$/i,
+        /^(Bcc):\s+(.*)$/i,
+        /^(Subject):\s+(.*)$/i,
+        /^(Date):\s+(.*)$/i,
+        /^(Message-Id):\s+<(.*)>$/i,
+        /^(References):\s+(.*)$/i,
+        /^(In-Reply-To):\s+(.*)$/i,
+        /^(Reply-To):\s+(.*)$/i,
+        /^(List-Post):\s+(.*)$/i,
+        /^(Status):\s+(.*)$/i: header[last = $1] = $2
+
+      ## these next three can occur multiple times, and we want the
+      ## first one
+      when /^(Delivered-To):\s+(.*)$/i,
+        /^(X-Original-To):\s+(.*)$/i,
+        /^(Envelope-To):\s+(.*)$/i: header[last = $1.downcase] ||= $2
+
       when /^$/: break
       when /:/: last = nil
       else
-        header[last] += line.gsub(/^\s+/, "") if last
+        header[last] += " " + line.chomp.gsub(/^\s+/, "") if last
       end
     end
     header