sup 0.0.4 → 0.0.5

data/lib/sup/imap.rb

@@ -31,6 +31,8 @@ require 'time'
 module Redwood
 
 class IMAP < Source
+  SCAN_INTERVAL = 60 # seconds
+
   attr_reader_cloned :labels
   attr_accessor :username, :password
 
@@ -46,26 +48,51 @@ class IMAP < Source
     @imap = nil
     @imap_ids = {}
     @ids = []
+    @last_scan = nil
     @labels = [:unread]
     @labels << :inbox unless archived?
     @labels << mailbox.intern unless mailbox =~ /inbox/i
     @mutex = Mutex.new
   end
 
-  def say s
-    @say_id = BufferManager.say s, @say_id if BufferManager.instantiated?
-    Redwood::log s
+  def host; @parsed_uri.host; end
+  def port; @parsed_uri.port || (ssl? ? 993 : 143); end
+  def mailbox
+    x = @parsed_uri.path[1..-1]
+    x.nil? || x.empty? ? 'INBOX' : x
   end
-  def shutup
-    BufferManager.clear @say_id if BufferManager.instantiated?
-    @say_id = nil
+  def ssl?; @parsed_uri.scheme == 'imaps' end
+
+  def load_header id
+    MBox::read_header StringIO.new(raw_header(id))
+  end
+
+  def load_message id
+    RMail::Parser.read raw_full_message(id)
+  end
+
+  def raw_header id
+    @mutex.synchronize do
+      connect
+      header, flags = get_imap_fields id, 'RFC822.HEADER', 'FLAGS'
+      header = "Status: RO\n" + header if flags.include? :Seen # fake an mbox-style read header
+      header.gsub(/\r\n/, "\n")
+    end
+  end
+
+  def raw_full_message id
+    @mutex.synchronize do
+      connect
+      get_imap_fields(id, 'RFC822').first.gsub(/\r\n/, "\n")
+    end
   end
-  private :say, :shutup
 
   def connect
     return false if broken?
     return true if @imap
 
+    say "Connecting to IMAP server #{host}:#{port}..."
+
     ## ok, this is FUCKING ANNOYING.
     ##
     ## what imap.rb likes to do is, if an exception occurs, catch it
@@ -79,88 +106,120 @@ class IMAP < Source
     ##
     ## FUCK!!!!!!!!!
 
-    say "Connecting to IMAP server #{host}:#{port}..."
-
+    exception = nil
     Redwood::reporting_thread do
       begin
         #raise Net::IMAP::ByeResponseError, "simulated imap failure"
-        @imap = Net::IMAP.new host, ssl? ? 993 : 143, ssl?
+        @imap = Net::IMAP.new host, port, ssl?
         say "Logging in..."
         @imap.authenticate 'LOGIN', @username, @password
-        say "Sizing mailbox..."
-        @imap.examine mailbox
-        last_id = @imap.responses["EXISTS"][-1]
-        
-        say "Reading headers (because IMAP sucks)..."
-        values = @imap.fetch(1 .. last_id, ['RFC822.SIZE', 'INTERNALDATE'])
-        
-        say "Successfully connected to #{@parsed_uri}"
-        
-        values.each do |v|
-          id = make_id v
-          @ids << id
-          @imap_ids[id] = v.seqno
-        end
+        scan_mailbox
+        say "Successfully connected to #{@parsed_uri}."
       rescue SocketError, Net::IMAP::Error, SourceError => e
-        self.broken_msg = e.message.chomp # fucking chomp! fuck!!!
-        @imap = nil
-        Redwood::log "error connecting to IMAP server: #{self.broken_msg}"
+        exception = e
       ensure
         shutup
       end
     end.join
 
-    !!@imap
+    die_from exception, :while => "connecting" if exception
   end
-  private :connect
 
-  def make_id imap_stuff
-    msize, mdate = imap_stuff.attr['RFC822.SIZE'], Time.parse(imap_stuff.attr["INTERNALDATE"])
-    sprintf("%d%07d", mdate.to_i, msize).to_i
-  end
-  private :make_id
+  def each
+    @mutex.synchronize { connect or raise SourceError, broken_msg }
 
-  def host; @parsed_uri.host; end
-  def port; @parsed_uri.port || (ssl? ? 993 : 143); end
-  def mailbox
-    x = @parsed_uri.path[1..-1]
-    x.nil? || x.empty? ? 'INBOX' : x
-  end
-  def ssl?; @parsed_uri.scheme == 'imaps' end
+    start = @ids.index(cur_offset || start_offset) or die_from "Unknown message id #{cur_offset || start_offset}.", :suggest_rebuild => true # couldn't find the most recent email
 
-  def load_header id
-    MBox::read_header StringIO.new(raw_header(id))
+    start.upto(@ids.length - 1) do |i|         
+      id = @ids[i]
+      self.cur_offset = id
+      yield id, labels
+    end
   end
 
-  def load_message id
-    RMail::Parser.read raw_full_message(id)
+  def start_offset
+    @mutex.synchronize { connect }
+    @ids.first
   end
 
-  ## load the full header text
-  def raw_header id
+  def end_offset
     @mutex.synchronize do
-      connect or raise SourceError, broken_msg
-      get_imap_field(id, 'RFC822.HEADER').gsub(/\r\n/, "\n")
+      begin
+        connect
+        scan_mailbox
+      rescue SocketError, Net::IMAP::Error => e
+        die_from e, :while => "scanning mailbox"
+      end
     end
+    @ids.last
   end
 
-  def raw_full_message id
-    @mutex.synchronize do
-      connect or raise SourceError, broken_msg
-      get_imap_field(id, 'RFC822').gsub(/\r\n/, "\n")
+  def pct_done; 100.0 * (@ids.index(cur_offset) || 0).to_f / (@ids.length - 1).to_f; end
+
+private
+
+  def say s
+    @say_id = BufferManager.say s, @say_id if BufferManager.instantiated?
+    Redwood::log s
+  end
+
+  def shutup
+    BufferManager.clear @say_id if BufferManager.instantiated?
+    @say_id = nil
+  end
+
+  def scan_mailbox
+    return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
+
+    @imap.examine mailbox
+    last_id = @imap.responses["EXISTS"].last
+    @last_scan = Time.now
+    return if last_id == @ids.length
+    Redwood::log "fetching IMAP headers #{(@ids.length + 1) .. last_id}"
+    values = @imap.fetch((@ids.length + 1) .. last_id, ['RFC822.SIZE', 'INTERNALDATE'])
+    values.each do |v|
+      id = make_id v
+      @ids << id
+      @imap_ids[id] = v.seqno
     end
   end
 
-  def get_imap_field id, field
+  def die_from e, opts={}
+    @imap = nil
+
+    message =
+      case e
+      when Exception
+        "Error while #{opts[:while]}: #{e.message.chomp}."
+      when String
+        e
+      end
+
+    message += " It is likely that messages have been deleted from this IMAP mailbox. Please run sup-import --rebuild #{to_s} to correct this problem." if opts[:suggest_rebuild]
+
+    self.broken_msg = message
+    Redwood::log message
+    BufferManager.flash "Error communicating with IMAP server. See log for details."
+    raise SourceError, message
+  end
+  
+  ## build a fake unique id
+  def make_id imap_stuff
+    # use 7 digits for the size. why 7? seems nice.
+    msize, mdate = imap_stuff.attr['RFC822.SIZE'] % 10000000, Time.parse(imap_stuff.attr["INTERNALDATE"])
+    sprintf("%d%07d", mdate.to_i, msize).to_i
+  end
+
+  def get_imap_fields id, *fields
     retries = 0
     f = nil
-    imap_id = @imap_ids[id] or raise SourceError, "Unknown message id #{id}. It is likely that messages have been deleted from this IMAP mailbox."
+    imap_id = @imap_ids[id] or die_from "Unknown message id #{id}.", :suggest_rebuild => true
     begin
-      f = @imap.fetch imap_id, [field, 'RFC822.SIZE', 'INTERNALDATE']
+      f = @imap.fetch imap_id, (fields + ['RFC822.SIZE', 'INTERNALDATE']).uniq
       got_id = make_id f[0]
-      raise SourceError, "IMAP message mismatch: requested #{id}, got #{got_id}. It is likely the IMAP mailbox has been modified." unless got_id == id
-    rescue Net::IMAP::Error => e
-      raise SourceError, e.message
+      die_from "IMAP message mismatch: requested #{id}, got #{got_id}.", :suggest_rebuild => true unless id == got_id
+    rescue SocketError, Net::IMAP::Error => e
+      die_from e, :while => "communicating with IMAP server"
     rescue Errno::EPIPE
       if (retries += 1) <= 3
         @imap = nil
@@ -168,31 +227,9 @@ class IMAP < Source
         retry
       end
     end
-    raise SourceError, "null IMAP field '#{field}' for message with id #{id} imap id #{imap_id}" if f.nil?
-
-    f[0].attr[field]
-  end
-  private :get_imap_field
-  
-  def each
-    @mutex.synchronize { connect or raise SourceError, broken_msg }
-
-    start = @ids.index(cur_offset || start_offset)
-    start.upto(@ids.length - 1) do |i|
-      id = @ids[i]
-      self.cur_offset = id
-      yield id, labels
-    end
-  end
+    die_from "Null IMAP field '#{field}' for message with id #{id} imap id #{imap_id}." if f.nil?
 
-  def start_offset
-    @mutex.synchronize { connect or raise SourceError, broken_msg }
-    @ids.first
-  end
-
-  def end_offset
-    @mutex.synchronize { connect or raise SourceError, broken_msg }
-    @ids.last
+    fields.map { |field| f[0].attr[field] }
   end
 end
 

data/lib/sup/index.rb

@@ -136,7 +136,7 @@ class Index
     ## significant slowdown.
     ##
     ## TODO: make this configurable, i guess
-    if false
+    if true
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
 
@@ -146,6 +146,7 @@ class Index
         sq.add_term t
       end
       q.add_query sq, :must
+      q.add_query Ferret::Search::TermQuery.new(:label, "spam"), :must_not
       q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
 
       pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }

data/lib/sup/mbox/loader.rb

@@ -29,7 +29,7 @@ class Loader < Source
 
   def start_offset; 0; end
   def end_offset; File.size @f; end
-  def total; end_offset; end
+  def pct_done; 100.0 * cur_offset.to_f / end_offset.to_f; end
 
   def load_header offset
     header = nil

data/lib/sup/mbox/ssh-file.rb

@@ -66,6 +66,7 @@ class Buffer
     x = @buf.index(what, start - @start)
     x.nil? ? nil : x + @start
   end
+
   def rindex what, start=0
     x = @buf.rindex(what, start - @start)
     x.nil? ? nil : x + @start
@@ -91,6 +92,9 @@ class SSHFile
   REASONABLE_TRANSFER_SIZE = 1024 * 32
   SIZE_CHECK_INTERVAL = 60 * 1 # seconds
 
+  @@shells = {}
+  @@shells_mutex = Mutex.new
+
   def initialize host, fn, ssh_opts={}
     @buf = Buffer.new
     @host = host
@@ -102,6 +106,7 @@ class SSHFile
     @broken_msg = nil
     @shell = nil
     @shell_mutex = Mutex.new
+    @buf_mutex = Mutex.new
   end
 
   def to_s; "mbox+ssh://#@host/#@fn"; end ## TODO: remove thisis EVILness
@@ -113,28 +118,32 @@ class SSHFile
     Redwood::log s
   end
   def shutup
-    BufferManager.clear @say_id if BufferManager.instantiated?
+    BufferManager.clear @say_id if BufferManager.instantiated? && @say_id
     @say_id = nil
   end
   private :say, :shutup
 
   def connect
     raise SSHFileError, @broken_msg if broken?
+    return if @shell
 
-    @shell_mutex.synchronize do
-      return if @shell
-
-      begin
-        say "Opening SSH connection to #{@host}..."
-        #raise SSHFileError, "simulated SSH file error"
-        session = Net::SSH.start @host, @ssh_opts
-        say "Starting SSH shell..."
-        @shell = session.shell.sync
-        say "Checking for #@fn..."
-        raise Errno::ENOENT, @fn unless @shell.test("-e #@fn").status == 0
-      ensure
-        shutup
+    @key = [@host, @ssh_opts[:username]]
+    begin
+      @shell = @@shells_mutex.synchronize do
+        unless @@shells.member? @key
+          say "Opening SSH connection to #{@host} for #@fn..."
+          #raise SSHFileError, "simulated SSH file error"
+          session = Net::SSH.start @host, @ssh_opts
+          say "Starting SSH shell..."
+          @@shells[@key] = session.shell.sync
+        end
+        @@shells[@key]
       end
+
+      say "Checking for #@fn..."
+      @shell_mutex.synchronize { raise Errno::ENOENT, @fn unless @shell.test("-e #@fn").status == 0 }
+    ensure
+      shutup
     end
   end
 
@@ -154,15 +163,19 @@ class SSHFile
 
   def gets
     return nil if eof?
-    make_buf_include @offset
-    expand_buf_forward while @buf.index("\n", @offset).nil? && @buf.endd < size
-    returning(@buf[@offset .. (@buf.index("\n", @offset) || -1)]) { |line| @offset += line.length }
+    @buf_mutex.synchronize do
+      make_buf_include @offset
+      expand_buf_forward while @buf.index("\n", @offset).nil? && @buf.endd < size
+      returning(@buf[@offset .. (@buf.index("\n", @offset) || -1)]) { |line| @offset += line.length }
+    end
   end
 
   def read n
     return nil if eof?
-    make_buf_include @offset, n
-    @buf[@offset ... (@offset += n)]
+    @buf_mutex.synchronize do
+      make_buf_include @offset, n
+      @buf[@offset ... (@offset += n)]
+    end
   end
 
 private
@@ -173,14 +186,17 @@ private
       connect
       # MBox::debug "sending command: #{cmd.inspect}"
       begin
-        result = @shell.send_command cmd
+        result = @shell_mutex.synchronize { x = @shell.send_command cmd; sleep 0.25; x }
         raise SSHFileError, "Failure during remote command #{cmd.inspect}: #{(result.stderr || result.stdout || "")[0 .. 100]}" unless result.status == 0
       rescue Net::SSH::Exception # these happen occasionally for no apparent reason. gotta love that nondeterminism!
         retry if (retries += 1) <= 3
         raise
       rescue Errno::EPIPE
         if (retries += 1) <= e
-          @shell = nil
+          @@shells_mutex.synchronize do
+            @shell = nil
+            @@shells[@key] = nil
+          end
           connect
           retry
         end

data/lib/sup/message.rb

@@ -82,9 +82,9 @@ class Message
 
   attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source,
               :cc, :bcc, :labels, :list_address, :recipient_email, :replyto,
-              :source_info, :status
+              :source_info
 
-  bool_reader :dirty
+  bool_reader :dirty, :source_marked_read
 
   ## if you specify a :header, will use values from that. otherwise, will try and
   ## load the header from the source.
@@ -92,6 +92,7 @@ class Message
     @source = opts[:source] or raise ArgumentError, "source can't be nil"
     @source_info = opts[:source_info] or raise ArgumentError, "source_info can't be nil"
     @snippet = opts[:snippet] || ""
+    @have_snippet = !opts[:snippet].nil?
     @labels = opts[:labels] || []
     @dirty = false
 
@@ -129,8 +130,8 @@ class Message
         nil
       end
 
-    @recipient_email = header["delivered-to"]
-    @status = header["status"]
+    @recipient_email = header["x-original-to"] || header["envelope-to"] || header["delivered-to"]
+    @source_marked_read = header["status"] == "RO"
   end
   private :read_header
 
@@ -177,9 +178,15 @@ class Message
         [Text.new(error_message(@source.broken_msg.split("\n")))]
       else
         begin
+          ## we need to re-read the header because it contains information
+          ## that we don't store in the index. actually i think it's just
+          ## the mailing list address (if any), so this is kinda overkill.
+          ## i could just store that in the index, but i think there might
+          ## be other things like that in the future, and i'd rather not
+          ## bloat the index.
           read_header @source.load_header(@source_info)
           message_to_chunks @source.load_message(@source_info)
-        rescue SourceError, SocketError => e
+        rescue SourceError, SocketError, MessageFormatError => e
           [Text.new(error_message(e.message))]
         end
       end
@@ -246,9 +253,7 @@ private
     ret = [] <<
       case m.header.content_type
       when "text/plain", nil
-        raise MessageFormatError, "no message body before decode (source #@source info #@source_info)" unless
-          m.body
-        body = m.decode or raise MessageFormatError, "no message body"
+        m.body && body = m.decode or raise MessageFormatError, "for some bizarre reason, RubyMail was unable to parse this message."
         text_to_chunks body.normalize_whitespace.split("\n")
       when /^multipart\//
         nil
@@ -322,8 +327,7 @@ private
         chunk_lines << line
       end
  
-      if state == :text && (@snippet.nil? || @snippet.length < SNIPPET_LEN) &&
-          line !~ /[=\*#_-]{3,}/ && line !~ /^\s*$/
+      if !@have_snippet && state == :text && (@snippet.nil? || @snippet.length < SNIPPET_LEN) && line !~ /[=\*#_-]{3,}/ && line !~ /^\s*$/
         @snippet += " " unless @snippet.empty?
         @snippet += line.gsub(/^\s+/, "").gsub(/[\r\n]/, "").gsub(/\s+/, " ")
         @snippet = @snippet[0 ... SNIPPET_LEN].chomp