sup 0.0.2 → 0.0.3

data/lib/sup/draft.rb

@@ -49,13 +49,8 @@ class DraftLoader < Source
   def id; DraftManager.source_id; end
   def to_s; DraftManager.source_name; end
 
-  def next
-    ret = nil
-    begin
-      ret = cur_offset
-      self.cur_offset = cur_offset + 1
-    end until File.exists? fn_for_offset(ret)
-    [ret, [:draft]]
+  def each
+    Dir.entries(@dir).select { |x| x =~ /^\d+$/ }.sort_by { |x| x.to_i }.each { |id| yield [id, [:draft]] }
   end
 
   def gen_offset
@@ -101,7 +96,7 @@ class DraftLoader < Source
   end
 
   def start_offset; 0; end
-  def end_offset; Dir.new(@dir).entries.sort.last.to_i + 1; end
+  def end_offset; Dir.new(@dir).entries.sort.last.to_i; end
 end
 
 Redwood::register_yaml(DraftLoader, %w(cur_offset))

data/lib/sup/imap.rb

@@ -1,32 +1,70 @@
 require 'uri'
 require 'net/imap'
 require 'stringio'
+require 'time'
+
+## fucking imap fucking sucks. what the FUCK kind of committee of
+## dunces designed this shit.
+
+## imap talks about 'unique ids' for messages, to be used for
+## cross-session identification. great---just what sup needs! except
+## it turns out the uids can be invalidated every time the
+## 'uidvalidity' value changes on the server, and 'uidvalidity' can
+## change without restriction. it can change any time you log in. it
+## can change EVERY time you log in. of course the imap spec "strongly
+## recommends" that it never change, but there's nothing to stop
+## people from just setting it to the current timestamp, and in fact
+## that's exactly what the one imap server i have at my disposal
+## does. thus the so-called uids are absolutely useless and imap
+## provides no cross-session way of uniquely identifying a
+## message. but thanks for the "strong recommendation", guys!
+
+## so right now i'm using the 'internal date' and the size of each
+## message to uniquely identify it, and i scan over the entire mailbox
+## each time i open it to map those things to message ids. that can be
+## slow for large mailboxes, and we'll just have to hope that there
+## are no collisions. ho ho! a perfectly reasonable solution!
+
+## fuck you, imap committee. you managed to design something as shitty
+## as mbox but goddamn THIRTY YEARS LATER.
 
 module Redwood
 
 class IMAP < Source
-  attr_reader :labels
-  
-  def initialize uri, username, password, last_uid=nil, usual=true, archived=false, id=nil
+  attr_reader_cloned :labels
+  attr_accessor :username, :password
+
+  def initialize uri, username, password, last_idate=nil, usual=true, archived=false, id=nil
     raise ArgumentError, "username and password must be specified" unless username && password
+    raise ArgumentError, "not an imap uri" unless uri =~ %r!imaps?://!
 
-    super uri, last_uid, usual, archived, id
+    super uri, last_idate, usual, archived, id
 
     @parsed_uri = URI(uri)
     @username = username
     @password = password
     @imap = nil
+    @imap_ids = {}
+    @ids = []
     @labels = [:unread]
-    @labels << mailbox.intern unless mailbox =~ /inbox/i || mailbox.empty?
     @labels << :inbox unless archived?
+    @labels << mailbox.intern unless mailbox =~ /inbox/i || mailbox.nil?
+    @mutex = Mutex.new
+  end
 
-    connect
+  def say s
+    @say_id = BufferManager.say s, @say_id if BufferManager.instantiated?
+    Redwood::log s
   end
+  def shutup
+    BufferManager.clear @say_id if BufferManager.instantiated?
+    @say_id = nil
+  end
+  private :say, :shutup
 
   def connect
     return false if broken?
     return true if @imap
-    Redwood::log "connecting to #{@parsed_uri.host} port #{ssl? ? 993 : 143}, ssl=#{ssl?} ..."
 
     ## ok, this is FUCKING ANNOYING.
     ##
@@ -40,17 +78,35 @@ class IMAP < Source
     ## problem.
     ##
     ## FUCK!!!!!!!!!
-    ::Thread.new do
+
+    say "Connecting to IMAP server #{host}:#{port}..."
+
+    Redwood::reporting_thread do
       begin
         #raise Net::IMAP::ByeResponseError, "simulated imap failure"
-        @imap = Net::IMAP.new @parsed_uri.host, ssl? ? 993 : 143, ssl?
+        @imap = Net::IMAP.new host, ssl? ? 993 : 143, ssl?
+        say "Logging in..."
         @imap.authenticate 'LOGIN', @username, @password
+        say "Sizing mailbox..."
         @imap.examine mailbox
-        Redwood::log "successfully connected to #{@parsed_uri}, mailbox #{mailbox}"
-      rescue Exception => e
+        last_id = @imap.responses["EXISTS"][-1]
+        
+        say "Reading headers (because IMAP sucks)..."
+        values = @imap.fetch(1 .. last_id, ['RFC822.SIZE', 'INTERNALDATE'])
+        
+        say "Successfully connected to #{@parsed_uri}"
+        
+        values.each do |v|
+          id = make_id v
+          @ids << id
+          @imap_ids[id] = v.seqno
+        end
+      rescue SocketError, Net::IMAP::Error, SourceError => e
         self.broken_msg = e.message.chomp # fucking chomp! fuck!!!
         @imap = nil
         Redwood::log "error connecting to IMAP server: #{self.broken_msg}"
+      ensure
+        shutup
       end
     end.join
 
@@ -58,50 +114,78 @@ class IMAP < Source
   end
   private :connect
 
-  def mailbox; @parsed_uri.path[1..-1] end ##XXXX TODO handle nil
+  def make_id imap_stuff
+    msize, mdate = imap_stuff.attr['RFC822.SIZE'], Time.parse(imap_stuff.attr["INTERNALDATE"])
+    sprintf("%d%07d", mdate.to_i, msize).to_i
+  end
+  private :make_id
+
+  def host; @parsed_uri.host; end
+  def port; @parsed_uri.port || (ssl? ? 993 : 143); end
+  def mailbox; @parsed_uri.path[1..-1] || 'INBOX'; end
   def ssl?; @parsed_uri.scheme == 'imaps' end
 
-  def load_header uid=nil
-    MBox::read_header StringIO.new(raw_header(uid))
+  def load_header id
+    MBox::read_header StringIO.new(raw_header(id))
   end
 
-  def load_message uid
-    RMail::Parser.read raw_full_message(uid)
+  def load_message id
+    RMail::Parser.read raw_full_message(id)
   end
 
   ## load the full header text
-  def raw_header uid
-    connect or return broken_msg
-    begin
-      connect or return broken_msg
-    rescue Exception => e
-      raise "wtf: #{e.inspect}"
+  def raw_header id
+    @mutex.synchronize do
+      connect or raise SourceError, broken_msg
+      get_imap_field(id, 'RFC822.HEADER').gsub(/\r\n/, "\n")
     end
-    @imap.uid_fetch(uid, 'RFC822.HEADER')[0].attr['RFC822.HEADER'].gsub(/\r\n/, "\n")
   end
 
-  def raw_full_message uid
-    connect or return broken_msg
-    @imap.uid_fetch(uid, 'RFC822')[0].attr['RFC822'].gsub(/\r\n/, "\n")
+  def raw_full_message id
+    @mutex.synchronize do
+      connect or raise SourceError, broken_msg
+      get_imap_field(id, 'RFC822').gsub(/\r\n/, "\n")
+    end
   end
+
+  def get_imap_field id, field
+    f = nil
+    imap_id = @imap_ids[id] or raise SourceError, "Unknown message id #{id}. It is likely that messages have been deleted from this IMAP mailbox."
+    begin
+      f = @imap.fetch imap_id, [field, 'RFC822.SIZE', 'INTERNALDATE']
+      got_id = make_id f[0]
+      raise SourceError, "IMAP message mismatch: requested #{id}, got #{got_id}. It is likely the IMAP mailbox has been modified." unless got_id == id
+    rescue Net::IMAP::Error => e
+      raise SourceError, e.message
+    end
+    raise SourceError, "null IMAP field '#{field}' for message with id #{id} imap id #{imap_id}" if f.nil?
+
+    f[0].attr[field]
+  end
+  private :get_imap_field
   
   def each
-    connect or return broken_msg
-    uids = @imap.uid_search ['UID', "#{cur_offset}:#{end_offset}"]
-    uids.each do |uid|
-      @last_uid = uid
-      @dirty = true
-      yield uid, labels
+    @mutex.synchronize { connect or raise SourceError, broken_msg }
+
+    start = @ids.index(cur_offset || start_offset)
+    start.upto(@ids.length - 1) do |i|
+      id = @ids[i]
+      self.cur_offset = id
+      yield id, labels
     end
   end
 
-  def start_offset; 1; end
+  def start_offset
+    @mutex.synchronize { connect or raise SourceError, broken_msg }
+    @ids.first
+  end
+
   def end_offset
-    connect or return start_offset
-    @imap.uid_search(['ALL']).last
+    @mutex.synchronize { connect or raise SourceError, broken_msg }
+    @ids.last
   end
 end
 
-Redwood::register_yaml(IMAP, %w(uri username password offset usual archived id))
+Redwood::register_yaml(IMAP, %w(uri username password cur_offset usual archived id))
 
 end

data/lib/sup/index.rb

@@ -2,30 +2,26 @@
 
 require 'thread'
 require 'fileutils'
-require_gem 'ferret', ">= 0.10.13"
+require 'ferret'
+#require_gem 'ferret', ">= 0.10.13"
 
 module Redwood
 
-class IndexError < StandardError
-  attr_reader :source
-
-  def initialize source, s
-    super s
-    @source = source
-  end
-end
-
 class Index
   include Singleton
 
-  attr_reader :index # debugging only
-  
+  attr_reader :index
   def initialize dir=BASE_DIR
     @dir = dir
-    @mutex = Mutex.new
     @sources = {}
     @sources_dirty = false
 
+    wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false
+    sa = Ferret::Analysis::StandardAnalyzer.new Ferret::Analysis::FULL_ENGLISH_STOP_WORDS, true
+    @analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa
+    @analyzer[:body] = sa
+    @qparser ||= Ferret::QueryParser.new :default_field => :body, :analyzer => @analyzer
+
     self.class.i_am_the_instance self
   end
 
@@ -51,18 +47,15 @@ class Index
 
   def source_for name; @sources.values.find { |s| s.is_source_for? name }; end
   def usual_sources; @sources.values.find_all { |s| s.usual? }; end
+  def sources; @sources.values; end
 
   def load_index dir=File.join(@dir, "ferret")
-    wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false
-    sa = Ferret::Analysis::StandardAnalyzer.new
-    analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa
-    analyzer[:body] = sa
-
     if File.exists? dir
-      Redwood::log "loading index"
-      @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer)
+      Redwood::log "loading index..."
+      @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
+      Redwood::log "loaded index of #{@index.size} messages"
     else
-      Redwood::log "creating index"
+      Redwood::log "creating index..."
       field_infos = Ferret::Index::FieldInfos.new :store => :yes
       field_infos.add_field :message_id
       field_infos.add_field :source_id
@@ -76,7 +69,7 @@ class Index
       field_infos.add_field :refs
       field_infos.add_field :snippet, :index => :no, :term_vector => :no
       field_infos.create_index dir
-      @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer)
+      @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
     end
   end
 
@@ -87,7 +80,9 @@ class Index
       source ||= entry[:source_id].to_i
       source_info ||= entry[:source_info].to_i
     end
-    raise "no entry and no source info for message #{m.id}" unless source && source_info
+
+    ## this happens sometimes. i'm not sure why. ferret bug?
+    raise "no entry and no source info for message #{m.id}: source #{source.inspect}, info #{source_info.inspect}, entry #{entry.inspect}, query #{Ferret::Search::TermQuery.new(:message_id, m.id)}, results #{@index.search(Ferret::Search::TermQuery.new(:message_id, m.id)).inspect}" unless source && source_info
 
     raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
     @index.delete docid
@@ -109,7 +104,7 @@ class Index
   ## in scotland, frikkin' huuuge.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
-    return if @index.size == 0 # otherwise ferret barfs
+    return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
     query = build_query opts
     offset = 0
     while true
@@ -122,12 +117,15 @@ class Index
   end
 
   def num_results_for opts={}
-    query = build_query opts
-    x = @index.search(query).total_hits
-    Redwood::log "num_results_for: have #{x} for query #{query}"
-    x
+    return 0 if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
+    q = build_query opts
+    index.search(q).total_hits
   end
 
+  ## yield all messages in the thread containing 'm' by repeatedly
+  ## querying the index.  yields pairs of message ids and
+  ## message-building lambdas, so that building an unwanted message
+  ## can be skipped in the block if desired.
   SAME_SUBJECT_DATE_LIMIT = 7
   def each_message_in_thread_for m, opts={}
     messages = {}
@@ -186,44 +184,18 @@ class Index
     #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})"
     raise "invalid source #{doc[:source_id]}" unless source
 
-    m = 
-      if source.broken?
-        nil
-      else
-        begin
-          Message.new :source => source, :source_info => doc[:source_info].to_i, 
-                      :labels => doc[:label].split(" ").map { |s| s.intern },
-                      :snippet => doc[:snippet]
-        rescue MessageFormatError => e
-          raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}")
-        rescue SourceError => e
-          nil
-        end
-      end
+    fake_header = {
+      "date" => Time.at(doc[:date].to_i),
+      "subject" => unwrap_subj(doc[:subject]),
+      "from" => doc[:from],
+      "to" => doc[:to],
+      "message-id" => doc[:message_id],
+      "references" => doc[:refs],
+    }
 
-    unless m
-      fake_header = {
-        "date" => Time.at(doc[:date].to_i),
-        "subject" => unwrap_subj(doc[:subject]),
-        "from" => doc[:from],
-        "to" => doc[:to],
-        "message-id" => doc[:message_id],
-        "references" => doc[:refs],
-      }
-
-      m = Message.new :labels => doc[:label].split(" ").map { |s| s.intern },
-                      :snippet => doc[:snippet], :header => fake_header, 
-                      :body => <<EOS
-#{doc[:snippet]}...
-
-An error occurred while loading this message. It is possible that the source
-has changed, or (in the case of remote sources) is down.
-
-The error message was:
-  #{source.broken_msg}
-EOS
-    end
-    m
+    Message.new :source => source, :source_info => doc[:source_info].to_i, 
+                :labels => doc[:label].split(" ").map { |s| s.intern },
+                :snippet => doc[:snippet], :header => fake_header
   end
 
   def fresh_thread_id; @next_thread_id += 1; end
@@ -237,7 +209,7 @@ EOS
       if m.source.is_a? Integer
         m.source
       else
-        m.source.id or raise "unregistered source #{m.source}"
+        m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
       end
 
     to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
@@ -305,21 +277,23 @@ EOS
 
 protected
 
-  ## TODO: convert this to query objects rather than strings
+  def parse_user_query_string str; @qparser.parse str; end
   def build_query opts
-    query = ""
-    query += opts[:labels].map { |t| "+label:#{t}" }.join(" ") if opts[:labels]
-    query += " +label:#{opts[:label]}" if opts[:label]
-    query += " #{opts[:content]}" if opts[:content]
+    query = Ferret::Search::BooleanQuery.new
+    query.add_query opts[:qobj], :must if opts[:qobj]
+    labels = ([opts[:label]] + (opts[:labels] || [])).compact
+    labels.each { |t| query.add_query Ferret::Search::TermQuery.new("label", t.to_s), :must }
     if opts[:participants]
-      query += "+(" + 
-        opts[:participants].map { |p| "from:#{p.email} OR to:#{p.email}" }.join(" OR ") + ")"
+      q2 = Ferret::Search::BooleanQuery.new
+      opts[:participants].each do |p|
+        q2.add_query Ferret::Search::TermQuery.new("from", p.email), :should
+        q2.add_query Ferret::Search::TermQuery.new("to", p.email), :should
+      end
+      query.add_query q2, :must
     end
         
-    query += " -label:spam" unless opts[:load_spam] || opts[:labels] == :spam || 
-      (opts[:labels] && opts[:labels].include?(:spam))
-    query += " -label:killed" unless opts[:load_killed] || opts[:labels] == :killed || 
-      (opts[:labels] && opts[:labels].include?(:killed))
+    query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam)
+    query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed)
     query
   end
 
@@ -335,7 +309,7 @@ protected
         File.chmod 0600, fn
         FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(bakfn) > File.size(fn)
       end
-      Redwood::save_yaml_obj @sources.values, fn 
+      Redwood::save_yaml_obj @sources.values, fn
       File.chmod 0600, fn
     end
     @sources_dirty = false