sup 0.9.1 → 0.10

data/lib/sup/poll.rb

@@ -28,19 +28,18 @@ num_inbox_total_unread: the total number of unread messages in the inbox
                         only those messages appearing in the inbox
 EOS
 
-  DELAY = 300
+  DELAY = $config[:poll_interval] || 300
 
   def initialize
     @mutex = Mutex.new
     @thread = nil
     @last_poll = nil
     @polling = false
+    @poll_sources = nil
     @mode = nil
   end
 
-  def poll
-    return if @polling
-    @polling = true
+  def poll_with_sources
     @mode ||= PollMode.new
     HookManager.run "before-poll"
 
@@ -54,6 +53,22 @@ EOS
 
     HookManager.run "after-poll", :num => num, :num_inbox => numi, :from_and_subj => from_and_subj, :from_and_subj_inbox => from_and_subj_inbox, :num_inbox_total_unread => lambda { Index.num_results_for :labels => [:inbox, :unread] }
 
+  end
+
+  def poll
+    return if @polling
+    @polling = true
+    @poll_sources = SourceManager.usual_sources
+    num, numi = poll_with_sources
+    @polling = false
+    [num, numi]
+  end
+
+  def poll_unusual
+    return if @polling
+    @polling = true
+    @poll_sources = SourceManager.unusual_sources
+    num, numi = poll_with_sources
     @polling = false
     [num, numi]
   end
@@ -79,7 +94,7 @@ EOS
     loaded_labels = Set.new
 
     @mutex.synchronize do
-      SourceManager.usual_sources.each do |source|
+      @poll_sources.each do |source|
 #        yield "source #{source} is done? #{source.done?} (cur_offset #{source.cur_offset} >= #{source.end_offset})"
         begin
           yield "Loading from #{source}... " unless source.done? || (source.respond_to?(:has_errors?) && source.has_errors?)

data/lib/sup/source.rb

@@ -207,6 +207,7 @@ class SourceManager
 
   def source_for uri; sources.find { |s| s.is_source_for? uri }; end
   def usual_sources; sources.find_all { |s| s.usual? }; end
+  def unusual_sources; sources.find_all { |s| !s.usual? }; end
 
   def load_sources fn=Redwood::SOURCE_FN
     source_array = (Redwood::load_yaml_obj(fn) || []).map { |o| Recoverable.new o }

data/lib/sup/thread.rb

@@ -112,7 +112,7 @@ class Thread
 
   def set_labels l; each { |m, *o| m && m.labels = l }; end
   def has_label? t; any? { |m, *o| m && m.has_label?(t) }; end
-  def save_state index; each { |m, *o| m && m.save_state(index) }; end
+  def each_dirty_message; each { |m, *o| m && m.dirty? && yield(m) }; end
 
   def direct_participants
     map { |m, *o| [m.from] + m.to if m }.flatten.compact.uniq

data/lib/sup/util.rb

@@ -177,7 +177,7 @@ class String
   ## nasty multibyte hack for ruby 1.8. if it's utf-8, split into chars using
   ## the utf8 regex and count those. otherwise, use the byte length.
   def display_length
-    if $encoding == "UTF-8" || $encoding == "utf8"
+    if RUBY_VERSION < '1.9.1' && ($encoding == "UTF-8" || $encoding == "utf8")
       scan(/./u).size
     else
       size
@@ -290,12 +290,45 @@ class String
     end
   end
 
+  unless method_defined? :each
+    def each &b
+      each_line &b
+    end
+  end
+
   ## takes a list of words, and returns an array of symbols.  typically used in
   ## Sup for translating Ferret's representation of a list of labels (a string)
   ## to an array of label symbols.
   ##
   ## split_on will be passed to String#split, so you can leave this nil for space.
   def to_set_of_symbols split_on=nil; Set.new split(split_on).map { |x| x.strip.intern } end
+
+  class CheckError < ArgumentError; end
+  def check
+    begin
+      fail "unexpected encoding #{encoding}" if respond_to?(:encoding) && !(encoding == Encoding::UTF_8 || encoding == Encoding::ASCII)
+      fail "invalid encoding" if respond_to?(:valid_encoding?) && !valid_encoding?
+    rescue
+      raise CheckError.new($!.message)
+    end
+  end
+
+  def ascii
+    out = ""
+    each_byte do |b|
+      if (b & 128) != 0
+        out << "\\x#{b.to_s 16}"
+      else
+        out << b.chr
+      end
+    end
+    out.force_encoding Encoding::UTF_8 if out.respond_to? :force_encoding
+    out
+  end
+
+  def transcode src_encoding=$encoding
+    Iconv.easy_decode $encoding, src_encoding, self
+  end
 end
 
 class Numeric
@@ -486,9 +519,9 @@ class Time
       strftime "%b %e"
     else
       if is_the_same_day? from
-        strftime("%l:%M%P")
+        strftime("%l:%M%p").downcase # emulate %P (missing on ruby 1.8 darwin)
       elsif is_the_day_before? from
-        "Yest."  + nearest_hour.strftime("%l%P")
+        "Yest."  + nearest_hour.strftime("%l%p").downcase # emulate %P
       else
         strftime "%b %e"
       end
@@ -641,21 +674,26 @@ class FinishLine
 end
 
 class Iconv
-  def self.easy_decode target, charset, text
-    return text if charset =~ /^(x-unknown|unknown[-_ ]?8bit|ascii[-_ ]?7[-_ ]?bit)$/i
-    charset = case charset
+  def self.easy_decode target, orig_charset, text
+    if text.respond_to? :force_encoding
+      text = text.dup
+      text.force_encoding Encoding::BINARY
+    end
+    charset = case orig_charset
       when /UTF[-_ ]?8/i then "utf-8"
       when /(iso[-_ ])?latin[-_ ]?1$/i then "ISO-8859-1"
       when /iso[-_ ]?8859[-_ ]?15/i then 'ISO-8859-15'
       when /unicode[-_ ]1[-_ ]1[-_ ]utf[-_]7/i then "utf-7"
-      else charset
+      when /^euc$/i then 'EUC-JP' # XXX try them all?
+      when /^(x-unknown|unknown[-_ ]?8bit|ascii[-_ ]?7[-_ ]?bit)$/i then 'ASCII'
+      else orig_charset
     end
 
     begin
-      Iconv.iconv(target + "//IGNORE", charset, text + " ").join[0 .. -2]
-    rescue Errno::EINVAL, Iconv::InvalidEncoding, Iconv::InvalidCharacter, Iconv::IllegalSequence => e
-      warn "couldn't transcode text from #{charset} to #{target} (\"#{text[0 ... 20]}\"...) (got #{e.message}); using original as is"
-      text
+      returning(Iconv.iconv(target, charset, text + " ").join[0 .. -2]) { |str| str.check }
+    rescue Errno::EINVAL, Iconv::InvalidEncoding, Iconv::InvalidCharacter, Iconv::IllegalSequence, String::CheckError
+      debug "couldn't transcode text from #{orig_charset} (#{charset}) to #{target}) (#{text[0 ... 20].inspect}...) (got #{$!.message} (#{$!.class}))"
+      text.ascii
     end
   end
 end

data/lib/sup/xapian_index.rb

@@ -1,3 +1,5 @@
+ENV["XAPIAN_FLUSH_THRESHOLD"] = "1000"
+
 require 'xapian'
 require 'set'
 
@@ -42,14 +44,14 @@ EOS
       @xapian = Xapian::WritableDatabase.new(path, Xapian::DB_CREATE)
       @xapian.set_metadata 'version', INDEX_VERSION
     end
-    @term_generator = Xapian::TermGenerator.new()
-    @term_generator.stemmer = Xapian::Stem.new(STEM_LANGUAGE)
     @enquire = Xapian::Enquire.new @xapian
     @enquire.weighting_scheme = Xapian::BoolWeight.new
     @enquire.docid_order = Xapian::Enquire::ASCENDING
   end
 
   def save_index
+    info "Flushing Xapian updates to disk. This may take a while..."
+    @xapian.flush
   end
 
   def optimize
@@ -91,41 +93,9 @@ EOS
     m
   end
 
-  def add_message m; sync_message m end
-  def update_message m; sync_message m end
-  def update_message_state m; sync_message m end
-
-  def sync_message m, opts={}
-    entry = synchronize { get_entry m.id }
-    snippet = m.snippet
-    entry ||= {}
-    labels = m.labels
-    entry = {} if opts[:force_overwrite]
-
-    d = {
-      :message_id => m.id,
-      :source_id => m.source.id,
-      :source_info => m.source_info,
-      :date => (entry[:date] || m.date),
-      :snippet => snippet,
-      :labels => labels,
-      :from => (entry[:from] || [m.from.email, m.from.name]),
-      :to => (entry[:to] || m.to.map { |p| [p.email, p.name] }),
-      :cc => (entry[:cc] || m.cc.map { |p| [p.email, p.name] }),
-      :bcc => (entry[:bcc] || m.bcc.map { |p| [p.email, p.name] }),
-      :subject => m.subj,
-      :refs => (entry[:refs] || m.refs),
-      :replytos => (entry[:replytos] || m.replytos),
-    }
-
-    labels.each { |l| LabelManager << l }
-
-    synchronize do
-      index_message m, d, opts
-    end
-    true
-  end
-  private :sync_message
+  def add_message m; sync_message m, true end
+  def update_message m; sync_message m, true end
+  def update_message_state m; sync_message m, false end
 
   def num_results_for query={}
     xapian_query = build_xapian_query query
@@ -153,7 +123,6 @@ EOS
 
   def each_message_in_thread_for m, opts={}
     # TODO thread by subject
-    # TODO handle killed threads
     return unless doc = find_doc(m.id)
     queue = doc.value(THREAD_VALUENO).split(',')
     msgids = [m.id]
@@ -162,7 +131,7 @@ EOS
     while not queue.empty?
       thread_id = queue.pop
       next if seen_threads.member? thread_id
-      return false if thread_killed? thread_id
+      return false if opts[:skip_killed] && thread_killed?(thread_id)
       seen_threads << thread_id
       docs = term_docids(mkterm(:thread, thread_id)).map { |x| @xapian.document x }
       docs.each do |doc|
@@ -194,14 +163,15 @@ EOS
 
     subs = HookManager.run("custom-search", :subs => s) || s
     subs = subs.gsub(/\b(to|from):(\S+)\b/) do
-      field, name = $1, $2
-      if(p = ContactManager.contact_for(name))
-        [field, p.email]
-      elsif name == "me"
-        [field, "(" + AccountManager.user_emails.join("||") + ")"]
+      field, value = $1, $2
+      email_field, name_field = %w(email name).map { |x| "#{field}_#{x}" }
+      if(p = ContactManager.contact_for(value))
+        "#{email_field}:#{p.email}"
+      elsif value == "me"
+        '(' + AccountManager.user_emails.map { |e| "#{email_field}:#{e}" }.join(' OR ') + ')'
       else
-        [field, name]
-      end.join(":")
+        "(#{email_field}:#{value} OR #{name_field}:#{value})"
+      end
     end
 
     ## if we see a label:deleted or a label:spam term anywhere in the query
@@ -285,6 +255,8 @@ EOS
       end
     end
 
+    debug "translated query: #{subs.inspect}"
+
     qp = Xapian::QueryParser.new
     qp.database = @xapian
     qp.stemmer = Xapian::Stem.new(STEM_LANGUAGE)
@@ -295,6 +267,8 @@ EOS
     BOOLEAN_PREFIX.each { |k,v| qp.add_boolean_prefix k, v }
     xapian_query = qp.parse_query(subs, Xapian::QueryParser::FLAG_PHRASE|Xapian::QueryParser::FLAG_BOOLEAN|Xapian::QueryParser::FLAG_LOVEHATE|Xapian::QueryParser::FLAG_WILDCARD, PREFIX['body'])
 
+    debug "parsed xapian query: #{xapian_query.description}"
+
     raise ParseError if xapian_query.nil? or xapian_query.empty?
     query[:qobj] = xapian_query
     query[:text] = s
@@ -438,100 +412,140 @@ EOS
     end
   end
 
-  def index_message m, entry, opts
-    terms = []
-    text = []
+  def sync_message m, overwrite
+    doc = synchronize { find_doc(m.id) }
+    existed = doc != nil
+    doc ||= Xapian::Document.new
+    do_index_static = overwrite || !existed
+    old_entry = !do_index_static && doc.entry
+    snippet = do_index_static ? m.snippet : old_entry[:snippet]
 
-    subject_text = m.indexable_subject
-    body_text = m.indexable_body
+    entry = {
+      :message_id => m.id,
+      :source_id => m.source.id,
+      :source_info => m.source_info,
+      :date => m.date,
+      :snippet => snippet,
+      :labels => m.labels.to_a,
+      :from => [m.from.email, m.from.name],
+      :to => m.to.map { |p| [p.email, p.name] },
+      :cc => m.cc.map { |p| [p.email, p.name] },
+      :bcc => m.bcc.map { |p| [p.email, p.name] },
+      :subject => m.subj,
+      :refs => m.refs.to_a,
+      :replytos => m.replytos.to_a,
+    }
+
+    if do_index_static
+      doc.clear_terms
+      doc.clear_values
+      index_message_static m, doc, entry
+    end
+
+    index_message_threading doc, entry, old_entry
+    index_message_labels doc, entry[:labels], (do_index_static ? [] : old_entry[:labels])
+    doc.entry = entry
 
+    synchronize do
+      unless docid = existed ? doc.docid : assign_docid(m, truncate_date(m.date))
+        # Could be triggered by spam
+        warn "docid underflow, dropping #{m.id.inspect}"
+        return
+      end
+      @xapian.replace_document docid, doc
+    end
+
+    m.labels.each { |l| LabelManager << l }
+    true
+  end
+
+  ## Index content that can't be changed by the user
+  def index_message_static m, doc, entry
     # Person names are indexed with several prefixes
     person_termer = lambda do |d|
       lambda do |p|
         ["#{d}_name", "name", "body"].each do |x|
-          text << [p.name, PREFIX[x]]
+          doc.index_text p.name, PREFIX[x]
         end if p.name
-        [d, :any].each { |x| terms << mkterm(:email, x, p.email) }
+        [d, :any].each { |x| doc.add_term mkterm(:email, x, p.email) }
       end
     end
 
     person_termer[:from][m.from] if m.from
     (m.to+m.cc+m.bcc).each(&(person_termer[:to]))
 
-    terms << mkterm(:date,m.date) if m.date
-    m.labels.each { |t| terms << mkterm(:label,t) }
-    terms << mkterm(:type, 'mail')
-    terms << mkterm(:msgid, m.id)
-    terms << mkterm(:source_id, m.source.id)
+    # Full text search content
+    subject_text = m.indexable_subject
+    body_text = m.indexable_body
+    doc.index_text subject_text, PREFIX['subject']
+    doc.index_text subject_text, PREFIX['body']
+    doc.index_text body_text, PREFIX['body']
+    m.attachments.each { |a| doc.index_text a, PREFIX['attachment'] }
+
+    # Miscellaneous terms
+    doc.add_term mkterm(:date, m.date) if m.date
+    doc.add_term mkterm(:type, 'mail')
+    doc.add_term mkterm(:msgid, m.id)
+    doc.add_term mkterm(:source_id, m.source.id)
     m.attachments.each do |a|
       a =~ /\.(\w+)$/ or next
-      t = mkterm(:attachment_extension, $1)
-      terms << t
+      doc.add_term mkterm(:attachment_extension, $1)
     end
 
-    ## Thread membership
-    children = term_docids(mkterm(:ref, m.id)).map { |docid| @xapian.document docid }
-    parent_ids = m.refs + m.replytos
+    # Date value for range queries
+    date_value = begin
+      Xapian.sortable_serialise m.date.to_i
+    rescue TypeError
+      Xapian.sortable_serialise 0
+    end
+
+    doc.add_value MSGID_VALUENO, m.id
+    doc.add_value DATE_VALUENO, date_value
+  end
+
+  def index_message_labels doc, new_labels, old_labels
+    return if new_labels == old_labels
+    added = new_labels.to_a - old_labels.to_a
+    removed = old_labels.to_a - new_labels.to_a
+    added.each { |t| doc.add_term mkterm(:label,t) }
+    removed.each { |t| doc.remove_term mkterm(:label,t) }
+  end
+
+  ## Assign a set of thread ids to the document. This is a hybrid of the runtime
+  ## search done by the Ferret index and the index-time union done by previous
+  ## versions of the Xapian index. We first find the thread ids of all messages
+  ## with a reference to or from us. If that set is empty, we use our own
+  ## message id. Otherwise, we use all the thread ids we previously found. In
+  ## the common case there's only one member in that set, but if we're the
+  ## missing link between multiple previously unrelated threads we can have
+  ## more. XapianIndex#each_message_in_thread_for follows the thread ids when
+  ## searching so the user sees a single unified thread.
+  def index_message_threading doc, entry, old_entry
+    return if old_entry && (entry[:refs] == old_entry[:refs]) && (entry[:replytos] == old_entry[:replytos])
+    children = term_docids(mkterm(:ref, entry[:message_id])).map { |docid| @xapian.document docid }
+    parent_ids = entry[:refs] + entry[:replytos]
     parents = parent_ids.map { |id| find_doc id }.compact
     thread_members = SavingHash.new { [] }
     (children + parents).each do |doc2|
       thread_ids = doc2.value(THREAD_VALUENO).split ','
       thread_ids.each { |thread_id| thread_members[thread_id] << doc2 }
     end
+    thread_ids = thread_members.empty? ? [entry[:message_id]] : thread_members.keys
+    thread_ids.each { |thread_id| doc.add_term mkterm(:thread, thread_id) }
+    parent_ids.each { |ref| doc.add_term mkterm(:ref, ref) }
+    doc.add_value THREAD_VALUENO, (thread_ids * ',')
+  end
 
-    thread_ids = thread_members.empty? ? [m.id] : thread_members.keys
-
-    thread_ids.each { |thread_id| terms << mkterm(:thread, thread_id) }
-    parent_ids.each do |ref|
-      terms << mkterm(:ref, ref)
-    end
-
-    # Full text search content
-    text << [subject_text, PREFIX['subject']]
-    text << [subject_text, PREFIX['body']]
-    text << [body_text, PREFIX['body']]
-    m.attachments.each { |a| text << [a, PREFIX['attachment']] }
-
-    truncated_date = if m.date < MIN_DATE
-      debug "warning: adjusting too-low date #{m.date} for indexing"
+  def truncate_date date
+    if date < MIN_DATE
+      debug "warning: adjusting too-low date #{date} for indexing"
       MIN_DATE
-    elsif m.date > MAX_DATE
-      debug "warning: adjusting too-high date #{m.date} for indexing"
+    elsif date > MAX_DATE
+      debug "warning: adjusting too-high date #{date} for indexing"
       MAX_DATE
     else
-      m.date
+      date
     end
-
-    # Date value for range queries
-    date_value = begin
-      Xapian.sortable_serialise truncated_date.to_i
-    rescue TypeError
-      Xapian.sortable_serialise 0
-    end
-
-    docid = nil
-    unless doc = find_doc(m.id)
-      doc = Xapian::Document.new
-      if not docid = assign_docid(m, truncated_date)
-        # Could be triggered by spam
-        Redwood::log "warning: docid underflow, dropping #{m.id.inspect}"
-        return
-      end
-    else
-      doc.clear_terms
-      doc.clear_values
-      docid = doc.docid
-    end
-
-    @term_generator.document = doc
-    text.each { |text,prefix| @term_generator.index_text text, 1, prefix }
-    terms.each { |term| doc.add_term term if term.length <= MAX_TERM_LENGTH }
-    doc.add_value MSGID_VALUENO, m.id
-    doc.add_value THREAD_VALUENO, (thread_ids * ',')
-    doc.add_value DATE_VALUENO, date_value
-    doc.data = Marshal.dump entry
-
-    @xapian.replace_document docid, doc
   end
 
   # Construct a Xapian term
@@ -560,7 +574,32 @@ EOS
       raise "Invalid term type #{type}"
     end
   end
+end
 
 end
 
+class Xapian::Document
+  def entry
+    Marshal.load data
+  end
+
+  def entry=(x)
+    self.data = Marshal.dump x
+  end
+
+  def index_text text, prefix, weight=1
+    term_generator = Xapian::TermGenerator.new
+    term_generator.stemmer = Xapian::Stem.new(Redwood::XapianIndex::STEM_LANGUAGE)
+    term_generator.document = self
+    term_generator.index_text text, weight, prefix
+  end
+
+  alias old_add_term add_term
+  def add_term term
+    if term.length <= Redwood::XapianIndex::MAX_TERM_LENGTH
+      old_add_term term, 0
+    else
+      warn "dropping excessively long term #{term}"
+    end
+  end
 end