sup 0.6 → 0.7

data/CONTRIBUTORS

@@ -1,13 +1,17 @@
 William Morgan <wmorgan-sup at the masanjin dot nets>
 Ismo Puustinen <ismo at the iki dot fis>
+Nicolas Pouillard <nicolas.pouillard at the gmail dot coms>
 Marcus Williams <marcus-sup at the bar-coded dot nets>
 Lionel Ott <white.magic at the gmx dot des>
-Nicolas Pouillard <nicolas.pouillard at the gmail dot coms>
+Mark Alexander <marka at the pobox dot coms>
+Christopher Warrington <chrisw at the rice dot edus>
 Marc Hartstein <marc.hartstein at the alum.vassar dot edus>
 Ben Walton <bwalton at the artsci.utoronto dot cas>
 Grant Hollingworth <grant at the antiflux dot orgs>
+Steve Goldman <sgoldman at the tower-research dot coms>
+Decklin Foster <decklin at the red-bean dot coms>
 Jeff Balogh <its.jeff.balogh at the gmail dot coms>
-Christopher Warrington <chrisw at the rice dot edus>
 Giorgio Lando <patroclo7 at the gmail dot coms>
-Decklin Foster <decklin at the red-bean dot coms>
+Israel Herraiz <israel.herraiz at the gmail dot coms>
 Ian Taylor <ian at the lorf dot orgs>
+Rich Lane <rlane at the club.cc.cmu dot edus>

data/History.txt

@@ -1,3 +1,15 @@
+== 0.7 / 2009-03-16
+* Ferret index corruption issues fixed (hopefully!)
+* Text entry now scrolls to the right on overflow, i.e. is actually usable
+* Ctrl-C now asks user if Sup should die ungracefully
+* Add a limit:<int> search operator to limit the number of results
+* Added a --query option to sup-tweak-labels
+* Added a new hook: shutdown
+* Automatically add self as recipient on crypted sent messages
+* Read in X-Foo headers
+* Added global keybinding 'U' shows only unread messages
+* As always, many bugfixes and tweaks
+
 == 0.6 / 2008-08-04
 * new hooks: mark-as-spam, reply-to, reply-from
 * configurable colors. finally!

data/README.txt

@@ -108,7 +108,7 @@ Current limitations which will be fixed:
 
 == INSTALL:
 
-* gem install sup -y
+* gem install sup
 
 == PROBLEMS:
 
@@ -116,7 +116,7 @@ See FAQ.txt for some common problems and their solutions.
 
 == LICENSE:
 
-Copyright (c) 2006, 2007 William Morgan.
+Copyright (c) 2006--2009 William Morgan.
 
 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License

data/ReleaseNotes

@@ -1,3 +1,14 @@
+Release 0.7:
+
+The big win in this release is that Ferret index corruption issues should now
+be fixed, thanks to an extensive programming of locking and
+thread-safety-adding.
+
+The other nice change is that text entry will now scroll to the right upon
+overflow, thanks to some arcane Curses magic that Steve Goldman discovered.
+
+As always, this release includes many other bugfixes and enhancements.
+
 Release 0.6:
 
 Message attachment searchability automatically takes effect on new messages,

data/bin/sup

@@ -8,7 +8,7 @@ require 'trollop'
 require 'fastthread'
 require "sup"
 
-BIN_VERSION = "0.6"
+BIN_VERSION = "0.7"
 
 unless Redwood::VERSION == BIN_VERSION
   $stderr.puts <<EOS
@@ -45,6 +45,14 @@ No variables.
 No return value.
 EOS
 
+Redwood::HookManager.register "shutdown", <<EOS 
+Executes when sup is shutting down. May be run when sup is crashing,
+so don\'t do anything too important. Run before the label, contacts,
+and people are saved.
+No variables.
+No return value.
+EOS
+
 if $opts[:list_hooks]
   Redwood::HookManager.print_hooks
   exit
@@ -57,7 +65,7 @@ module Redwood
 global_keymap = Keymap.new do |k|
   k.add :quit_ask, "Quit Sup, but ask first", 'q'
   k.add :quit_now, "Quit Sup immediately", 'Q'
-  k.add :help, "Show help", 'H', '?'
+  k.add :help, "Show help", '?'
   k.add :roll_buffers, "Switch to next buffer", 'b'
 #  k.add :roll_buffers_backwards, "Switch to previous buffer", 'B'
   k.add :kill_buffer, "Kill the current buffer", 'x'
@@ -65,6 +73,7 @@ global_keymap = Keymap.new do |k|
   k.add :list_contacts, "List contacts", 'C'
   k.add :redraw, "Redraw screen", :ctrl_l
   k.add :search, "Search all messages", '\\', 'F'
+  k.add :search_unread, "Show all unread messages", 'U'
   k.add :list_labels, "List labels", 'L'
   k.add :poll, "Poll for new messages", 'P'
   k.add :compose, "Compose new message", 'm', 'c'
@@ -101,7 +110,7 @@ rescue Index::LockError => e
   h.say Index.fancy_lock_error_message_for(e)
 
   case h.ask("Should I ask that process to kill itself? ")
-  when /^\s*y\s*$/i
+  when /^\s*y(es)?\s*$/i
     h.say "Ok, suggesting seppuku..."
     FileUtils.touch Redwood::SUICIDE_FN
     sleep SuicideManager::DELAY * 2
@@ -180,7 +189,16 @@ begin
   end
 
   until Redwood::exceptions.nonempty? || SuicideManager.die?
-    c = Ncurses.nonblocking_getch
+    c = 
+       begin
+         Ncurses.nonblocking_getch
+       rescue Exception => e
+         if e.is_a?(Interrupt)
+           raise if BufferManager.ask_yes_or_no("Die ungracefully now?")
+           bm.draw_screen
+           nil
+         end
+       end
     next unless c
     bm.erase_flash
 
@@ -194,7 +212,6 @@ begin
       rescue InputSequenceAborted
         :nothing
       end
-
     case action
     when :quit_now
       break if bm.kill_all_buffers_safely
@@ -220,6 +237,8 @@ begin
       query = BufferManager.ask :search, "search all messages: "
       next unless query && query !~ /^\s*$/
       SearchResultsMode.spawn_from_query query
+    when :search_unread
+      SearchResultsMode.spawn_from_query "is:unread"
     when :list_labels
       labels = LabelManager.listable_labels.map { |l| LabelManager.string_for l }
       user_label = bm.ask_with_completions :label, "Show threads with label (enter for listing): ", labels
@@ -268,6 +287,8 @@ ensure
     Index.stop_lock_update_thread
   end
 
+  HookManager.run "shutdown"
+
   Redwood::finish
   stop_cursing
   Redwood::log "stopped cursing"

data/bin/sup-tweak-labels

@@ -39,6 +39,7 @@ Options:
 EOS
   opt :add, "One or more labels (comma-separated) to add to every message from the specified sources", :type => String
   opt :remove, "One or more labels (comma-separated) to remove from every message from the specified sources, if those labels are present", :type => String
+  opt :query, "A Sup search query", :type => String
 
   text <<EOS
 
@@ -76,6 +77,10 @@ begin
     ## query to only messages with those labels
     query += " +(" + remove_labels.map { |l| "label:#{l}" }.join(" ") + ")"
   end
+  query += ' ' + opts[:query] if opts[:query]
+
+  qobj, opts = Redwood::Index.parse_user_query_string query
+  query = Redwood::Index.build_query opts.merge(:qobj => qobj)
 
   results = index.ferret.search query, :limit => :all
   num_total = results.total_hits

data/lib/sup.rb

@@ -46,7 +46,7 @@ class Module
 end
 
 module Redwood
-  VERSION = "0.6"
+  VERSION = "0.7"
 
   BASE_DIR   = ENV["SUP_BASE"] || File.join(ENV["HOME"], ".sup")
   CONFIG_FN  = File.join(BASE_DIR, "config.yaml")
@@ -221,6 +221,7 @@ else
     :confirm_no_attachments => true,
     :confirm_top_posting => true,
     :discard_snippets_from_encrypted_messages => false,
+    :default_attachment_save_dir => "",
   }
   begin
     FileUtils.mkdir_p Redwood::BASE_DIR

data/lib/sup/buffer.rb

@@ -433,7 +433,7 @@ EOS
       prefix, target = partial.split_on_commas_with_remainder
       target ||= prefix.pop || ""
       prefix = prefix.join(", ") + (prefix.empty? ? "" : ", ")
-      completions.select { |x| x =~ /^#{Regexp::escape target}/i }.map { |x| [prefix + x, x] }
+      completions.select { |x| x =~ /^#{Regexp::escape target}/i }.sort_by { |c| [ContactManager.contact_for(c) ? 0 : 1, c] }.map { |x| [prefix + x, x] }
     end
   end
 
@@ -501,12 +501,12 @@ EOS
     recent = Index.load_contacts(AccountManager.user_emails, :num => 10).map { |c| [c.full_address, c.email] }
     contacts = ContactManager.contacts.map { |c| [ContactManager.alias_for(c), c.full_address, c.email] }
 
-    completions = (recent + contacts).flatten.uniq.sort
+    completions = (recent + contacts).flatten.uniq
     completions += HookManager.run("extra-contact-addresses") || []
     answer = BufferManager.ask_many_emails_with_completions domain, question, completions, default
 
     if answer
-      answer.split_on_commas.map { |x| ContactManager.contact_for(x.downcase) || PersonManager.person_for(x) }
+      answer.split_on_commas.map { |x| ContactManager.contact_for(x) || PersonManager.person_for(x) }
     end
   end
 

data/lib/sup/crypto.rb

@@ -53,7 +53,7 @@ class CryptoManager
     payload_fn.write format_payload(payload)
     payload_fn.close
 
-    recipient_opts = to.map { |r| "--recipient '<#{r}>'" }.join(" ")
+    recipient_opts = (to + [ from ] ).map { |r| "--recipient '<#{r}>'" }.join(" ")
     sign_opts = sign ? "--sign --local-user '#{from}'" : ""
     gpg_output = run_gpg "--output - --armor --encrypt --textmode #{sign_opts} #{recipient_opts} #{payload_fn.path}"
     raise Error, (gpg_output || "gpg command failed: #{cmd}") unless $?.success?

data/lib/sup/draft.rb

@@ -32,7 +32,10 @@ class DraftManager
 
   def discard m
     docid, entry = Index.load_entry_for_id m.id
-    raise ArgumentError, "can't find entry for draft: #{m.id.inspect}" unless entry
+    unless entry
+      Redwood::log "can't find entry for draft: #{m.id.inspect}. You probably already discarded it."
+      return
+    end
     raise ArgumentError, "not a draft: source id #{entry[:source_id].inspect}, should be #{DraftManager.source_id.inspect} for #{m.id.inspect} / docno #{docid}" unless entry[:source_id].to_i == DraftManager.source_id
     Index.drop_entry docid
     File.delete @source.fn_for_offset(entry[:source_info])

data/lib/sup/hook.rb

@@ -52,6 +52,14 @@ class HookManager
       end
     end
 
+    def get tag
+      HookManager.tags[tag]
+    end
+
+    def set tag, value
+      HookManager.tags[tag] = value
+    end
+
     def __binding 
       binding
     end
@@ -68,12 +76,15 @@ class HookManager
     @hooks = {}
     @descs = {}
     @contexts = {}
-    
+    @tags = {}
+
     Dir.mkdir dir unless File.exists? dir
 
     self.class.i_am_the_instance self
   end
 
+  attr_reader :tags
+
   def run name, locals={}
     hook = hook_for(name) or return
     context = @contexts[hook] ||= HookContext.new(name)

data/lib/sup/imap.rb

@@ -5,6 +5,9 @@ require 'time'
 require 'rmail'
 require 'cgi'
 
+## TODO: remove synchronized method protector calls; use a Monitor instead
+## (ruby's reentrant mutex)
+
 ## fucking imap fucking sucks. what the FUCK kind of committee of dunces
 ## designed this shit.
 ##
@@ -15,7 +18,7 @@ require 'cgi'
 ## restriction. it can change any time you log in. it can change EVERY
 ## time you log in. of course the imap spec "strongly recommends" that it
 ## never change, but there's nothing to stop people from just setting it
-## to the current timestamp, and in fact that's exactly what the one imap
+## to the current timestamp, and in fact that's EXACTLY what the one imap
 ## server i have at my disposal does. thus the so-called uids are
 ## absolutely useless and imap provides no cross-session way of uniquely
 ## identifying a message. but thanks for the "strong recommendation",
@@ -114,20 +117,37 @@ class IMAP < Source
   end
   synchronized :raw_message
 
+  def mark_as_deleted ids
+    ids = [ids].flatten # accept single arguments
+    unsynchronized_scan_mailbox
+    imap_ids = ids.map { |i| @imap_state[i] && @imap_state[i][:id] }.compact
+    return if imap_ids.empty?
+    @imap.store imap_ids, "+FLAGS", [:Deleted]
+  end
+  synchronized :mark_as_deleted
+
+  def expunge
+    @imap.expunge
+    unsynchronized_scan_mailbox true
+    true
+  end
+  synchronized :expunge
+
   def connect
     return if @imap
     safely { } # do nothing!
   end
   synchronized :connect
 
-  def scan_mailbox
-    return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
+  def scan_mailbox force=false
+    return if !force && @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
     last_id = safely do
       @imap.examine mailbox
       @imap.responses["EXISTS"].last
     end
     @last_scan = Time.now
 
+    @ids = [] if force
     return if last_id == @ids.length
 
     range = (@ids.length + 1) .. last_id
@@ -259,7 +279,7 @@ private
     %w(RFC822.SIZE INTERNALDATE).each do |w|
       raise FatalSourceError, "requested data not in IMAP response: #{w}" unless imap_stuff.attr[w]
     end
-    
+
     msize, mdate = imap_stuff.attr['RFC822.SIZE'] % 10000000, Time.parse(imap_stuff.attr["INTERNALDATE"])
     sprintf("%d%07d", mdate.to_i, msize).to_i
   end

data/lib/sup/index.rb

@@ -2,6 +2,8 @@
 
 require 'fileutils'
 require 'ferret'
+require 'fastthread'
+
 begin
   require 'chronic'
   $have_chronic = true
@@ -23,12 +25,18 @@ class Index
 
   include Singleton
 
+  ## these two accessors should ONLY be used by single-threaded programs.
+  ## otherwise you will have a naughty ferret on your hands.
   attr_reader :index
   alias ferret index
+
   def initialize dir=BASE_DIR
+    @index_mutex = Monitor.new
+
     @dir = dir
     @sources = {}
     @sources_dirty = false
+    @source_mutex = Monitor.new
 
     wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false
     sa = Ferret::Analysis::StandardAnalyzer.new [], true
@@ -66,14 +74,19 @@ class Index
     @lock_update_thread = nil
   end
 
+  def possibly_pluralize number_of, kind
+    "#{number_of} #{kind}" +
+        if number_of == 1 then "" else "s" end
+  end
+
   def fancy_lock_error_message_for e
-    secs = Time.now - e.mtime
-    mins = secs.to_i / 60
+    secs = (Time.now - e.mtime).to_i
+    mins = secs / 60
     time =
       if mins == 0
-        "#{secs.to_i} seconds"
+        possibly_pluralize secs , "second"
       else
-        "#{mins} minutes"
+        possibly_pluralize mins, "minute"
       end
 
     <<EOS
@@ -117,17 +130,19 @@ EOS
   end
 
   def add_source source
-    raise "duplicate source!" if @sources.include? source
-    @sources_dirty = true
-    max = @sources.max_of { |id, s| s.is_a?(DraftLoader) || s.is_a?(SentLoader) ? 0 : id }
-    source.id ||= (max || 0) + 1
-    ##source.id += 1 while @sources.member? source.id
-    @sources[source.id] = source
+    @source_mutex.synchronize do
+      raise "duplicate source!" if @sources.include? source
+      @sources_dirty = true
+      max = @sources.max_of { |id, s| s.is_a?(DraftLoader) || s.is_a?(SentLoader) ? 0 : id }
+      source.id ||= (max || 0) + 1
+      ##source.id += 1 while @sources.member? source.id
+      @sources[source.id] = source
+    end
   end
 
   def sources
     ## favour the inbox by listing non-archived sources first
-    @sources.values.sort_by { |s| s.id }.partition { |s| !s.archived? }.flatten
+    @source_mutex.synchronize { @sources.values }.sort_by { |s| s.id }.partition { |s| !s.archived? }.flatten
   end
 
   def source_for uri; sources.find { |s| s.is_source_for? uri }; end
@@ -136,25 +151,29 @@ EOS
   def load_index dir=File.join(@dir, "ferret")
     if File.exists? dir
       Redwood::log "loading index..."
-      @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
-      Redwood::log "loaded index of #{@index.size} messages"
+      @index_mutex.synchronize do
+        @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
+        Redwood::log "loaded index of #{@index.size} messages"
+      end
     else
       Redwood::log "creating index..."
-      field_infos = Ferret::Index::FieldInfos.new :store => :yes
-      field_infos.add_field :message_id, :index => :untokenized
-      field_infos.add_field :source_id
-      field_infos.add_field :source_info
-      field_infos.add_field :date, :index => :untokenized
-      field_infos.add_field :body
-      field_infos.add_field :label
-      field_infos.add_field :attachments
-      field_infos.add_field :subject
-      field_infos.add_field :from
-      field_infos.add_field :to
-      field_infos.add_field :refs
-      field_infos.add_field :snippet, :index => :no, :term_vector => :no
-      field_infos.create_index dir
-      @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
+      @index_mutex.synchronize do
+        field_infos = Ferret::Index::FieldInfos.new :store => :yes
+        field_infos.add_field :message_id, :index => :untokenized
+        field_infos.add_field :source_id
+        field_infos.add_field :source_info
+        field_infos.add_field :date, :index => :untokenized
+        field_infos.add_field :body
+        field_infos.add_field :label
+        field_infos.add_field :attachments
+        field_infos.add_field :subject
+        field_infos.add_field :from
+        field_infos.add_field :to
+        field_infos.add_field :refs
+        field_infos.add_field :snippet, :index => :no, :term_vector => :no
+        field_infos.create_index dir
+        @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
+      end
     end
   end
 
@@ -166,7 +185,9 @@ EOS
     docid, entry = load_entry_for_id m.id unless docid && entry
 
     raise "no source info for message #{m.id}" unless m.source && m.source_info
-    raise "trying to delete non-corresponding entry #{docid} with index message-id #{@index[docid][:message_id].inspect} and parameter message id #{m.id.inspect}" if docid && @index[docid][:message_id] != m.id
+    @index_mutex.synchronize do
+      raise "trying to delete non-corresponding entry #{docid} with index message-id #{@index[docid][:message_id].inspect} and parameter message id #{m.id.inspect}" if docid && @index[docid][:message_id] != m.id
+    end
 
     source_id = 
       if m.source.is_a? Integer
@@ -231,9 +252,11 @@ EOS
       :refs => (entry[:refs] || (m.refs + m.replytos).uniq.join(" ")),
     }
 
-    @index.delete docid if docid
-    @index.add_document d
-    
+    @index_mutex.synchronize  do
+      @index.delete docid if docid
+      @index.add_document d
+    end
+
     docid, entry = load_entry_for_id m.id
     ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
     raise "just added message #{m.id.inspect} but couldn't find it in a search" unless docid
@@ -245,32 +268,37 @@ EOS
   end
 
   def contains_id? id
-    @index.search(Ferret::Search::TermQuery.new(:message_id, id)).total_hits > 0
+    @index_mutex.synchronize { @index.search(Ferret::Search::TermQuery.new(:message_id, id)).total_hits > 0 }
   end
-  def contains? m; contains_id? m.id; end
-  def size; @index.size; end
+  def contains? m; contains_id? m.id end
+  def size; @index_mutex.synchronize { @index.size } end
+  def empty?; size == 0 end
 
   ## you should probably not call this on a block that doesn't break
   ## rather quickly because the results can be very large.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
-    return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
+    return if empty? # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
     query = build_query opts
     offset = 0
     while true
-      results = @index.search(query, :sort => "date DESC", :limit => EACH_BY_DATE_NUM, :offset => offset)
+      limit = (opts[:limit])? [EACH_BY_DATE_NUM, opts[:limit] - offset].min : EACH_BY_DATE_NUM
+      results = @index_mutex.synchronize { @index.search query, :sort => "date DESC", :limit => limit, :offset => offset }
       Redwood::log "got #{results.total_hits} results for query (offset #{offset}) #{query.inspect}"
-      results.hits.each { |hit| yield @index[hit.doc][:message_id], lambda { build_message hit.doc } }
-      break if offset >= results.total_hits - EACH_BY_DATE_NUM
-      offset += EACH_BY_DATE_NUM
+      results.hits.each do |hit|
+        yield @index_mutex.synchronize { @index[hit.doc][:message_id] }, lambda { build_message hit.doc }
+      end
+      break if opts[:limit] and offset >= opts[:limit] - limit
+      break if offset >= results.total_hits - limit
+      offset += limit
     end
   end
 
   def num_results_for opts={}
-    return 0 if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
+    return 0 if empty? # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
 
     q = build_query opts
-    index.search(q, :limit => 1).total_hits
+    @index_mutex.synchronize { @index.search(q, :limit => 1).total_hits }
   end
 
   ## yield all messages in the thread containing 'm' by repeatedly
@@ -304,10 +332,10 @@ EOS
 
       q = build_query :qobj => q
 
-      p1 = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
+      p1 = @index_mutex.synchronize { @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] } }
       Redwood::log "found #{p1.size} results for subject query #{q}"
 
-      p2 = @index.search(q.to_s, :limit => :all).hits.map { |hit| @index[hit.doc][:message_id] }
+      p2 = @index_mutex.synchronize { @index.search(q.to_s, :limit => :all).hits.map { |hit| @index[hit.doc][:message_id] } }
       Redwood::log "found #{p2.size} results in string form"
 
       pending = (pending + p1 + p2).uniq
@@ -330,18 +358,20 @@ EOS
 
       num_queries += 1
       killed = false
-      @index.search_each(q, :limit => :all) do |docid, score|
-        break if opts[:limit] && messages.size >= opts[:limit]
-        if @index[docid][:label].split(/\s+/).include?("killed") && opts[:skip_killed]
-          killed = true
-          break
-        end
-        mid = @index[docid][:message_id]
-        unless messages.member?(mid)
-          #Redwood::log "got #{mid} as a child of #{id}"
-          messages[mid] ||= lambda { build_message docid }
-          refs = @index[docid][:refs].split(" ")
-          pending += refs.select { |id| !searched[id] }
+      @index_mutex.synchronize do
+        @index.search_each(q, :limit => :all) do |docid, score|
+          break if opts[:limit] && messages.size >= opts[:limit]
+          if @index[docid][:label].split(/\s+/).include?("killed") && opts[:skip_killed]
+            killed = true
+            break
+          end
+          mid = @index[docid][:message_id]
+          unless messages.member?(mid)
+            #Redwood::log "got #{mid} as a child of #{id}"
+            messages[mid] ||= lambda { build_message docid }
+            refs = @index[docid][:refs].split(" ")
+            pending += refs.select { |id| !searched[id] }
+          end
         end
       end
     end
@@ -358,36 +388,44 @@ EOS
 
   ## builds a message object from a ferret result
   def build_message docid
-    doc = @index[docid]
-    source = @sources[doc[:source_id].to_i]
-    #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})"
-    raise "invalid source #{doc[:source_id]}" unless source
-
-    fake_header = {
-      "date" => Time.at(doc[:date].to_i),
-      "subject" => unwrap_subj(doc[:subject]),
-      "from" => doc[:from],
-      "to" => doc[:to].split(/\s+/).join(", "), # reformat
-      "message-id" => doc[:message_id],
-      "references" => doc[:refs].split(/\s+/).map { |x| "<#{x}>" }.join(" "),
-    }
-
-    Message.new :source => source, :source_info => doc[:source_info].to_i, 
-                :labels => doc[:label].split(" ").map { |s| s.intern },
-                :snippet => doc[:snippet], :header => fake_header
+    @index_mutex.synchronize do
+      doc = @index[docid]
+
+      source = @source_mutex.synchronize { @sources[doc[:source_id].to_i] }
+      raise "invalid source #{doc[:source_id]}" unless source
+
+      #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})"
+
+      fake_header = {
+        "date" => Time.at(doc[:date].to_i),
+        "subject" => unwrap_subj(doc[:subject]),
+        "from" => doc[:from],
+        "to" => doc[:to].split(/\s+/).join(", "), # reformat
+        "message-id" => doc[:message_id],
+        "references" => doc[:refs].split(/\s+/).map { |x| "<#{x}>" }.join(" "),
+      }
+
+      Message.new :source => source, :source_info => doc[:source_info].to_i,
+                  :labels => doc[:label].split(" ").map { |s| s.intern },
+                  :snippet => doc[:snippet], :header => fake_header
+    end
   end
 
   def fresh_thread_id; @next_thread_id += 1; end
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
   def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
-  def drop_entry docno; @index.delete docno; end
+  def drop_entry docno; @index_mutex.synchronize { @index.delete docno } end
 
   def load_entry_for_id mid
-    results = @index.search(Ferret::Search::TermQuery.new(:message_id, mid))
-    return if results.total_hits == 0
-    docid = results.hits[0].doc
-    [docid, @index[docid]]
+    @index_mutex.synchronize do
+      results = @index.search Ferret::Search::TermQuery.new(:message_id, mid)
+      return if results.total_hits == 0
+      docid = results.hits[0].doc
+      entry = @index[docid]
+      entry_dup = entry.fields.inject({}) { |h, f| h[f] = entry[f]; h }
+      [docid, entry_dup]
+    end
   end
 
   def load_contacts emails, h={}
@@ -403,16 +441,18 @@ EOS
     Redwood::log "contact search: #{q}"
     contacts = {}
     num = h[:num] || 20
-    @index.search_each(q, :sort => "date DESC", :limit => :all) do |docid, score|
-      break if contacts.size >= num
-      #Redwood::log "got message #{docid} to: #{@index[docid][:to].inspect} and from: #{@index[docid][:from].inspect}"
-      f = @index[docid][:from]
-      t = @index[docid][:to]
-
-      if AccountManager.is_account_email? f
-        t.split(" ").each { |e| contacts[PersonManager.person_for(e)] = true }
-      else
-        contacts[PersonManager.person_for(f)] = true
+    @index_mutex.synchronize do
+      @index.search_each q, :sort => "date DESC", :limit => :all do |docid, score|
+        break if contacts.size >= num
+        #Redwood::log "got message #{docid} to: #{@index[docid][:to].inspect} and from: #{@index[docid][:from].inspect}"
+        f = @index[docid][:from]
+        t = @index[docid][:to]
+
+        if AccountManager.is_account_email? f
+          t.split(" ").each { |e| contacts[PersonManager.person_for(e)] = true }
+        else
+          contacts[PersonManager.person_for(f)] = true
+        end
       end
     end
 
@@ -421,15 +461,17 @@ EOS
 
   def load_sources fn=Redwood::SOURCE_FN
     source_array = (Redwood::load_yaml_obj(fn) || []).map { |o| Recoverable.new o }
-    @sources = Hash[*(source_array).map { |s| [s.id, s] }.flatten]
-    @sources_dirty = false
+    @source_mutex.synchronize do
+      @sources = Hash[*(source_array).map { |s| [s.id, s] }.flatten]
+      @sources_dirty = false
+    end
   end
 
   def has_any_from_source_with_label? source, label
     q = Ferret::Search::BooleanQuery.new
     q.add_query Ferret::Search::TermQuery.new("source_id", source.id.to_s), :must
     q.add_query Ferret::Search::TermQuery.new("label", label.to_s), :must
-    index.search(q, :limit => 1).total_hits > 0
+    @index_mutex.synchronize { @index.search(q, :limit => 1).total_hits > 0 }
   end
 
 protected
@@ -521,6 +563,18 @@ protected
       end
       subs = nil if chronic_failure
     end
+
+    ## limit:42 restrict the search to 42 results
+    subs = subs.gsub(/\blimit:(\S+)\b/) do
+      lim = $1
+      if lim =~ /^\d+$/
+        extraopts[:limit] = lim.to_i
+        ''
+      else
+        BufferManager.flash "Can't understand limit #{lim.inspect}!"
+        subs = nil
+      end
+    end
     
     if subs
       [@qparser.parse(subs), extraopts]
@@ -550,16 +604,18 @@ protected
   end
 
   def save_sources fn=Redwood::SOURCE_FN
-    if @sources_dirty || @sources.any? { |id, s| s.dirty? }
-      bakfn = fn + ".bak"
-      if File.exists? fn
+    @source_mutex.synchronize do
+      if @sources_dirty || @sources.any? { |id, s| s.dirty? }
+        bakfn = fn + ".bak"
+        if File.exists? fn
+          File.chmod 0600, fn
+          FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(fn) == 0
+        end
+        Redwood::save_yaml_obj sources.sort_by { |s| s.id.to_i }, fn, true
         File.chmod 0600, fn
-        FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(fn) == 0
       end
-      Redwood::save_yaml_obj sources.sort_by { |s| s.id.to_i }, fn, true
-      File.chmod 0600, fn
+      @sources_dirty = false
     end
-    @sources_dirty = false
   end
 end