sup 0.4 → 0.5

data/History.txt

@@ -1,3 +1,11 @@
+== 0.5 / 2008-04-22
+* new hooks: extra-contact-addresses, startup
+* '!!' now loads all threads in current search
+* general state saving speedup
+* threads with unsent draft messages are now shown in red
+* --compose spawns a compose-message buffer on startup
+* Many bugfixes and UI improvements
+
 == 0.4 / 2008-01-23
 * GPG support for signing and encrypting outgoing mail
 * New hooks: mime attachment, attribution line

data/Manifest.txt

@@ -4,6 +4,7 @@ LICENSE
 Manifest.txt
 README.txt
 Rakefile
+ReleaseNotes
 bin/sup
 bin/sup-add
 bin/sup-config

data/Rakefile

@@ -9,9 +9,13 @@ end # thanks to "Mike H"
 
 ## allow people who use development versions by running "rake gem"
 ## and installing the resulting gem it to be able to do this. (gem
-## versions must be in dotted-digit notation only).
-version = Redwood::VERSION == "git" ? "999" : Redwood::VERSION
-
+## versions must be in dotted-digit notation only and can be passed
+## with the REL environment variable to "rake gem").
+if ENV['REL']
+  version = ENV['REL']
+else
+  version = Redwood::VERSION == "git" ? "999" : Redwood::VERSION
+end
 Hoe.new('sup', version) do |p|
   p.rubyforge_name = 'sup'
   p.author = "William Morgan"
@@ -50,3 +54,7 @@ end
 
 # vim: syntax=ruby
 # -*- ruby -*-
+task :upload_report do |t|
+  sh "ditz html ditz"
+  sh "rsync -essh -cavz ditz wmorgan@rubyforge.org:/var/www/gforge-projects/sup/"
+end

data/ReleaseNotes

@@ -0,0 +1,6 @@
+Release 0.5:
+
+Saving message state (pressing "$") has been sped up. However, this is only
+automatically in effect for new messages. To make it effective for older
+messages (i.e. messages indexed with versions of Sup before 0.5), you must
+reindex them, e.g. by running sup-sync --all on a source.

data/bin/sup

@@ -7,7 +7,7 @@ require 'fileutils'
 require 'trollop'
 require "sup"
 
-BIN_VERSION = "0.4"
+BIN_VERSION = "0.5"
 
 unless Redwood::VERSION == BIN_VERSION
   $stderr.puts <<EOS
@@ -32,12 +32,19 @@ Usage:
 
 Options are:
 EOS
-  opt :list_hooks, "List all hooks and descriptions thereof, and quit."
-  opt :no_threads, "Turn of threading. Helps with debugging. (Necessarily disables background polling for new messages.)"
+  opt :list_hooks, "List all hooks and descriptions, and quit."
+  opt :no_threads, "Turn off threading. Helps with debugging. (Necessarily disables background polling for new messages.)"
   opt :no_initial_poll, "Don't poll for new messages when starting."
-  opt :search, "Search for threads ", :type => String
+  opt :search, "Search for this query upon startup", :type => String
+  opt :compose, "Compose message to this recipient upon startup", :type => String
 end
 
+Redwood::HookManager.register "startup", <<EOS
+Executes at startup
+No variables.
+No return value.
+EOS
+
 if $opts[:list_hooks]
   Redwood::HookManager.print_hooks
   exit
@@ -94,7 +101,7 @@ rescue Index::LockError => e
 
   case h.ask("Should I ask that process to kill itself? ")
   when /^\s*y\s*$/i
-    h.say "Ok, suggesting sepuku..."
+    h.say "Ok, suggesting seppuku..."
     FileUtils.touch Redwood::SUICIDE_FN
     sleep SuicideManager::DELAY * 2
     FileUtils.rm_f Redwood::SUICIDE_FN
@@ -127,6 +134,8 @@ begin
     Index.add_source SentManager.new_source
   end
 
+  HookManager.run "startup"
+
   log "starting curses"
   start_cursing
 
@@ -137,6 +146,8 @@ begin
            Ncurses::A_BOLD
     c.add :index_starred_color, Ncurses::COLOR_YELLOW, Ncurses::COLOR_BLACK, 
            Ncurses::A_BOLD
+    c.add :index_draft_color, Ncurses::COLOR_RED, Ncurses::COLOR_BLACK,
+           Ncurses::A_BOLD
     c.add :labellist_old_color, Ncurses::COLOR_WHITE, Ncurses::COLOR_BLACK
     c.add :labellist_new_color, Ncurses::COLOR_WHITE, Ncurses::COLOR_BLACK, 
            Ncurses::A_BOLD
@@ -198,6 +209,10 @@ begin
   
   imode.load_threads :num => ibuf.content_height, :when_done => lambda { reporting_thread("poll after loading inbox") { sleep 1; PollManager.poll } unless $opts[:no_threads] || $opts[:no_initial_poll] }
 
+  if $opts[:compose]
+    ComposeMode.spawn_nicely :to_default => $opts[:compose]
+  end
+
   unless $opts[:no_threads]
     PollManager.start
     SuicideManager.start
@@ -298,7 +313,7 @@ ensure
   Redwood::log "stopped cursing"
 
   if SuicideManager.instantiated? && SuicideManager.die?
-    Redwood::log "I've been ordered to commit sepuku. I obey!"
+    Redwood::log "I've been ordered to commit seppuku. I obey!"
   end
 
   if $exceptions.empty?

data/bin/sup-sync

@@ -133,7 +133,7 @@ begin
     num_added = num_updated = num_scanned = num_restored = 0
     last_info_time = start_time = Time.now
 
-    Redwood::PollManager.add_messages_from source do |m, offset, entry|
+    Redwood::PollManager.add_messages_from source, :force_overwrite => true do |m, offset, entry|
       num_scanned += 1
       seen[m.id] = true
 
@@ -227,6 +227,8 @@ begin
     $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
   end
 
+  index.save
+
   if opts[:optimize]
     $stderr.puts "Optimizing index..."
     optt = time { index.index.optimize unless opts[:dry_run] }
@@ -238,7 +240,6 @@ rescue Exception => e
   File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
   raise
 ensure
-  index.save
   Redwood::finish
   index.unlock
 end

data/bin/sup-sync-back

@@ -51,7 +51,11 @@ EOS
 end
 
 unless opts[:drop_deleted] || opts[:move_deleted] || opts[:drop_spam] || opts[:move_spam]
-  puts "Nothing to do."
+  puts <<EOS
+Nothing to do. Please specify at least one of --drop-deleted, --move-deleted,
+--drop-spam, or --move-spam.
+EOS
+
   exit
 end
 

data/doc/FAQ.txt

@@ -111,7 +111,7 @@ S: The current solution is to directly modify RubyMail. Change line 159 of
    multipart.rb to:
      chunk = chunk[0..start]
    This is because RubyMail hasn't been updated since like Ruby 1.8.2.
-   Please bug Matt Lickey.
+   Please bug Matt Armstrong.
 
 P: I see this error:
      /usr/local/lib/ruby/1.8/yaml.rb:133:in `transfer': allocator undefined for Bignum (TypeError)

data/lib/sup.rb

@@ -33,7 +33,7 @@ class Module
 end
 
 module Redwood
-  VERSION = "0.4"
+  VERSION = "0.5"
 
   BASE_DIR   = ENV["SUP_BASE"] || File.join(ENV["HOME"], ".sup")
   CONFIG_FN  = File.join(BASE_DIR, "config.yaml")
@@ -72,7 +72,7 @@ module Redwood
   def save_yaml_obj object, fn, safe=false
     if safe
       safe_fn = "#{File.dirname fn}/safe_#{File.basename fn}"
-      mode = File.stat(fn) if File.exists? fn
+      mode = File.stat(fn).mode if File.exists? fn
       File.open(safe_fn, "w", mode) { |f| f.puts object.to_yaml }
       FileUtils.mv safe_fn, fn
     else
@@ -169,7 +169,8 @@ if File.exists? Redwood::CONFIG_FN
 else
   require 'etc'
   require 'socket'
-  name = Etc.getpwnam(ENV["USER"]).gecos.split(/,/).first
+  name = Etc.getpwnam(ENV["USER"]).gecos.split(/,/).first rescue nil
+  name ||= ENV["USER"]
   email = ENV["USER"] + "@" + 
     begin
       Socket.gethostbyname(Socket.gethostname).first

data/lib/sup/buffer.rb

@@ -165,6 +165,15 @@ called at least once per keystroke, so excessive computation is discouraged.
 
 Variables: the same as status-bar-text hook.
 Return value: a string to be used as the terminal title.
+EOS
+
+  HookManager.register "extra-contact-addresses", <<EOS
+A list of extra addresses to propose for tab completion, etc. when the
+user is entering an email address. Can be plain email addresses or can
+be full "User Name <email@domain.tld>" entries.
+
+Variables: none
+Return value: an array of email address strings.
 EOS
 
   def initialize
@@ -263,7 +272,8 @@ EOS
         get_status_and_title @focus_buf # must be called outside of the ncurses lock
       end
 
-    print "\033]2;#{title}\07" if title && @in_x
+    ## http://rtfm.etla.org/xterm/ctlseq.html (see Operating System Controls)
+    print "\033]0;#{title}\07" if title && @in_x
 
     Ncurses.mutex.lock unless opts[:sync] == false
 
@@ -455,7 +465,7 @@ EOS
         elsif File.directory?(answer)
           spawn_modal "file browser", FileBrowserMode.new(answer)
         else
-          answer
+          File.expand_path answer
         end
     end
 
@@ -492,6 +502,7 @@ EOS
     contacts = ContactManager.contacts.map { |c| [ContactManager.alias_for(c), c.full_address, c.email] }
 
     completions = (recent + contacts).flatten.uniq.sort
+    completions += HookManager.run("extra-contact-addresses") || []
     answer = BufferManager.ask_many_emails_with_completions domain, question, completions, default
 
     if answer
@@ -557,7 +568,6 @@ EOS
 
   def ask_getch question, accept=nil
     raise "impossible!" if @asking
-    @asking = true
 
     accept = accept.split(//).map { |x| x[0] } if accept
 
@@ -570,6 +580,7 @@ EOS
       Ncurses.refresh
     end
 
+    @asking = true
     ret = nil
     done = false
     until done

data/lib/sup/contact.rb

@@ -51,7 +51,7 @@ class ContactManager
 
   def save
     File.open(@fn, "w") do |f|
-      @p2a.each do |p, a|
+      @p2a.sort_by { |(p, a)| [p.full_address, a] }.each do |(p, a)|
         f.puts "#{a || ''}: #{p.full_address}"
       end
     end

data/lib/sup/crypto.rb

@@ -40,7 +40,7 @@ class CryptoManager
     raise Error, (output || "gpg command failed: #{cmd}") unless $?.success?
 
     envelope = RMail::Message.new
-    envelope.header["Content-Type"] = 'multipart/signed; protocol="application/pgp-signature"; micalg=pgp-sha1'
+    envelope.header["Content-Type"] = 'multipart/signed; protocol=application/pgp-signature; micalg=pgp-sha1'
 
     envelope.add_part payload
     signature = RMail::Message.make_attachment output, "application/pgp-signature", nil, "signature.asc"

data/lib/sup/hook.rb

@@ -86,7 +86,7 @@ class HookManager
       log "error running hook: #{e.message}"
       log e.backtrace.join("\n")
       @hooks[name] = nil # disable it
-      BufferManager.flash "Error running hook: #{e.message}"
+      BufferManager.flash "Error running hook: #{e.message}" if BufferManager.instantiated?
     end
     context.__cleanup
     result

data/lib/sup/imap.rb

@@ -176,7 +176,7 @@ class IMAP < Source
 
   def end_offset
     unsynchronized_scan_mailbox
-    @ids.last
+    @ids.last + 1
   end
   synchronized :end_offset
 

data/lib/sup/index.rb

@@ -125,9 +125,13 @@ EOS
     @sources[source.id] = source
   end
 
-  def source_for uri; @sources.values.find { |s| s.is_source_for? uri }; end
-  def usual_sources; @sources.values.find_all { |s| s.usual? }; end
-  def sources; @sources.values; end
+  def sources
+    ## favour the inbox by listing non-archived sources first
+    @sources.values.sort_by { |s| s.id }.partition { |s| !s.archived? }.flatten
+  end
+
+  def source_for uri; sources.find { |s| s.is_source_for? uri }; end
+  def usual_sources; sources.find_all { |s| s.usual? }; end
 
   def load_index dir=File.join(@dir, "ferret")
     if File.exists? dir
@@ -137,11 +141,11 @@ EOS
     else
       Redwood::log "creating index..."
       field_infos = Ferret::Index::FieldInfos.new :store => :yes
-      field_infos.add_field :message_id
+      field_infos.add_field :message_id, :index => :untokenized
       field_infos.add_field :source_id
       field_infos.add_field :source_info
       field_infos.add_field :date, :index => :untokenized
-      field_infos.add_field :body, :store => :no
+      field_infos.add_field :body
       field_infos.add_field :label
       field_infos.add_field :subject
       field_infos.add_field :from
@@ -157,7 +161,7 @@ EOS
   ## and adding either way. Index state will be determined by m.labels.
   ##
   ## docid and entry can be specified if they're already known.
-  def sync_message m, docid=nil, entry=nil
+  def sync_message m, docid=nil, entry=nil, opts={}
     docid, entry = load_entry_for_id m.id unless docid && entry
 
     raise "no source info for message #{m.id}" unless m.source && m.source_info
@@ -170,7 +174,6 @@ EOS
         m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
       end
 
-    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
     snippet = 
       if m.snippet_contains_encrypted_content? && $config[:discard_snippets_from_encrypted_messages]
         ""
@@ -178,18 +181,52 @@ EOS
         m.snippet
       end
 
+    ## write the new document to the index. if the entry already exists in the
+    ## index, reuse it (which avoids having to reload the entry from the source,
+    ## which can be quite expensive for e.g. large threads of IMAP actions.)
+    ##
+    ## exception: if the index entry belongs to an earlier version of the
+    ## message, use everything from the new message instead, but union the
+    ## flags. this allows messages sent to mailing lists to have their header
+    ## updated and to have flags set properly.
+    ##
+    ## minor hack: messages in sources with lower ids have priority over
+    ## messages in sources with higher ids. so messages in the inbox will
+    ## override everyone, and messages in the sent box will be overridden
+    ## by everyone else.
+    ##
+    ## written in this manner to support previous versions of the index which
+    ## did not keep around the entry body. upgrading is thus seamless.
+    entry ||= {}
+    labels = m.labels.uniq # override because this is the new state, unless...
+
+    ## if we are a later version of a message, ignore what's in the index,
+    ## but merge in the labels.
+    if entry[:source_id] && entry[:source_info] && entry[:label] &&
+      ((entry[:source_id].to_i > source_id) || (entry[:source_info].to_i < m.source_info))
+      labels = (entry[:label].split(/\s+/).map { |l| l.intern } + m.labels).uniq
+      #Redwood::log "found updated version of message #{m.id}: #{m.subj}"
+      #Redwood::log "previous version was at #{entry[:source_id].inspect}:#{entry[:source_info].inspect}, this version at #{source_id.inspect}:#{m.source_info.inspect}"
+      #Redwood::log "merged labels are #{labels.inspect} (index #{entry[:label].inspect}, message #{m.labels.inspect})"
+      entry = {}
+    end
+
+    ## if force_overwite is true, ignore what's in the index. this is used
+    ## primarily by sup-sync to force index updates.
+    entry = {} if opts[:force_overwrite]
+
     d = {
       :message_id => m.id,
       :source_id => source_id,
       :source_info => m.source_info,
-      :date => m.date.to_indexable_s,
-      :body => m.content,
-      :snippet => snippet,
-      :label => m.labels.uniq.join(" "),
-      :from => m.from ? m.from.email : "",
-      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
-      :subject => wrap_subj(m.subj),
-      :refs => (m.refs + m.replytos).uniq.join(" "),
+      :date => (entry[:date] || m.date.to_indexable_s),
+      :body => (entry[:body] || m.indexable_content),
+      :snippet => snippet, # always override
+      :label => labels.uniq.join(" "),
+      :from => (entry[:from] || (m.from ? m.from.indexable_content : "")),
+      :to => (entry[:to] || (m.to + m.cc + m.bcc).map { |x| x.indexable_content }.join(" ")),
+      :subject => (entry[:subject] || wrap_subj(Message.normalize_subj(m.subj))),
+      :refs => (entry[:refs] || (m.refs + m.replytos).uniq.join(" ")),
     }
 
     @index.delete docid if docid
@@ -250,6 +287,7 @@ EOS
     searched = {}
     num_queries = 0
 
+    pending = [m.id]
     if $config[:thread_by_subject] # do subject queries
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
@@ -264,10 +302,13 @@ EOS
 
       q = build_query :qobj => q
 
-      pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
-      Redwood::log "found #{pending.size} results for subject query #{q}"
-    else
-      pending = [m.id]
+      p1 = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
+      Redwood::log "found #{p1.size} results for subject query #{q}"
+
+      p2 = @index.search(q.to_s, :limit => :all).hits.map { |hit| @index[hit.doc][:message_id] }
+      Redwood::log "found #{p2.size} results in string form"
+
+      pending = (pending + p1 + p2).uniq
     end
 
     until pending.empty? || (opts[:limit] && messages.size >= opts[:limit])
@@ -396,18 +437,7 @@ protected
   def parse_user_query_string s
     extraopts = {}
 
-    ## this is a little hacky, but it works, at least until ferret changes
-    ## its api. we parse the user query string with ferret twice: the first
-    ## time we just turn the resulting object back into a string, which has
-    ## the next effect of transforming the original string into a nice
-    ## normalized form with + and - instead of AND, OR, etc. then we do some
-    ## string substitutions which depend on this normalized form, re-parse
-    ## the string with Ferret, and return the resulting query object.
-
-    norms = @qparser.parse(s).to_s
-    Redwood::log "normalized #{s.inspect} to #{norms.inspect}" unless s == norms
-
-    subs = norms.gsub(/\b(to|from):(\S+)\b/) do
+    subs = s.gsub(/\b(to|from):(\S+)\b/) do
       field, name = $1, $2
       if(p = ContactManager.contact_for(name))
         [field, p.email]
@@ -477,7 +507,6 @@ protected
       subs = nil if chronic_failure
     end
     
-    Redwood::log "translated #{norms.inspect} to #{subs.inspect}" unless subs == norms
     if subs
       [@qparser.parse(subs), extraopts]
     else
@@ -512,7 +541,7 @@ protected
         File.chmod 0600, fn
         FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(fn) == 0
       end
-      Redwood::save_yaml_obj @sources.values.sort_by { |s| s.id.to_i }, fn, true
+      Redwood::save_yaml_obj sources.sort_by { |s| s.id.to_i }, fn, true
       File.chmod 0600, fn
     end
     @sources_dirty = false