sup 0.11 → 0.12

data/CONTRIBUTORS

@@ -3,6 +3,7 @@ Rich Lane <rlane at the club.cc.cmu dot edus>
 Ismo Puustinen <ismo at the iki dot fis>
 Nicolas Pouillard <nicolas.pouillard at the gmail dot coms>
 Eric Sherman <hyperbolist at the gmail dot coms>
+Michael Stapelberg <michael at the stapelberg dot des>
 Ben Walton <bwalton at the artsci.utoronto dot cas>
 Mike Stipicevic <stipim at the rpi dot edus>
 Marcus Williams <marcus-sup at the bar-coded dot nets>
@@ -10,34 +11,44 @@ Lionel Ott <white.magic at the gmx dot des>
 Tero Tilus <tero at the tilus dot nets>
 Ingmar Vanhassel <ingmar at the exherbo dot orgs>
 Mark Alexander <marka at the pobox dot coms>
+Gaute Hope <eg at the gaute.vetsj dot coms>
 Christopher Warrington <chrisw at the rice dot edus>
+W. Trevor King <wking at the drexel dot edus>
+Gaudenz Steinlin <gaudenz at the soziologie dot chs>
 Richard Brown <rbrown at the exherbo dot orgs>
 Marc Hartstein <marc.hartstein at the alum.vassar dot edus>
+Sascha Silbe <sascha-pgp at the silbe dot orgs>
 Israel Herraiz <israel.herraiz at the gmail dot coms>
 Anthony Martinez <pi+sup at the pihost dot uss>
-Michael Stapelberg <michael at the stapelberg dot des>
+Hamish Downer <dmishd at the gmail dot coms>
 Bo Borgerson <gigabo at the gmail dot coms>
 William Erik Baxter <web at the superscript dot coms>
+Michael Hamann <michael at the content-space dot des>
 Grant Hollingworth <grant at the antiflux dot orgs>
 Adeodato Simó <dato at the net.com.org dot ess>
 Daniel Schoepe <daniel.schoepe at the googlemail dot coms>
+Jason Petsod <jason at the petsod dot orgs>
 Steve Goldman <sgoldman at the tower-research dot coms>
 Edward Z. Yang <ezyang at the MIT dot EDUs>
 Decklin Foster <decklin at the red-bean dot coms>
 Cameron Matheson <cam+sup at the cammunism dot orgs>
 Carl Worth <cworth at the cworth dot orgs>
+Jeff Balogh <its.jeff.balogh at the gmail dot coms>
 Andrew Pimlott <andrew at the pimlott dot nets>
 Alex Vandiver <alexmv at the mit dot edus>
-Jeff Balogh <its.jeff.balogh at the gmail dot coms>
 Peter Harkins <ph at the malaprop dot orgs>
 Kornilios Kourtis <kkourt at the cslab.ece.ntua dot grs>
-Michael Hamann <michael at the content-space dot des>
 Giorgio Lando <patroclo7 at the gmail dot coms>
+Damien Leone <damien.leone at the fensalir dot frs>
 Benoît PIERRE <benoit.pierre at the gmail dot coms>
+Alvaro Herrera <alvherre at the alvh.no-ip dot orgs>
 Jonah <Jonah at the GoodCoffee dot cas>
+Adam Lloyd <adam at the alloy-d dot nets>
+Todd Eisenberger <teisenbe at the andrew.cmu dot edus>
 ian <ian at the lorf dot orgs>
 Steven Walter <swalter at the monarch.(none)>
-Alex Vandiver <alex at the chmrr dot nets>
-Stefan Lundström <lundst at the snabb.(none)>
+ian <itaylor at the uark dot edus>
 Jon M. Dugan <jdugan at the es dot nets>
+Gregor Hoffleit <gregor at the sam.mediasupervision dot des>
+Stefan Lundström <lundst at the snabb.(none)>
 Kirill Smelkov <kirr at the landau.phys.spbu dot rus>

data/History.txt

@@ -1,3 +1,12 @@
+== 0.12 / 2011-01-13
+* Remove deprecated IMAP, IMAPS, and mbox+ssh sources
+* Inline GPG support
+* Robust maildir support
+* sup-dump compatibility between Sup versions
+* New hook: sendmail
+* Better Ruby 1.9/UTF8 support
+* As always, many bugfixes and tweaks.
+
 == 0.11 / 2010-03-07
 * Remove deprecated Ferret backend.
 * Add deprecation notices to IMAP, IMAPS, and mbox+ssh sources.

data/ReleaseNotes

@@ -1,3 +1,12 @@
+Release 0.12:
+
+Deprecated remote sources have been removed.
+
+Maildir support has been improved to gracefully handle messages that
+move or disappear. The "out of sync" errors should no longer occur.
+
+Inline GPG is now supported.
+
 Release 0.11:
 
 The deprecated Ferret index has been removed.

data/bin/sup

@@ -12,7 +12,7 @@ end
 
 require 'fileutils'
 require 'trollop'
-require "sup"; Redwood::check_library_version_against "0.11"
+require "sup"; Redwood::check_library_version_against "0.12"
 
 if ENV['SUP_PROFILE']
   require 'ruby-prof'
@@ -49,7 +49,7 @@ No variables.
 No return value.
 EOS
 
-Redwood::HookManager.register "shutdown", <<EOS 
+Redwood::HookManager.register "shutdown", <<EOS
 Executes when sup is shutting down. May be run when sup is crashing,
 so don\'t do anything too important. Run before the label, contacts,
 and people are saved.
@@ -58,6 +58,7 @@ No return value.
 EOS
 
 if $opts[:list_hooks]
+  Redwood.start
   Redwood::HookManager.print_hooks
   exit
 end
@@ -86,6 +87,7 @@ global_keymap = Keymap.new do |k|
   k.add :nothing, "Do nothing", :ctrl_g
   k.add :recall_draft, "Edit most recent draft message", 'R'
   k.add :show_inbox, "Show the Inbox buffer", 'I'
+  k.add :clear_hooks, "Clear all hooks", 'H'
   k.add :show_console, "Show the Console buffer", '~'
 
   ## Submap for less often used keybindings
@@ -94,8 +96,6 @@ global_keymap = Keymap.new do |k|
     kk.add :run_keybindings_hook, "Rerun keybindings hook", 'k'
   end
 end
-  
-Redwood::Keymap.run_hook global_keymap
 
 ## the following magic enables wide characters when used with a ruby
 ## ncurses.so that's been compiled against libncursesw. (note the w.) why
@@ -154,26 +154,6 @@ begin
   Index.load
   Index.start_sync_worker unless $opts[:no_threads]
 
-  if Redwood::SourceManager.sources.any? { |x| x.is_a? Redwood::MBox::SSHLoader }
-    $stderr.puts <<EOS
-mbox+ssh sources are deprecated and will be removed in the next release.
-Running rsync in your before-poll hook is a good alternative.
-
-Press enter to continue.
-EOS
-    $stdin.gets
-  end
-
-  if Redwood::SourceManager.sources.any? { |x| x.is_a? Redwood::IMAP }
-    $stderr.puts <<EOS
-IMAP sources are deprecated and will be removed in the next release.
-Running offlineimap or fetchmail in your before-poll hook is a good alternative.
-
-Press enter to continue.
-EOS
-    $stdin.gets
-  end
-
   $die = false
   trap("TERM") { |x| $die = true }
   trap("WINCH") { |x| BufferManager.sigwinch_happened! }
@@ -192,6 +172,7 @@ EOS
   end
 
   HookManager.run "startup"
+  Redwood::Keymap.run_hook global_keymap
 
   debug "starting curses"
   Redwood::Logger.remove_sink $stderr
@@ -318,6 +299,8 @@ EOS
       SearchResultsMode.spawn_from_query "is:unread"
     when :list_labels
       labels = LabelManager.all_labels.map { |l| LabelManager.string_for l }
+      labels = labels.each { |l| l.force_encoding 'UTF-8' if l.methods.include?(:encoding) }
+
       user_label = bm.ask_with_completions :label, "Show threads with label (enter for listing): ", labels
       unless user_label.nil?
         if user_label.empty?
@@ -350,6 +333,8 @@ EOS
       end
     when :show_inbox
       BufferManager.raise_to_front ibuf
+    when :clear_hooks
+      HookManager.clear
     when :show_console
       b, new = bm.spawn_unless_exists("Console", :system => true) { ConsoleMode.new }
       b.mode.run

data/bin/sup-add

@@ -4,7 +4,7 @@ require 'uri'
 require 'rubygems'
 require 'highline/import'
 require 'trollop'
-require "sup"; Redwood::check_library_version_against "0.11"
+require "sup"; Redwood::check_library_version_against "0.12"
 
 $opts = Trollop::options do
   version "sup-add (sup #{Redwood::VERSION})"
@@ -97,9 +97,9 @@ begin
     source = 
       case parsed_uri.scheme
       when "maildir"
-        Redwood::Maildir.new uri, nil, !$opts[:unusual], $opts[:archive], nil, labels
+        Redwood::Maildir.new uri, !$opts[:unusual], $opts[:archive], nil, labels
       when "mbox"
-        Redwood::MBox::Loader.new uri, nil, !$opts[:unusual], $opts[:archive], nil, labels
+        Redwood::MBox.new uri, !$opts[:unusual], $opts[:archive], nil, labels
       when nil
         Trollop::die "Sources must be specified with an URI"
       else

data/bin/sup-config

@@ -57,7 +57,7 @@ def add_source
       return if fn.nil? || fn.empty?
 
       $last_fn = fn
-      [Redwood::MBox::Loader.suggest_labels_for(fn),
+      [Redwood::MBox.suggest_labels_for(fn),
        { :scheme => "mbox", :path => fn }]
     when :maildir
       $last_fn ||= ENV["MAIL"]

data/bin/sup-dump

@@ -1,17 +1,20 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
+require 'xapian'
 require 'trollop'
-require "sup"; Redwood::check_library_version_against "0.11"
+require 'set'
+
+BASE_DIR = ENV["SUP_BASE"] || File.join(ENV["HOME"], ".sup")
 
 $opts = Trollop::options do
-  version "sup-dump (sup #{Redwood::VERSION})"
+  version "sup-dump"
   banner <<EOS
 Dumps all message state from the sup index to standard out. You can
 later use sup-sync --restored --restore <filename> to recover the index.
 
-This tool is primarily useful in the event that a Ferret upgrade breaks
-the index format. This happened, for example, at Ferret version 0.11.
+This tool is primarily useful in the event that a Sup upgrade breaks index
+format compatibility.
 
 Usage:
   sup-dump > <filename>
@@ -19,10 +22,20 @@ Usage:
 EOS
 end
 
-index = Redwood::Index.init
-Redwood::SourceManager.init
-index.load
+xapian = Xapian::Database.new File.join(BASE_DIR, 'xapian')
+version = xapian.get_metadata 'rescue-version'
+version = '0' if version.empty?
 
-index.each_message :load_spam => true, :load_deleted => true, :load_killed => true do |m|
-  puts "#{m.id} (#{m.labels.to_a.sort_by { |l| l.to_s } * ' '})"
+case version
+when '0'
+  xapian.postlist('Kmail').each do |x|
+    begin
+      entry = Marshal.load(xapian.document(x.docid).data)
+      puts "#{entry[:message_id]} (#{entry[:labels].sort_by { |l| l.to_s } * ' '})"
+    rescue
+      $stderr.puts "failed to dump document #{x.docid}"
+    end
+  end
+else
+  abort "this sup-dump version doesn't understand your index"
 end

data/bin/sup-recover-sources

@@ -58,17 +58,7 @@ ARGV.each do |fn|
   next if Redwood::SourceManager.source_for fn
 
   ## TODO: merge this code with the same snippet in import
-  source = 
-    case fn
-    when %r!^imaps?://!
-      print "Username for #{fn}: "
-      username = $stdin.gets.chomp
-      print "Password for #{fn} (warning: cleartext): "
-      password = $stdin.gets.chomp
-      Redwood::IMAP.new(fn, username, password, nil, !$opts[:unusual], $opts[:archive])
-    else
-      Redwood::MBox::Loader.new(fn, nil, !$opts[:unusual], $opts[:archive])
-    end
+  source = Redwood::MBox.new(fn, nil, !$opts[:unusual], $opts[:archive])
 
   source_ids = Hash.new 0
   count = 0

data/bin/sup-sync

@@ -3,7 +3,7 @@
 require 'uri'
 require 'rubygems'
 require 'trollop'
-require "sup"; Redwood::check_library_version_against "0.11"
+require "sup"; Redwood::check_library_version_against "0.12"
 
 PROGRESS_UPDATE_INTERVAL = 15 # seconds
 
@@ -53,16 +53,6 @@ where <source>* is zero or more source URIs. If no sources are given,
 sync from all usual sources. Supported source URI schemes can be seen
 by running "sup-add --help".
 
-Options controlling WHICH messages sup-sync operates on:
-EOS
-  opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
-  opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source."
-  opt :restored, "Operate only on those messages included in a dump file as specified by --restore which have changed state."
-  opt :all, "Operate on all messages in the source, regardless of newness or changedness."
-  opt :start_at, "For --changed, --restored and --all, start at a particular offset.", :type => :int
-
-text <<EOS
-
 Options controlling HOW message state is altered:
 EOS
   opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
@@ -82,16 +72,9 @@ EOS
   opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
   opt :version, "Show version information", :short => :none
 
-  conflicts :changed, :all, :new, :restored
   conflicts :asis, :restore, :discard
 end
-Trollop::die :restored, "requires --restore" if opts[:restored] unless opts[:restore]
-if opts[:start_at]
-  Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
-  Trollop::die :start_at, "requires either --changed, --restored or --all" unless opts[:changed] || opts[:restored] || opts[:all]
-end
 
-target = [:new, :changed, :all, :restored].find { |x| opts[x] } || :new
 op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
 
 Redwood::start
@@ -116,6 +99,12 @@ index.lock_interactively or exit
 begin
   index.load
 
+  if(s = Redwood::SourceManager.source_for Redwood::SentManager.source_uri)
+    Redwood::SentManager.source = s
+  else
+    Redwood::SourceManager.add_source Redwood::SentManager.default_source
+  end
+
   sources = if opts[:all_sources]
     Redwood::SourceManager.sources
   elsif ARGV.empty?
@@ -126,134 +115,80 @@ begin
     end
   end
 
-  ## for all target specifications except for only-new messages, reset the
-  ## source to the beginning (or to the user-specified starting point.)
-  unless target == :new
-    if opts[:start_at]
-      Trollop::die :start_at, "can only be used on one source" unless sources.size == 1
-      sources.first.seek_to! opts[:start_at]
-      sources.first.correct_offset! if sources.first.respond_to?(:correct_offset!)
-    else
-      sources.each { |s| s.reset! }
-    end
-  end
-
   sources.each do |source|
     puts "Scanning #{source}..."
     num_added = num_updated = num_scanned = num_restored = 0
     last_info_time = start_time = Time.now
 
-    Redwood::PollManager.each_message_from source do |m|
-      num_scanned += 1
-      seen[m.id] = true
-      old_m = index.build_message m.id
-
-      case target
-      when :changed
-        ## skip this message if we're operating only on changed messages, the
-        ## message is in the index, and it's unchanged from what the source is
-        ## reporting.
-        next if old_m && old_m.source.id == m.source.id && old_m.source_info == m.source_info
-      when :restored
-        ## skip if we're operating on restored messages, and this one
-        ## ain't (or we wouldn't be making a change)
-        next unless old_m && restored_state[m.id] && restored_state[m.id] != old_m.labels
-      when :new
-        ## nothing to do; we'll consider all messages starting at the start offset, which
-        ## hasn't been changed.
-      when :all
-        ## nothing to do; we'll consider all messages starting at the start offset, which
-        ## was reset to the beginning above.
-      end
-
-      ## tweak source labels according to commandline arguments if necessary
-      m.labels.delete :inbox if opts[:archive]
-      m.labels.delete :unread if opts[:read]
-      m.labels += opts[:extra_labels].to_set_of_symbols(",")
-
-      ## decide what to do based on message labels and the operation we're performing
-      dothis, new_labels = case
-      when (op == :restore) && restored_state[m.id]
-        if old_m && (old_m.labels != restored_state[m.id])
-          num_restored += 1
-          [:update_message_state, restored_state[m.id]]
-        elsif old_m.nil?
-          num_restored += 1
-          m.labels = restored_state[m.id]
-          :add_message
-        else
-          # labels are the same; don't do anything
-        end
-      when op == :discard
-        if old_m && (old_m.labels != m.labels)
-          [:update_message_state, m.labels]
-        else
-          # labels are the same; don't do anything
-        end
-      else
-        ## duplicate behavior of poll mode: if index_state is non-nil, this is a newer
-        ## version of an older message, so merge in any new labels except :unread and
-        ## :inbox.
-        ##
-        ## TODO: refactor such that this isn't duplicated
-        if old_m
-          m.labels = old_m.labels + (m.labels - [:unread, :inbox])
-          :update_message
+    Redwood::PollManager.poll_from source do |action,m,old_m,progress|
+      if action == :delete
+        puts "Deleting #{m.id}"
+      elsif action == :add
+        num_scanned += 1
+        seen[m.id] = true
+
+        ## tweak source labels according to commandline arguments if necessary
+        m.labels.delete :inbox if opts[:archive]
+        m.labels.delete :unread if opts[:read]
+        m.labels += opts[:extra_labels].to_set_of_symbols(",")
+
+        ## decide what to do based on message labels and the operation we're performing
+        dothis = case
+        when (op == :restore) && restored_state[m.id]
+          if old_m && (old_m.labels != restored_state[m.id])
+            num_restored += 1
+            m.labels = restored_state[m.id]
+            :update_message_state
+          elsif old_m.nil?
+            num_restored += 1
+            m.labels = restored_state[m.id]
+            :add_message
+          else
+            # labels are the same; don't do anything
+          end
+        when op == :discard
+          if old_m && (old_m.labels != m.labels)
+            :update_message_state
+          else
+            # labels are the same; don't do anything
+          end
         else
-          :add_message
+          if old_m
+            :update_message
+          else
+            :add_message
+          end
         end
-      end
 
-      ## now, actually do the operation
-      case dothis
-      when :add_message
-        puts "Adding new message #{source}##{m.source_info} with labels #{m.labels}" if opts[:verbose]
-        index.add_message m unless opts[:dry_run]
-        num_added += 1
-      when :update_message
-        puts "Updating message #{source}##{m.source_info}; labels #{old_m.labels} => #{m.labels}; offset #{old_m.source_info} => #{m.source_info}" if opts[:verbose]
-        index.update_message m unless opts[:dry_run]
-        num_updated += 1
-      when :update_message_state
-        puts "Changing flags for #{source}##{m.source_info} from #{m.labels} to #{new_labels}" if opts[:verbose]
-        m.labels = new_labels
-        index.update_message_state m unless opts[:dry_run]
-        num_updated += 1
-      end
+        ## now, actually do the operation
+        case dothis
+        when :add_message
+          puts "Adding new message #{source}##{m.source_info} with labels #{m.labels}" if opts[:verbose]
+          num_added += 1
+        when :update_message
+          puts "Updating message #{source}##{m.source_info}; labels #{old_m.labels} => #{m.labels}; offset #{old_m.source_info} => #{m.source_info}" if opts[:verbose]
+          num_updated += 1
+        when :update_message_state
+          puts "Changing flags for #{source}##{m.source_info} from #{old_m.labels} to #{m.labels}" if opts[:verbose]
+          num_updated += 1
+        end
 
-      if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
-        last_info_time = Time.now
-        elapsed = last_info_time - start_time
-        pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
-        remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
-        printf "## read %dm (~%.0f%%) @ %.1fm/s. %s elapsed, ~%s remaining, offset #{source.cur_offset}\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
+        if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
+          last_info_time = Time.now
+          elapsed = last_info_time - start_time
+          pctdone = progress * 100.0
+          remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+          printf "## read %dm (~%.0f%%) @ %.1fm/s. %s elapsed, ~%s remaining\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
+        end
+      else fail
       end
+      next if opts[:dry_run]
     end
 
     puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated} messages from #{source}."
     puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
   end
 
-  ## delete any messages in the index that claim they're from one of
-  ## these sources, but that we didn't see.
-  if (target == :all || target == :changed)
-    puts "Deleting missing messages from the index..."
-    num_del, num_scanned = 0, 0
-    sources.each do |source|
-      raise "no source id for #{source}" unless source.id
-      index.each_message :source_id => source.id, :load_spam => true, :load_deleted => true, :load_killed => true do |m|
-        num_scanned += 1
-        unless seen[m.id]
-          next unless m.source_info >= opts[:start_at] if opts[:start_at]
-          puts "Deleting #{m.id}" if opts[:verbose]
-          index.delete m.id unless opts[:dry_run]
-          num_del += 1
-        end
-      end
-    end
-    puts "Deleted #{num_del} / #{num_scanned} messages"
-  end
-
   index.save
 
   if opts[:optimize]