sup 0.0.8 → 0.1

data/bin/sup-add

@@ -37,6 +37,7 @@ Options are:
 EOS
   opt :archive, "Automatically archive all new messages from these sources."
   opt :unusual, "Do not automatically poll these sources for new messages."
+  opt :labels, "A comma-separated set of labels to apply to all messages from this source", :type => String
   opt :force_new, "Create a new account for this source, even if one already exists."
 end
 
@@ -77,36 +78,45 @@ end
 $terminal.wrap_at = :auto
 Redwood::start
 index = Redwood::Index.new
-index.load
 
-ARGV.each do |uri|
-  if !$opts[:force_new] && index.source_for(uri) 
-    say "Already know about #{uri}; skipping."
-    next
-  end
+index.lock_or_die
+
+begin
+  index.load_sources
 
-  parsed_uri = URI(uri)
-
-  source = 
-    case parsed_uri.scheme
-    when "mbox+ssh"
-      say "For SSH connections, if you will use public key authentication, you may leave the username and password blank."
-      say ""
-      username, password = get_login_info uri, index.sources
-      Redwood::MBox::SSHLoader.new uri, username, password, nil, !$opts[:unusual], $opts[:archive]
-    when "imap", "imaps"
-      username, password = get_login_info uri, index.sources
-      Redwood::IMAP.new uri, username, password, nil, !$opts[:unusual], $opts[:archive]
-    when "maildir"
-      Redwood::Maildir.new uri, nil, !$opts[:unusual], $opts[:archive]
-    when "mbox"
-      Redwood::MBox::Loader.new uri, nil, !$opts[:unusual], $opts[:archive]
-    else
-      Trollop::die "Unknown source type #{parsed_uri.scheme.inspect}"      
+  ARGV.each do |uri|
+    labels = $opts[:labels] ? $opts[:labels].split(/\s*,\s*/).uniq : []
+
+    if !$opts[:force_new] && index.source_for(uri) 
+      say "Already know about #{uri}; skipping."
+      next
     end
-  say "Adding #{source}..."
-  index.add_source source
-end
 
-index.save
-Redwood::finish
+    parsed_uri = URI(uri)
+    Trollop::die "no path component to uri: #{parsed_uri}" unless parsed_uri.path
+
+    source = 
+      case parsed_uri.scheme
+      when "mbox+ssh"
+        say "For SSH connections, if you will use public key authentication, you may leave the username and password blank."
+        say ""
+        username, password = get_login_info uri, index.sources
+        Redwood::MBox::SSHLoader.new uri, username, password, nil, !$opts[:unusual], $opts[:archive], nil, labels
+      when "imap", "imaps"
+        username, password = get_login_info uri, index.sources
+        Redwood::IMAP.new uri, username, password, nil, !$opts[:unusual], $opts[:archive], nil, labels
+      when "maildir"
+        Redwood::Maildir.new uri, nil, !$opts[:unusual], $opts[:archive], nil, labels
+      when "mbox"
+        Redwood::MBox::Loader.new uri, nil, !$opts[:unusual], $opts[:archive], nil, labels
+      else
+        Trollop::die "Unknown source type #{parsed_uri.scheme.inspect}"      
+      end
+    say "Adding #{source}..."
+    index.add_source source
+  end
+ensure
+  index.save
+  index.unlock
+  Redwood::finish
+end

data/bin/sup-config

@@ -21,11 +21,11 @@ end #' stupid ruby-mode
 
 def axe q, default=nil
   ans = 
-  if default && !default.empty?
-    ask "#{q} (enter for \"#{default}\"): "
-  else
-    ask "#{q}: "
-  end
+    if default && !default.empty?
+      ask "#{q} (enter for \"#{default}\"): "
+    else
+      ask "#{q}: "
+    end
   ans.empty? ? default : ans
 end
 
@@ -42,7 +42,7 @@ def add_source
 
   say "Ok, adding a new source."
   choose do |menu|
-    menu.prompt = "What type of mail source is it?"
+    menu.prompt = "What type of mail source is it? "
     menu.choice("mbox file") { type = :mbox }
     menu.choice("maildir directory") { type = :maildir }
     menu.choice("remote mbox file (accessible via ssh)") { type = :mboxssh }
@@ -52,34 +52,57 @@ def add_source
   end
 
   while true do
-    say "Now for the details."
+    say "Ok, now for the details."
 
-    components = 
+    default_labels, components = 
       case type
       when :mbox
-        fn = axe "What's the full path to the mbox file?", ENV["MAIL"] #"srm
+        $last_fn ||= ENV["MAIL"]
+        fn = axe "What's the full path to the mbox file?", $last_fn #"srm
         return if fn.nil? || fn.empty?
-        { :scheme => "mbox", :path => fn }
+
+        $last_fn = fn
+        [Redwood::MBox::Loader.suggest_labels_for(fn),
+         { :scheme => "mbox", :path => fn }]
       when :maildir
-        fn = axe "What's the full path to the maildir directory?", ENV["MAIL"] #"srm
+        $last_fn ||= ENV["MAIL"]
+        fn = axe "What's the full path to the maildir directory?", $last_fn #"srm
         return if fn.nil? || fn.empty?
-        { :scheme => "maildir", :path => fn }
+
+        $last_fn = fn
+        [Redwood::Maildir.suggest_labels_for(fn),
+         { :scheme => "maildir", :path => fn }]
       when :mboxssh
-        srv = axe "What server is the mbox file located on?", $last_server 
+        $last_server ||= "localhost"
+        srv = axe "What machine is the mbox file located on?", $last_server 
         return if srv.nil? || srv.empty?
         $last_server = srv
-        fn = axe "What's the full path to the mbox file?", ENV["MAIL"] #"srm
+
+        fn = axe "What's the path to the mbox file?", $last_fn #" stupid ruby-mode
         return if fn.nil? || fn.empty?
+        $last_fn = fn
         fn = "/#{fn}" # lame
-        { :scheme => "mbox+ssh", :host => srv, :path => fn }
+        [Redwood::MBox::SSHLoader.suggest_labels_for(fn),
+         { :scheme => "mbox+ssh", :host => srv, :path => fn }]
       when :imap, :imaps
-        srv = axe "What is the IMAP server?", $last_server
+        $last_server ||= "localhost"
+        srv = axe "What is the IMAP server (host, or host:port notation)?", $last_server
         return if srv.nil? || srv.empty?
         $last_server = srv
-        fn = axe "What's the folder path?", "INBOX" #"srm 
+
+        $last_folder ||= "INBOX"
+        fn = axe "What's the folder path?", $last_folder #"srm 
         return if fn.nil? || fn.empty?
+        $last_folder = fn
+
         fn = "/#{fn}" # lame
-        { :scheme => type.to_s, :host => srv, :path => fn }
+        if srv =~ /^(\w+):(\d+)$/
+          host, port = $1, $2.to_i
+        else
+          host, port = srv, nil
+        end
+        [Redwood::IMAP.suggest_labels_for(fn),
+         { :scheme => type.to_s, :host => host, :port => port, :path => fn }]
       end
     
     uri = 
@@ -90,17 +113,27 @@ def add_source
         if axe_yes("Try again?") then next else return end
       end
 
-    say "I'm going to add this source: #{uri}."
+    say "I'm going to add this source: #{uri}"
     unless axe("Does that look right?", "y") =~ /^y|yes$/i
       if axe_yes("Try again?") then next else return end
     end
 
     usual = axe_yes "Does this source ever receive new messages?", "y"
-    archive = usual ? axe_yes("Should those new messages be automatically archived?") : false
+    archive = usual ? axe_yes("Should new messages be automatically archived? (I.e. not appear in your inbox, though still be accessible via search.)") : false
+
+    labels_str = axe("Enter any labels to be automatically added to all messages from this source, separated by spaces (or 'none')", default_labels.join(","))
+
+    labels =
+      if labels_str =~ /^\s*none\s*$/i
+        nil
+      else
+        labels_str.split(/\s+/)
+      end
     
     cmd = build_cmd "sup-add"
     cmd += " --unusual" unless usual
     cmd += " --archive" if archive
+    cmd += " --labels=#{labels.join(',')}" if labels && !labels.empty?
     cmd += " #{uri}"
 
     puts "Ok, trying to run \"#{cmd}\"..."
@@ -128,8 +161,7 @@ program. Get ready to be the envy of everyone in your internets
 with your amazing keyboarding skills! Jump from email to email with
 nary a click of the mouse!
 
-Just answer these simple questions and you'll be on your way! Press
-enter at any point to accept the default answer.
+Just answer these simple questions and you'll be on your way.
 
 EOS
 #' stupid ruby-mode
@@ -137,7 +169,7 @@ EOS
 account = $config[:accounts][:default]
 
 name = axe "What's your name?", account[:name]
-email = axe "What's your email address?", account[:email] #'srm
+email = axe "What's your (primary) email address?", account[:email] #'srm
 
 say "Ok, your header will look like this:"
 say "  From: #{name} <#{email}>"
@@ -172,7 +204,7 @@ until done
 
   say "\n"
   choose do |menu|
-    menu.prompt = "Your wish?"
+    menu.prompt = "Your wish? "
     menu.choice("Add a new source.") { add_source }
     menu.choice("Done adding sources!") { done = true }
   end
@@ -184,12 +216,6 @@ Ok. The final step is to import all your messages into the Sup index.
 Depending on how many messages are in the sources, this could take
 quite a while.
 
-IMPORTANT NOTE: this import will archive messages if the source is
-marked archival, and won't otherwise. It will preserve read/unread
-status as given by the source, and it will automatically add one label
-per source. All of this behavior can be controlled on per-source
-basis by running sup-sync manually.
-
 EOS
 #'
 if axe_yes "Run sup-sync to import all messages now?"
@@ -216,7 +242,7 @@ like you're ready to jack in to cyberspace there, cowboy.
 
 Just one last command:
 
-  sup
+  #{build_cmd "sup"}
 
 Have fun!
 EOS

data/bin/sup-sync

@@ -43,8 +43,8 @@ Usage:
   sup-sync [options] <source>*
 
 where <source>* is zero or more source URIs. If no sources are given,
-sync from all usual sources. All supported source URI schemes can
-be seen by running "sup-add --help".
+sync from all usual sources. Supported source URI schemes can be seen
+by running "sup-add --help".
 
 Options controlling WHICH messages sup-sync operates on:
 EOS
@@ -89,7 +89,6 @@ op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
 
 Redwood::start
 index = Redwood::Index.new
-index.load
 
 restored_state =
   if opts[:restore]
@@ -106,24 +105,26 @@ restored_state =
     {}
   end
 
-sources = ARGV.map do |uri|
-  uri = "mbox://#{uri}" unless uri =~ %r!://!
-  index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
-end
-
-sources = index.usual_sources if sources.empty?
-sources = index.sources if opts[:all_sources]
-
-unless target == :new
-  if opts[:start_at]
-    sources.each { |s| s.seek_to! opts[:start_at] }
-  else
-    sources.each { |s| s.reset! }
-  end
-end
-
 seen = {}
+index.lock_or_die
 begin
+  index.load
+
+  sources = ARGV.map do |uri|
+    index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+  end
+  
+  sources = index.usual_sources if sources.empty?
+  sources = index.sources if opts[:all_sources]
+
+  unless target == :new
+    if opts[:start_at]
+      sources.each { |s| s.seek_to! opts[:start_at] }
+    else
+      sources.each { |s| s.reset! }
+    end
+  end
+  
   sources.each do |source|
     $stderr.puts "Scanning #{source}..."
     num_added = num_updated = num_scanned = num_restored = 0
@@ -148,7 +149,7 @@ begin
 
       ## skip if we're operating on restored messages, and this one
       ## ain't.
-      next if target == :restored && (!restored_state[m.id] || restored_state[m.id].sort_by { |s| s.to_s } == index_state.sort_by { |s| s.to_s })
+      next if target == :restored && (!restored_state[m.id] || (index_state && restored_state[m.id].sort_by { |s| s.to_s } == index_state.sort_by { |s| s.to_s }))
 
       ## m.labels is the default source labels. tweak these according
       ## to default source state modification flags.
@@ -177,7 +178,7 @@ begin
         elapsed = last_info_time - start_time
         pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
         remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
-        $stderr.puts "## #{num_added + num_updated} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining (for this source)"
+        $stderr.puts "## #{num_scanned} (#{pctdone}%) read; #{elapsed.to_time_s} elapsed; #{remaining.to_time_s} remaining"
       end
 
       if index_state.nil?
@@ -193,43 +194,48 @@ begin
     $stderr.puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated} messages from #{source}."
     $stderr.puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
   end
+
+  ## delete any messages in the index that claim they're from one of
+  ## these sources, but that we didn't see.
+  ##
+  ## kinda crappy code here, because we delve directly into the Ferret
+  ## API.
+  ##
+  ## TODO: move this to Index, i suppose.
+
+
+  if target == :all || target == :changed
+    $stderr.puts "Deleting missing messages from the index..."
+    num_del, num_scanned = 0, 0
+    sources.each do |source|
+      raise "no source id for #{source}" unless source.id
+      q = "+source_id:#{source.id}"
+      q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
+      index.index.search_each(q, :limit => :all) do |docid, score|
+        num_scanned += 1
+        mid = index.index[docid][:message_id]
+        unless seen[mid]
+          puts "Deleting #{mid}" if opts[:verbose]
+          index.index.delete docid unless opts[:dry_run]
+          num_del += 1
+        end
+      end
+    end
+    $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
+  end
+
+  if opts[:optimize]
+    $stderr.puts "Optimizing index..."
+    optt = time { index.index.optimize unless opts[:dry_run] }
+    $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
+  end
+rescue Redwood::FatalSourceError => e
+  $stderr.puts "Sorry, I couldn't communicate with a source: #{e.message}"
 rescue Exception => e
   File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
   raise
 ensure
   index.save
   Redwood::finish
-end
-
-## delete any messages in the index that claim they're from one of
-## these sources, but that we didn't see.
-##
-## kinda crappy code here, because we delve directly into the Ferret
-## API.
-##
-## TODO: move this to Index, i suppose.
-if target == :all || target == :changed
-  $stderr.puts "Deleting missing messages from the index..."
-  num_del, num_scanned = 0, 0
-  sources.each do |source|
-    raise "no source id for #{source}" unless source.id
-    q = "+source_id:#{source.id}"
-    q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
-    index.index.search_each(q, :limit => :all) do |docid, score|
-      num_scanned += 1
-      mid = index.index[docid][:message_id]
-      unless seen[mid]
-        puts "Deleting #{mid}" if opts[:verbose]
-        index.index.delete docid unless opts[:dry_run]
-        num_del += 1
-      end
-    end
-  end
-  $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
-end
-
-if opts[:optimize]
-  $stderr.puts "Optimizing index..."
-  optt = time { index.index.optimize unless opts[:dry_run] }
-  $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
+  index.unlock
 end

data/bin/sup-sync-back

@@ -0,0 +1,143 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'uri'
+require 'tempfile'
+require 'trollop'
+require "sup"
+
+## save a message 'm' to an open file pointer 'fp'
+def save m, fp
+  m.source.each_raw_full_message_line(m.source_info) { |l| fp.print l }
+end
+
+opts = Trollop::options do
+  version "sup-sync-back (sup #{Redwood::VERSION})"
+  banner <<EOS
+Partially synchronizes a message source with the Sup index by deleting
+or moving any messages from the source that are marked as deleted or
+spam in the Sup index.
+
+Currently only works with mbox sources.
+
+Usage:
+  sup-sync-back [options] <source>*
+
+where <source>* is zero or more source URIs. If no sources are given,
+sync back all usual sources.
+
+You probably want to run sup-sync --changed after this command.
+
+Options include:
+EOS
+  opt :delete_deleted, "Delete deleted messages.", :default => false, :short => "d"
+  opt :move_deleted, "Move deleted messages to a local mbox file.", :type => String, :short => :none
+  opt :delete_spam, "Delete spam messages.", :default => false, :short => "s"
+  opt :move_spam, "Move spam messages to a local mbox file.", :type => String, :short => :none
+  opt :verbose, "Print message ids as they're processed."
+  opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
+  opt :version, "Show version information", :short => :none
+
+  conflicts :delete_deleted, :move_deleted
+  conflicts :delete_spam, :move_spam
+end
+
+Redwood::start
+index = Redwood::Index.new
+index.lock_or_die
+
+deleted_fp, spam_fp = nil
+unless opts[:dry_run]
+  deleted_fp = File.open(opts[:move_deleted], "a") if opts[:move_deleted] 
+  spam_fp = File.open(opts[:move_spam], "a") if opts[:move_spam]
+end
+
+begin
+  index.load
+
+  sources = ARGV.map do |uri|
+    s = index.source_for(uri) or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+    s.is_a?(Redwood::MBox::Loader) or Trollop::die "#{uri} is not an mbox source."
+    s
+  end
+
+  if sources.empty?
+    sources = index.usual_sources.select { |s| s.is_a? Redwood::MBox::Loader } 
+  end
+
+  modified_sources = []
+  sources.each do |source|
+    $stderr.puts "Scanning #{source}..."
+
+    unless ((opts[:delete_deleted] || opts[:move_deleted]) && index.has_any_from_source_with_label?(source, :deleted)) || ((opts[:delete_spam] || opts[:move_spam]) && index.has_any_from_source_with_label?(source, :spam))
+      $stderr.puts "Nothing to do from this source; skipping"
+      next
+    end
+
+    source.reset!
+    num_deleted = num_moved = num_scanned = 0
+    
+    out_fp = Tempfile.new "sup-sync-back-#{source.id}"
+    Redwood::PollManager.add_messages_from source do |m, offset, entry|
+      num_scanned += 1
+
+      if entry
+        labels = entry[:label].split.map { |x| x.intern }.to_boolean_h
+
+        if labels.member? :deleted
+          if opts[:delete_deleted]
+            puts "Dropping deleted message #{source}##{offset}" if opts[:verbose]
+            num_deleted += 1
+          elsif opts[:move_deleted] && labels.member?(:deleted)
+            puts "Moving deleted message #{source}##{offset}" if opts[:verbose]
+            save m, deleted_fp unless opts[:dry_run]
+            num_moved += 1
+          end
+
+        elsif labels.member? :spam
+          if opts[:delete_spam]
+            puts "Deleting spam message #{source}##{offset}" if opts[:verbose]
+            num_deleted += 1
+          elsif opts[:move_spam] && labels.member?(:spam)
+            puts "Moving spam message #{source}##{offset}" if opts[:verbose]
+            save m, spam_fp unless opts[:dry_run]
+            num_moved += 1
+          end
+        else
+          save m, out_fp unless opts[:dry_run]
+        end
+      else
+        save m, out_fp unless opts[:dry_run]
+      end
+
+      nil # don't actually add anything!
+    end
+    $stderr.puts "Scanned #{num_scanned}, deleted #{num_deleted}, moved #{num_moved} messages from #{source}."
+    modified_sources << source if num_deleted > 0 || num_moved > 0
+    out_fp.close unless opts[:dry_run]
+
+    unless opts[:dry_run] || (num_deleted == 0 && num_moved == 0)
+      deleted_fp.flush if deleted_fp
+      spam_fp.flush if spam_fp
+      $stderr.puts "Moving #{out_fp.path} to #{source.file_path}"
+      FileUtils.mv out_fp.path, source.file_path
+    end
+  end
+
+  unless opts[:dry_run]
+    deleted_fp.close if deleted_fp
+    spam_fp.close if spam_fp
+  end
+
+  $stderr.puts "Done."
+  unless modified_sources.empty?
+    $stderr.puts "You should now run: sup-sync --changed #{modified_sources.join(' ')}"
+  end
+rescue Exception => e
+  File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+  raise
+ensure
+  index.save
+  Redwood::finish
+  index.unlock
+end