sup 0.19.0

data/bin/sup-sync

@@ -0,0 +1,210 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'uri'
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+PROGRESS_UPDATE_INTERVAL = 15 # seconds
+
+class Float
+  def to_s; sprintf '%.2f', self; end
+  def to_time_s; infinite? ? "unknown" : super end
+end
+
+class Numeric
+  def to_time_s
+    i = to_i
+    sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
+  end
+end
+
+class Set
+  def to_s; to_a * ',' end
+end
+
+def time
+  startt = Time.now
+  yield
+  Time.now - startt
+end
+
+opts = Trollop::options do
+  version "sup-sync (sup #{Redwood::VERSION})"
+  banner <<EOS
+Synchronizes the Sup index with one or more message sources by adding
+messages, deleting messages, or changing message state in the index as
+appropriate.
+
+"Message state" means read/unread, archived/inbox, starred/unstarred,
+and all user-defined labels on each message.
+
+"Default source state" refers to any state that a source itself has
+keeps about a message. Sup-sync uses this information when adding a
+new message to the index. The source state is typically limited to
+read/unread, archived/inbox status and a single label based on the
+source name. Messages using the default source state are placed in
+the inbox (i.e. not archived) and unstarred.
+
+Usage:
+  sup-sync [options] <source>*
+
+where <source>* is zero or more source URIs. If no sources are given,
+sync from all usual sources. Supported source URI schemes can be seen
+by running "sup-add --help".
+
+Options controlling HOW message state is altered:
+EOS
+  opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
+  opt :restore, "Restore message state from a dump file created with sup-dump. If a message is not in this dumpfile, act as --asis.", :type => String, :short => :none
+  opt :discard, "Discard any message state in the index and use the default source state. Dangerous!", :short => :none
+  opt :archive, "When using the default source state, mark messages as archived.", :short => "-x"
+  opt :read, "When using the default source state, mark messages as read."
+  opt :extra_labels, "When using the default source state, also apply these user-defined labels (a comma-separated list)", :default => "", :short => :none
+
+text <<EOS
+
+Other options:
+EOS
+  opt :verbose, "Print message ids as they're processed."
+  opt :optimize, "As the final operation, optimize the index."
+  opt :all_sources, "Scan over all sources.", :short => :none
+  opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
+  opt :version, "Show version information", :short => :none
+
+  conflicts :asis, :restore, :discard
+end
+
+op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
+
+Redwood::start
+index = Redwood::Index.init
+
+restored_state = if opts[:restore]
+  dump = {}
+  puts "Loading state dump from #{opts[:restore]}..."
+  IO.foreach opts[:restore] do |l|
+    l =~ /^(\S+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
+    mid, labels = $1, $2
+    dump[mid] = labels.to_set_of_symbols
+  end
+  puts "Read #{dump.size} entries from dump file."
+  dump
+else
+  {}
+end
+
+seen = {}
+index.lock_interactively or exit
+begin
+  index.load
+
+  if(s = Redwood::SourceManager.source_for Redwood::SentManager.source_uri)
+    Redwood::SentManager.source = s
+  else
+    Redwood::SourceManager.add_source Redwood::SentManager.default_source
+  end
+
+  sources = if opts[:all_sources]
+    Redwood::SourceManager.sources
+  elsif ARGV.empty?
+    Redwood::SourceManager.usual_sources
+  else
+    ARGV.map do |uri|
+      Redwood::SourceManager.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+    end
+  end
+
+  sources.each do |source|
+    puts "Scanning #{source}..."
+    num_added = num_updated = num_deleted = num_scanned = num_restored = 0
+    last_info_time = start_time = Time.now
+
+    Redwood::PollManager.poll_from source do |action,m,old_m,progress|
+      num_scanned += 1
+      if action == :delete
+        num_deleted += 1
+        puts "Deleting #{m.id}" if opts[:verbose]
+      elsif action == :add
+        seen[m.id] = true
+
+        ## tweak source labels according to commandline arguments if necessary
+        m.labels.delete :inbox if opts[:archive]
+        m.labels.delete :unread if opts[:read]
+        m.labels += opts[:extra_labels].to_set_of_symbols(",")
+
+        ## decide what to do based on message labels and the operation we're performing
+        dothis = case
+        when (op == :restore) && restored_state[m.id]
+          if old_m && (old_m.labels != restored_state[m.id])
+            num_restored += 1
+            m.labels = restored_state[m.id]
+            :update_message_state
+          elsif old_m.nil?
+            num_restored += 1
+            m.labels = restored_state[m.id]
+            :add_message
+          else
+            # labels are the same; don't do anything
+          end
+        when op == :discard
+          if old_m && (old_m.labels != m.labels)
+            :update_message_state
+          else
+            # labels are the same; don't do anything
+          end
+        else
+          if old_m
+            :update_message
+          else
+            :add_message
+          end
+        end
+
+        ## now, actually do the operation
+        case dothis
+        when :add_message
+          puts "Adding new message #{source}##{m.source_info} with labels #{m.labels}" if opts[:verbose]
+          num_added += 1
+        when :update_message
+          puts "Updating message #{source}##{m.source_info}; labels #{old_m.labels} => #{m.labels}; offset #{old_m.source_info} => #{m.source_info}" if opts[:verbose]
+          num_updated += 1
+        when :update_message_state
+          puts "Changing flags for #{source}##{m.source_info} from #{old_m.labels} to #{m.labels}" if opts[:verbose]
+          num_updated += 1
+        end
+      else fail "sup-sync cannot handle :update's"
+      end
+
+      if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
+        last_info_time = Time.now
+        elapsed = last_info_time - start_time
+        pctdone = progress * 100.0
+        remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+        printf "## scanned %dm (~%.0f%%) @ %.1fm/s. %s elapsed, ~%s remaining\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
+      end
+      next if opts[:dry_run]
+    end
+
+    puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated}, deleted #{num_deleted} messages from #{source}."
+    puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
+  end
+
+  index.save
+
+  if opts[:optimize]
+    puts "Optimizing index..."
+    optt = time { index.optimize unless opts[:dry_run] }
+    puts "Optimized index of size #{index.size} in #{optt}s."
+  end
+rescue Redwood::FatalSourceError => e
+  $stderr.puts "Sorry, I couldn't communicate with a source: #{e.message}"
+rescue Exception => e
+  File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+  raise
+ensure
+  Redwood::finish
+  index.unlock
+end

data/bin/sup-sync-back-maildir

@@ -0,0 +1,127 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+opts = Trollop::options do
+  version "sup-sync-back-maildir (sup #{Redwood::VERSION})"
+  banner <<EOS
+Export Xapian entries to Maildir sources on disk.
+
+This script parses the Xapian entries for a given Maildir source and renames
+(changes maildir flags) e-mail files on disk according to the labels stored in
+the index. It will export all the changes you made in Sup to your
+Maildirs so that they can be propagated to your IMAP server with e.g. offlineimap.
+
+The script also merges some Maildir flags into Sup such
+as R (replied) and P (passed, forwarded), for instance suppose you
+have an e-mail file like this: foo_bar:2,FRS (flags are favorite,
+replied, seen) and its Xapian entry has labels 'starred', the merge
+operation will add the 'replied' label to the Xapian entry.
+
+If you choose not to merge (-m) you will lose information ('replied'), and in
+the previous example the file will be renamed to foo_bar:2,FS.
+
+Running this script is *strongly* recommended when setting the
+"sync_back_to_maildir" option from false to true in config.yaml or changing the
+"sync_back" flag to true for a source in sources.yaml.
+
+Usage:
+  sup-sync-back-maildir [options] <source>*
+
+where <source>* is source URIs. If no source is given, the default behavior is
+to sync back all Maildir sources marked as usual and that have not disabled
+sync back using the configuration parameter sync_back = false in sources.yaml.
+
+Options include:
+EOS
+  opt :no_confirm, "Don't ask for confirmation before synchronizing", :default => false, :short => "n"
+  opt :no_merge, "Don't merge new supported Maildir flags (R and P)", :default => false, :short => "m"
+  opt :list_sources, "List your Maildir sources and exit", :default => false, :short => "l"
+  opt :unusual_sources_too, "Sync unusual sources too if no specific source information is given", :default => false, :short => "u"
+end
+
+def die msg
+  $stderr.puts "Error: #{msg}"
+  exit(-1)
+end
+
+Redwood::start true
+index = Redwood::Index.init
+index.lock_interactively or exit
+index.load
+
+## Force sync_back_to_maildir option otherwise nothing will happen
+$config[:sync_back_to_maildir] = true
+
+begin
+  sync_performed = []
+  sync_performed = File.readlines(Redwood::SYNC_OK_FN).collect { |e| e.strip }.find_all { |e| not e.empty? } if File.exists? Redwood::SYNC_OK_FN
+  sources = []
+
+  ## Try to find out sources given in parameters
+  sources = ARGV.map do |uri|
+    s = Redwood::SourceManager.source_for(uri) or die "unknown source: #{uri}. Did you add it with sup-add first?"
+    s.is_a?(Redwood::Maildir) or die "#{uri} is not a Maildir source."
+    s.sync_back_enabled? or die "#{uri} has disabled sync back - check your configuration."
+    s
+  end unless opts[:list_sources]
+
+  ## Otherwise, check all sources in sources.yaml
+  if sources.empty? or opts[:list_sources] == true
+    if opts[:unusual_sources_too]
+      sources = Redwood::SourceManager.sources.select do |s|
+        s.is_a? Redwood::Maildir and s.sync_back_enabled?
+      end
+    else
+      sources = Redwood::SourceManager.usual_sources.select do |s|
+        s.is_a? Redwood::Maildir and s.sync_back_enabled?
+      end
+    end
+  end
+
+  if opts[:list_sources] == true
+    sources.each do |s|
+      puts "id: #{s.id}, uri: #{s.uri}"
+    end
+  else
+    sources.each do |s|
+      if opts[:no_confirm] == false
+        print "Are you sure you want to synchronize '#{s.uri}'? (Y/n) "
+        next if STDIN.gets.chomp.downcase == 'n'
+      end
+
+      infos = Enumerator.new(index, :each_source_info, s.id).to_a
+      counter = 0
+      infos.each do |info|
+        print "\rSynchronizing '#{s.uri}'... #{((counter += 1)/infos.size.to_f*100).to_i}%"
+        index.each_message({:location => [s.id, info]}, false) do |m|
+          if opts[:no_merge] == false
+            m.merge_labels_from_locations [:replied, :forwarded]
+          end
+
+          if Redwood::Index.message_joining_killed? m
+            m.labels += [:killed]
+          end
+
+          index.save_message m
+        end
+      end
+      print "\n"
+      sync_performed << s.uri
+    end
+    ## Write a flag file to tell sup that the synchronization has been performed
+    File.open(Redwood::SYNC_OK_FN, 'w') {|f| f.write(sync_performed.join("\n")) }
+  end
+rescue Exception => e
+  File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+  raise
+ensure
+  index.save_index
+  Redwood::finish
+  index.unlock
+end

data/bin/sup-tweak-labels

@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+class Float
+  def to_s; sprintf '%.2f', self; end
+   def to_time_s
+     infinite? ? "unknown" : super
+   end
+end
+
+class Numeric
+  def to_time_s
+    i = to_i
+    sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
+  end
+end
+
+def time
+  startt = Time.now
+  yield
+  Time.now - startt
+end
+
+opts = Trollop::options do
+  version "sup-tweak-labels (sup #{Redwood::VERSION})"
+  banner <<EOS
+Batch modification of message state for messages already in the index.
+
+Usage:
+  sup-tweak-labels [options] <source>*
+
+where <source>* is zero or more source URIs. Supported source URI schemes can
+be seen by running "sup-add --help".
+
+Options:
+EOS
+  opt :add, "One or more labels (comma-separated) to add to every message from the specified sources", :default => ""
+  opt :remove, "One or more labels (comma-separated) to remove from every message from the specified sources, if those labels are present", :default => ""
+  opt :query, "A Sup search query", :type => String
+
+  text <<EOS
+
+Other options:
+EOS
+  opt :verbose, "Print message ids as they're processed."
+  opt :very_verbose, "Print message names and subjects as they're processed."
+  opt :all_sources, "Scan over all sources.", :short => :none
+  opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
+  opt :no_sync_back, "Do not sync back to the original Maildir."
+  opt :version, "Show version information", :short => :none
+end
+opts[:verbose] = true if opts[:very_verbose]
+
+add_labels = opts[:add].to_set_of_symbols ","
+remove_labels = opts[:remove].to_set_of_symbols ","
+
+Trollop::die "nothing to do: no labels to add or remove" if add_labels.empty? && remove_labels.empty?
+
+Redwood::start
+index = Redwood::Index.init
+index.lock_interactively or exit
+
+begin
+  index.load
+
+  source_ids = if opts[:all_sources]
+    Redwood::SourceManager.sources
+  else
+    ARGV.map do |uri|
+      Redwood::SourceManager.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+    end
+  end.map { |s| s.id }
+  Trollop::die "nothing to do: no sources" if source_ids.empty?
+
+  query = "(" + source_ids.map { |id| "source_id:#{id}" }.join(" OR ") + ")"
+  if add_labels.empty?
+    ## if all we're doing is removing labels, we can further restrict the
+    ## query to only messages with those labels
+    query += " (" + remove_labels.map { |l| "label:#{l}" }.join(" OR ") + ")"
+  end
+  query += ' ' + opts[:query] if opts[:query]
+
+  parsed_query = index.parse_query query
+  parsed_query.merge! :load_spam => true, :load_deleted => true, :load_killed => true
+  ids = index.to_enum(:each_id, parsed_query)
+  num_total = index.num_results_for parsed_query
+
+  $stderr.puts "Found #{num_total} documents across #{source_ids.length} sources. Scanning..."
+
+  num_changed = num_scanned = 0
+  last_info_time = start_time = Time.now
+  ids.each do |id|
+    num_scanned += 1
+
+    m = index.build_message id
+    old_labels = m.labels.dup
+
+    m.labels += add_labels
+    m.labels -= remove_labels
+
+    unless m.labels == old_labels
+      num_changed += 1
+      puts "From #{m.from}, subject: #{m.subj}" if opts[:very_verbose]
+      puts "#{m.id}: {#{old_labels.to_a.join ','}} => {#{m.labels.to_a.join ','}}" if opts[:verbose]
+      puts if opts[:very_verbose]
+      unless opts[:dry_run]
+        index.update_message_state [m, false]
+        m.sync_back unless opts[:no_sync_back]
+      end
+    end
+
+    if Time.now - last_info_time > 60
+      last_info_time = Time.now
+      elapsed = last_info_time - start_time
+      pctdone = 100.0 * num_scanned.to_f / num_total.to_f
+      remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+      $stderr.puts "## #{num_scanned} (#{pctdone}%) read; #{elapsed.to_time_s} elapsed; #{remaining.to_time_s} remaining"
+    end
+  end
+  $stderr.puts "Scanned #{num_scanned} / #{num_total} messages and changed #{num_changed}."
+
+  unless num_changed == 0
+    $stderr.puts "Optimizing index..."
+    index.optimize unless opts[:dry_run]
+  end
+
+rescue Exception => e
+  File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+  raise
+ensure
+  index.save
+  Redwood::finish
+  index.unlock
+end
+