sup 0.0.7 → 0.0.8

data/bin/sup-dump

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+$opts = Trollop::options do
+  version "sup-dump (sup #{Redwood::VERSION})"
+  banner <<EOS
+Dumps all message state from the sup index to standard out. You can
+later use sup-sync --restored --restore <filename> to recover the index.
+
+This tool is primarily useful in the event that a Ferret upgrade breaks
+the index format. This happened, for example, at Ferret version 0.11.
+
+Usage:
+  sup-dump > <filename>
+  sup-dump | bzip2 > <filename> # even better
+
+No options.
+EOS
+end
+
+index = Redwood::Index.new
+index.load
+
+(1 ... index.index.reader.max_doc).each do |i|
+  next if index.index.deleted? i
+  d = index.index[i]
+  puts [d[:message_id], "(" + d[:label] + ")"] * " "
+end

data/bin/sup-sync

@@ -0,0 +1,235 @@
+#!/usr/bin/env ruby
+
+require 'uri'
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+class Float
+  def to_s; sprintf '%.2f', self; end
+end
+
+class Numeric
+  def to_time_s
+    i = to_i
+    sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
+  end
+end
+
+def time
+  startt = Time.now
+  yield
+  Time.now - startt
+end
+
+opts = Trollop::options do
+  version "sup-sync (sup #{Redwood::VERSION})"
+  banner <<EOS
+Synchronizes the Sup index with one or more message sources by adding
+messages, deleting messages, or changing message state in the index as
+appropriate.
+
+"Message state" means read/unread, archived/inbox, starred/unstarred,
+and all user-defined labels on each message.
+
+"Default source state" refers to any state that a source itself has
+keeps about a message. Sup-sync uses this information when adding a
+new message to the index. The source state is typically limited to
+read/unread, archived/inbox status and a single label based on the
+source name. Messages using the default source state are placed in
+the inbox (i.e. not archived) and unstarred.
+
+Usage:
+  sup-sync [options] <source>*
+
+where <source>* is zero or more source URIs. If no sources are given,
+sync from all usual sources. All supported source URI schemes can
+be seen by running "sup-add --help".
+
+Options controlling WHICH messages sup-sync operates on:
+EOS
+  opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
+  opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source. (In the case of mbox sources, this includes all messages AFTER an altered message.)"
+  opt :restored, "Operate only on those messages included in a dump file as specified by --restore which have changed state."
+  opt :all, "Operate on all messages in the source, regardless of newness or changedness."
+  opt :start_at, "For --changed and --all, start at a particular offset.", :type => :int
+
+text <<EOS
+
+Options controlling HOW message state is altered:
+EOS
+  opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
+  opt :restore, "Restore message state from a dump file created with sup-dump. If a message is not in this dumpfile, act as --asis.", :type => String, :short => :none
+  opt :discard, "Discard any message state in the index and use the default source state. Dangerous!", :short => :none
+  opt :archive, "When using the default source state, mark messages as archived.", :short => "-x"
+  opt :read, "When using the default source state, mark messages as read."
+  opt :extra_labels, "When using the default source state, also apply these user-defined labels. Should be a comma-separated list.", :type => String, :short => :none
+
+text <<EOS
+
+Other options:
+EOS
+  opt :verbose, "Print message ids as they're processed."
+  opt :optimize, "As the final operation, optimize the index."
+  opt :all_sources, "Scan over all sources.", :short => :none
+  opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
+  opt :version, "Show version information", :short => :none
+
+  conflicts :changed, :all, :new, :restored
+  conflicts :asis, :restore, :discard
+end
+Trollop::die :restored, "requires --restore" if opts[:restored] unless opts[:restore]
+if opts[:start_at]
+  Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
+  Trollop::die :start_at, "requires either --changed or --all" unless opts[:changed] || opts[:all]
+end
+
+target = [:new, :changed, :all, :restored].find { |x| opts[x] } || :new
+op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
+
+Redwood::start
+index = Redwood::Index.new
+index.load
+
+restored_state =
+  if opts[:restore]
+    dump = {}
+    $stderr.puts "Loading state dump from #{opts[:restore]}..."
+    IO.foreach opts[:restore] do |l|
+      l =~ /^(\S+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
+      mid, labels = $1, $2
+      dump[mid] = labels.split(" ").map { |x| x.intern }
+    end
+    $stderr.puts "Read #{dump.size} entries from dump file."
+    dump
+  else
+    {}
+  end
+
+sources = ARGV.map do |uri|
+  uri = "mbox://#{uri}" unless uri =~ %r!://!
+  index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+end
+
+sources = index.usual_sources if sources.empty?
+sources = index.sources if opts[:all_sources]
+
+unless target == :new
+  if opts[:start_at]
+    sources.each { |s| s.seek_to! opts[:start_at] }
+  else
+    sources.each { |s| s.reset! }
+  end
+end
+
+seen = {}
+begin
+  sources.each do |source|
+    $stderr.puts "Scanning #{source}..."
+    num_added = num_updated = num_scanned = num_restored = 0
+    last_info_time = start_time = Time.now
+
+    Redwood::PollManager.add_messages_from source do |m, offset, entry|
+      num_scanned += 1
+      seen[m.id] = true
+
+      ## skip if we're operating only on changed messages, the message
+      ## is in the index, and it's unchanged from what the source is
+      ## reporting.
+      next if target == :changed && entry && entry[:source_id].to_i == source.id && entry[:source_info].to_i == offset
+
+      ## get the state currently in the index
+      index_state =
+        if entry
+          entry[:label].split(/\s+/).map { |x| x.intern }
+        else
+          nil
+        end
+
+      ## skip if we're operating on restored messages, and this one
+      ## ain't.
+      next if target == :restored && (!restored_state[m.id] || restored_state[m.id].sort_by { |s| s.to_s } == index_state.sort_by { |s| s.to_s })
+
+      ## m.labels is the default source labels. tweak these according
+      ## to default source state modification flags.
+      m.labels -= [:inbox] if opts[:archive]
+      m.labels -= [:unread] if opts[:read]
+      m.labels += opts[:extra_labels].split(/\s*,\s*/).map { |x| x.intern } if opts[:extra_labels]
+
+      ## assign message labels based on the operation we're performing
+      case op
+      when :asis
+        m.labels = index_state if index_state
+      when :restore
+        ## if the entry exists on disk
+        if restored_state[m.id]
+          m.labels = restored_state[m.id]
+          num_restored += 1
+        elsif index_state
+          m.labels = index_state
+        end
+      when :discard
+        ## nothin! use default source labels
+      end
+
+      if Time.now - last_info_time > 60
+        last_info_time = Time.now
+        elapsed = last_info_time - start_time
+        pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
+        remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+        $stderr.puts "## #{num_added + num_updated} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining (for this source)"
+      end
+
+      if index_state.nil?
+        puts "Adding message #{source}##{offset} with state {#{m.labels * ', '}}" if opts[:verbose]
+        num_added += 1
+      else
+        puts "Updating message #{source}##{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
+        num_updated += 1
+      end
+
+      opts[:dry_run] ? nil : m
+    end
+    $stderr.puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated} messages from #{source}."
+    $stderr.puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
+  end
+rescue Exception => e
+  File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+  raise
+ensure
+  index.save
+  Redwood::finish
+end
+
+## delete any messages in the index that claim they're from one of
+## these sources, but that we didn't see.
+##
+## kinda crappy code here, because we delve directly into the Ferret
+## API.
+##
+## TODO: move this to Index, i suppose.
+if target == :all || target == :changed
+  $stderr.puts "Deleting missing messages from the index..."
+  num_del, num_scanned = 0, 0
+  sources.each do |source|
+    raise "no source id for #{source}" unless source.id
+    q = "+source_id:#{source.id}"
+    q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
+    index.index.search_each(q, :limit => :all) do |docid, score|
+      num_scanned += 1
+      mid = index.index[docid][:message_id]
+      unless seen[mid]
+        puts "Deleting #{mid}" if opts[:verbose]
+        index.index.delete docid unless opts[:dry_run]
+        num_del += 1
+      end
+    end
+  end
+  $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
+end
+
+if opts[:optimize]
+  $stderr.puts "Optimizing index..."
+  optt = time { index.index.optimize unless opts[:dry_run] }
+  $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
+end

data/doc/FAQ.txt

@@ -8,11 +8,9 @@ Q: If you love GMail so much, why not just use it?
 A: I hate ads, I hate using a mouse, and I hate non-programmability
    and non-extensibility.
 
-   Also, GMail encourages top-posting in a variety of ways. THIS
-   CANNOT BE TOLERATED!
+   Also, GMail encourages top-posting. THIS CANNOT BE TOLERATED!
 
 Q: Why the console?
-
 A: Because a keystroke is with a hundred mouse clicks (as any Unix
    user knows). Because you don't need web browser. Because you get
    instantaneous response and a simple interface.
@@ -23,33 +21,71 @@ A: You can manually mark messages as spam, which prevents them from
    filtering should be done by a dedicated tool like SpamAssassin.
 
 Q: How do I delete a message?
-A: Press the 'd' key.
+A: Why delete? Unless it's spam, you might as well just archive it.
+
+Q: C'mon, really now!
+A: Ok, press the 'd' key.
 
 Q: But I want to delete it for real, not just add a 'deleted' flag in
    the index. I want it gone from disk!
-A: Deleting a message is an old-fashioned concept. In the modern
-   world, disk space is cheap enough that you should never have to
-   delete a message. If it's spam, save it for future analysis.
-
-Q: C'mon, really now!
 A: Ok, at some point I plan to have a batch deletion tool that will
    run through a source and delete all messages that have a 'spam' or
-   'deleted' tags (and, for mbox sources, will update the offsets of
-   all later messages). But that doesn't exist yet.
+   'deleted' tags. But that doesn't exist yet.
 
-Q: I got some error message about needing to run sup-import --rescan
+Q: I got some error message about needing to run sup-sync --changed
    when I tried to read a message. What's that about?
 A: If messages have been moved, deleted, or altered in a source, Sup
    may have to rebuild its index for that source. For example, for
-   mbox files, even reading a message changes the offsets of every
-   file on disk. Rather than rescanning every time, Sup assumes
+   mbox files, reading a single unread message changes the offsets of
+   every file on disk. Rather than rescanning every time, Sup assumes
    sources don't change except by having new messages added. If that
-   assumption is violated, you'll have to run sup-import --rescan.
+   assumption is violated, you'll have to sync the index.
 
    The alternative is to rescan every source when Sup starts
    up. Because Sup is designed to work with arbitrarily large mbox
    files, this would not be a good idea.
 
+Q: How do I back up my index?
+Q: How do I make a state dump?
+A: Since the contents of the messages are recoverable from their
+   sources using sup-sync, all you need to back up is the message
+   state. To do this, simply run:
+      sup-dump > <dumpfile>
+   This will save all message state in a big text file, which you
+   should probably compress.
+
+Q: How do I restore the message state I saved in my state dump?
+A: Run:
+     sup-sync [<source>+] --restored --restore <dumpfile>
+   where <dumpfile> was created as above.
+
+Q: I see this message from Ferret:
+     Error occured in index.c:825 - sis_find_segments_file
+A: Yikes! You've upgraded Ferret and the index format changed beneath
+   you. Follow the index rebuild instructions below.
+
+Q: I upgraded Ferret and the index format changed. I need to
+   completely rebuild my index. How do I do this?
+A: First, you'll need a complete state dump. If you haven't made
+   one, you'll need to downgrade Ferret and make a state dump as
+   above. Then run these commands:
+     rm -rf ~/.sup/ferret    # omg wtf
+     sup-sync --all-sources --all --restore <dumpfile>
+   Voila! A brand new index.
+
+Q: I want to move messages from one source to another. (E.g., my
+   primary inbox is an IMAP server with a quota, and I want to move
+   some of those messages to local mbox files.) How do I do that while
+   preserving message state?
+A: Move the messages from the source to the target using whatever tool
+   you'd like. Then (and this is the important part), run:
+     sup-sync --changed <source1> <source2>
+
+   If you sup-sync only one source at a time, depending on the order,
+   the messages may be treated as missing and then deleted from the
+   index, which means that their state will be lost when you sync the
+   other source.
+
 Q: What are all these "Redwood" references I see in the code?
 A: That was Sup's original name. (Think pine, elm. Although I am a
    Mutt user, I couldn't think of a good progression there.) But it was
@@ -59,18 +95,9 @@ A: That was Sup's original name. (Think pine, elm. Although I am a
    Maybe one day I'll do a huge search-and-replace on the code, but it
    doesn't seem that important at this point.
 
-Q: I want to move messages from one source to another. (E.g., my
-   primary inbox is an IMAP server with a quota, and I want to move
-   some of those messages to local mbox files.) How do I do that while
-   preserving message state?
-A: Move the messages from the source to the target using whatever tool
-   you'd like. Then (and this is the important part), sup-import
-   --rebuild both sources at once. If you do it one at a time, you may
-   lose message state. (Depending, actually, on which order you do it
-   in. But just do them both at once.)
-
 Q: How is Sup possible?
 A: Sup is only possible through the hard work of Dave Balmain, the
    author of ferret, which is the search engine behind Sup. Ferret is
    really a first-class piece of software, and it's due to the
    tremendous amount of time and effort he's put in to it.
+

data/doc/Philosophy.txt

@@ -1,24 +1,24 @@
-Should an email client have a philosophy? I think so. For many people,
-email is one of our primary means of communication. Something so
-important ought to warrant a little thought.
+Should an email client have a philosophy? For many people, email is
+one of our primary means of communication. Something so important
+ought to warrant a little thought.
 
-So here's Sup's philosophy.
+Here's Sup's philosophy.
 
 Using "traditional" email clients today is increasingly problematic.
 Anyone who's on a high-traffic mailing list knows this.  My ruby-talk
-folder is 350 megs and Mutt sits there for 60 seconds while it opens
+folder is 430 megs and Mutt sits there for 60 seconds while it opens
 it. Keeping up with the all the new traffic is painful, even with
 Mutt's excellent threading features, simply because there's so much of
 it. A single thread can span several pages in the folder index view
 alone! And Mutt is probably the fastest email client out there, and
-the most featureful and in terms of threading and mailing list
+certainly most featureful and in terms of threading and mailing list
 support. God help me if I try and throw Thunderbird at that.
 
 The principle problem with traditional clients is that they deal with
 individual pieces of email. This places a high mental cost on the user
 for each incoming email, by forcing them to ask: Should I keep this
-email, or delete it? If I keep it, where should I file it?
-For example, I've spent the last 10 years of my life laboriously
+email, or delete it? If I keep it, where should I file it?  For
+example, I've spent the last 10 years of my life laboriously
 hand-filing every email message I received and feeling a mild sense of
 panic every time an email was both "from Mom" and "about school". The
 massive amounts of email that many people receive, and the cheap cost
@@ -26,14 +26,14 @@ of storage, have made these questions both more costly and less useful
 to answer.
 
 I think GMail has taken the right approach. As a long-time Mutt user,
-I was blown away when I first saw people use GMail, because I saw them
-treat their email differently from how I had ever treated mine. I saw
+I was pretty much blown away when I first saw people use GMail,
+because I saw them treat email differently from how I ever had.  I saw
 that making certain operations quantitatively easier (namely, search)
-resulted in a qualitative difference in usage. Uou didn't have to
+resulted in a qualitative difference in usage. You didn't have to
 worry about filing things into folders correctly, because you could
-just find things later by searching for them. I also saw that
-thread-centrism had many advantages over message-centrism when message
-volume was high.
+just find things later by searching for them. I also saw how
+thread-centrism was advantageous over message-centrism when message
+volume was high: if nothing else, there's simply less of them.
 
 Much of the inspiration for Sup was based on GMail. I think it's to
 the GMail designers' credit that they started with a somewhat ad-hoc
@@ -43,11 +43,11 @@ actually better than everything else out there. At least, that's how I
 imagine in happened. Maybe they knew what they were doing from the
 start.
 
-But ultimately, GMail wasn't right for me, which is why the idea for
-Sup was born.
+But ultimately, GMail wasn't right for me (fuck top posting and HTML
+mail), which is why the idea for Sup was born.
 
-Sup is based on the following principles, which I more or less stole
-directly from GMail:
+Sup is based on the following principles, which I stole directly from
+GMail:
 
 - An immediately accessible and fast search capability over the
   entire email archive eliminates most of the need for folders,
@@ -65,4 +65,4 @@ do with the fantastic productivity of a console- and keyboard-based
 application, the usefulness of multiple buffers, the necessity of
 handling multiple email accounts, etc. But those are just details!
 
-So give it a go and let me know what you think.
+Let me know what you think.