sup 0.0.6 → 0.0.7

data/lib/sup/person.rb

@@ -14,7 +14,7 @@ class PersonManager
   def register email, name
     return unless name
 
-    name = name.gsub(/^\s+|\s+$/, "").gsub(/\s+/, " ")
+    name = name.gsub(/^\s+|\s+$/, "").gsub(/\s+/, " ").gsub(/^['"]|['"]$/, "")
 
     ## all else being equal, prefer longer names, unless the prior name
     ## doesn't contain any capitalization

data/lib/sup/poll.rb

@@ -24,7 +24,7 @@ class PollManager
     if num > 0
       BufferManager.flash "Loaded #{num} new messages, #{numi} to inbox." 
     else
-      BufferManager.flash "No new messages."
+      BufferManager.flash "No new messages." 
     end
     [num, numi]
   end
@@ -38,59 +38,27 @@ class PollManager
     end
   end
 
-  ## TODO: merge this with sup-import
   def do_poll
     total_num = total_numi = 0
     @mutex.synchronize do
-      found = {}
       Index.usual_sources.each do |source|
-        next if source.broken? || source.done?
-
-        yield "Loading from #{source}... "
-        start_offset = nil
+#        yield "source #{source} is done? #{source.done?} (cur_offset #{source.cur_offset} >= #{source.end_offset})"
+        yield "Loading from #{source}... " unless source.done? || source.broken?
         num = 0
-        num_inbox = 0
-
-        source.each do |offset, labels|
-          break if source.broken?
-          start_offset ||= offset
-          yield "Found message at #{offset} with labels #{labels * ', '}"
-
-          begin
-            begin
-              m = Redwood::Message.new :source => source, :source_info => offset, :labels => labels
-            rescue MessageFormatError => e
-              yield "Non-fatal error loading message #{source}##{offset}: #{e.message}"
-              next
-            end
-
-            if found[m.id]
-              yield "Skipping duplicate message #{m.id}"
-              next
-            end
-            found[m.id] = true
-          
-            if Index.add_message m
-              UpdateManager.relay :add, m
-              num += 1
-              total_num += 1
-              total_numi += 1 if m.labels.include? :inbox
-            end
-        
-            if num % 1000 == 0 && num > 0
-              elapsed = Time.now - start
-              pctdone = source.pct_done
-              remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
-              yield "## #{num} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
-            end
-          rescue SourceError => e
-            msg = "Fatal error loading from #{source}: #{e.message}"
-            Redwood::log msg
-            yield msg
-            break
+        numi = 0
+        add_new_messages_from source do |m, offset, entry|
+          ## always preserve the labels on disk.
+          m.labels = entry[:label].split(/\s+/).map { |x| x.intern } if entry
+          yield "Found message at #{offset} with labels {#{m.labels * ', '}}"
+          unless entry
+            num += 1
+            numi += 1 if m.labels.include? :inbox
           end
+          m
         end
-        yield "Found #{num} messages" unless num == 0
+        yield "Found #{num} messages, #{numi} to inbox" unless num == 0
+        total_num += num
+        total_numi += numi
       end
 
       yield "Done polling; loaded #{total_num} new messages total"
@@ -99,6 +67,51 @@ class PollManager
     end
     [total_num, total_numi]
   end
+
+  ## this is the main mechanism for adding new messages to the
+  ## index. it's called both by sup-import and by PollMode.
+  ##
+  ## for each new message in the source, this yields the message, the
+  ## source offset, and the index entry on disk (if any). it expects
+  ## the yield to return the message (possibly altered in some way),
+  ## and then adds it (if new) or updates it (if previously seen).
+  ##
+  ## the labels of the yielded message are the source labels. it is
+  ## likely that callers will want to replace these with the index
+  ## labels, if they exist, so that state is not lost when e.g. a new
+  ## version of a message from a mailing list comes in.
+  def add_new_messages_from source
+    return if source.done? || source.broken?
+
+    source.each do |offset, labels|
+      if source.broken?
+        Redwood::log "error loading messages from #{source}: #{source.broken_msg}"
+        return
+      end
+      
+      labels.each { |l| LabelManager << l }
+
+      begin
+        m = Message.new :source => source, :source_info => offset, :labels => labels
+        if m.source_marked_read?
+          m.remove_label :unread
+          labels.delete :unread
+        end
+
+        docid, entry = Index.load_entry_for_id m.id
+        m = yield m, offset, entry
+        next unless m
+        if entry
+          Index.update_message m, docid, entry
+        else
+          Index.add_message m
+          UpdateManager.relay self, :add, m
+        end
+      rescue MessageFormatError, SourceError => e
+        Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}"
+      end
+    end
+  end
 end
 
 end

data/lib/sup/sent.rb

@@ -10,7 +10,7 @@ class SentManager
     self.class.i_am_the_instance self
   end
 
-  def self.source_name; "sent://"; end
+  def self.source_name; "sup://sent"; end
   def self.source_id; 9998; end
   def new_source; @source = SentLoader.new; end
 
@@ -24,7 +24,7 @@ class SentManager
     @source.each do |offset, labels|
       m = Message.new :source => @source, :source_info => offset, :labels => labels
       Index.add_message m
-      UpdateManager.relay :add, m
+      UpdateManager.relay self, :add, m
     end
   end
 end
@@ -36,6 +36,7 @@ class SentLoader < MBox::Loader
     super "mbox://" + filename, cur_offset, true, true
   end
 
+  def uri; SentManager.source_name; end
   def to_s; SentManager.source_name; end
   def id; SentManager.source_id; end
   def labels; [:sent, :inbox]; end

data/lib/sup/source.rb

@@ -10,8 +10,8 @@ class Source
   ##  3. (optional) see whether the source has marked it read or not
   ##
   ## In particular, Sup doesn't need to move messages, mark them as
-  ## read, delete them, or anything else. (Well, maybe delete at some
-  ## point.)
+  ## read, delete them, or anything else. (Well, at some point it will
+  ## need to delete them, but that will be an optional capability.)
   ##
   ## On the other hand, Sup assumes that you can assign each message a
   ## unique integer id, such that newer messages have higher ids than
@@ -22,16 +22,19 @@ class Source
   ## capability, e.g. IMAP, then you have to do a little more work to
   ## simulate it.
   ##
-  ## To write a new source, subclass this class, and should implement:
+  ## To write a new source, subclass this class, and implement:
   ##
   ## - start_offset
-  ## - end_offset
+  ## - end_offset (exclusive!)
   ## - load_header offset
   ## - load_message offset
   ## - raw_header offset
-  ## - raw_full_message offset-
+  ## - raw_full_message offset
   ## - next (or each, if you prefer)
   ##
+  ## ... where "offset" really means unique id. (You can tell I
+  ## started with mbox.)
+  ##
   ## You can throw SourceErrors from any of those, but we don't catch
   ## anything else, so make sure you catch *all* errors and reraise
   ## them as SourceErrors, and set broken_msg to something if the
@@ -39,6 +42,11 @@ class Source
   ##
   ## Also, be sure to make the source thread-safe, since it WILL be
   ## pummeled from multiple threads at once.
+  ##
+  ## Two examples for you to look at, though sadly neither of them is
+  ## as simple as I'd like: mbox/loader.rb and imap.rb
+
+
 
   ## dirty? described whether cur_offset has changed, which means the
   ## source info needs to be re-saved to sources.yaml.
@@ -78,7 +86,7 @@ class Source
       true
     end
   end
-  def is_source_for? s; to_s == s; end
+  def is_source_for? uri; URI(self.uri) == URI(uri); end
 
   def each
     return if broken?

data/lib/sup/textfield.rb

@@ -29,6 +29,7 @@ class TextField
     @w.mvaddstr @y, 0, @question
     Ncurses.curs_set 1
     Ncurses::Form.form_driver @form, Ncurses::Form::REQ_END_FIELD
+    Ncurses::Form.form_driver @form, Ncurses::Form::REQ_NEXT_CHAR if @history[@i] =~ / $/ # fucking RETARDED!!!!
   end
 
   def deactivate

data/lib/sup/thread.rb

@@ -1,4 +1,24 @@
-require 'date'
+## Herein is all the code responsible for threading messages. I use an
+## online version of the JWZ threading algorithm:
+## http://www.jwz.org/doc/threading.html
+##
+## I certainly didn't implement it for efficiency, but thanks to our
+## search engine backend, it's typically not applied to very many
+## messages at once.
+
+## At the top level, we have a ThreadSet. A ThreadSet represents a set
+## of threads, e.g. a message folder or an inbox. Each ThreadSet
+## contains zero or more Threads. A Thread represents all the message
+## related to a particular subject. Each Thread has one or more
+## Containers. A Container is a recursive structure that holds the
+## tree structure as determined by the references: and in-reply-to:
+## headers. A Thread with multiple Containers occurs if they have the
+## same subject, but (most likely due to someone using a primitive
+## MUA) we don't have evidence from in-reply-to: or references:
+## headers, only subject: (and thus our tree is probably broken). A
+## Container holds zero or one message. In the case of no message, it
+## means we've seen a reference to the message but haven't seen the
+## message itself (yet).
 
 module Redwood
 
@@ -30,9 +50,8 @@ class Thread
     puts "=== end thread ==="
   end
 
-  ## yields each message, its depth, and its parent
-  ## note that the message can be a Message object, or :fake_root,
-  ## or nil.
+  ## yields each message, its depth, and its parent.  note that the
+  ## message can be a Message object, or :fake_root, or nil.
   def each fake_root=false
     adj = 0
     root = @containers.find_all { |c| !Message.subj_is_reply?(c) }.argmin { |c| c.date }
@@ -59,15 +78,14 @@ class Thread
     end
   end
 
+  def first; each { |m, *o| return m if m }; nil; end
   def dirty?; any? { |m, *o| m && m.dirty? }; end
   def date; map { |m, *o| m.date if m }.compact.max; end
   def snippet; argfind { |m, *o| m && m.snippet }; end
   def authors; map { |m, *o| m.from if m }.compact.uniq; end
 
   def apply_label t; each { |m, *o| m && m.add_label(t) }; end
-  def remove_label t
-    each { |m, *o| m && m.remove_label(t) }
-  end
+  def remove_label t; each { |m, *o| m && m.remove_label(t) }; end
 
   def toggle_label label
     if has_label? label
@@ -82,7 +100,6 @@ class Thread
   def set_labels l; each { |m, *o| m && m.labels = l }; end
   
   def has_label? t; any? { |m, *o| m && m.has_label?(t) }; end
-  def dirty?; any? { |m, *o| m && m.dirty? }; end
   def save index; each { |m, *o| m && m.save(index) }; end
 
   def direct_participants
@@ -220,13 +237,13 @@ class ThreadSet
     (c = @messages[m.id]) && c.root.thread
   end
 
-  def delete_empties
-    @subj_thread.each { |k, v| @subj_thread.delete(k) if v.empty? }
+  def delete_cruft
+    @subj_thread.each { |k, v| @subj_thread.delete(k) if v.empty? || v.subj != k }
   end
-  private :delete_empties
+  private :delete_cruft
 
-  def threads; delete_empties; @subj_thread.values; end
-  def size; delete_empties; @subj_thread.size; end
+  def threads; delete_cruft; @subj_thread.values; end
+  def size; delete_cruft; @subj_thread.size; end
 
   def dump
     @subj_thread.each do |s, t|
@@ -273,19 +290,25 @@ class ThreadSet
 
       m = builder.call
       add_message m
-      load_thread_for_message m
+      load_thread_for_message m, :load_killed => opts[:load_killed]
       yield @subj_thread.size if block_given?
     end
   end
 
   ## loads in all messages needed to thread m
-  def load_thread_for_message m
-    @index.each_message_in_thread_for m, :limit => 100 do |mid, builder|
+  def load_thread_for_message m, opts={}
+    @index.each_message_in_thread_for m, opts.merge({:limit => 100}) do |mid, builder|
       next if contains_id? mid
       add_message builder.call
     end
   end
 
+  ## merges in a pre-loaded thread
+  def add_thread t
+    raise "duplicate" if @subj_thread.values.member? t
+    t.each { |m, *o| add_message m }
+  end
+
   def is_relevant? m
     m.refs.any? { |ref_id| @messages[ref_id] }
   end
@@ -323,38 +346,68 @@ class ThreadSet
 
     if root == oldroot
       if oldroot.thread
-        #        puts "*** root (#{root.subj}) == oldroot (#{oldroot.subj}); ignoring"
+        ## check to see if the subject is still the same (in the case
+        ## that we first added a child message with a different
+        ## subject)
+
+        ## this code is duplicated below. sorry! TODO: refactor
+        s = Message.normalize_subj(root.subj)
+        unless @subj_thread[s] == root.thread
+          ## Redwood::log "[1] moving thread to new subject #{root.subj}"
+          if @subj_thread[s]
+            @subj_thread[s] << root
+            root.thread = @subj_thread[s]
+          else
+            @subj_thread[s] = root.thread
+          end
+        end
+
       else
         ## to disable subject grouping, use the next line instead
         ## (and the same for below)
-        #Redwood::log "[1] normalized subject for #{id} is #{Message.normalize_subj(root.subj)}"
+        #Redwood::log "[1] for #{root}, subject #{Message.normalize_subj(root.subj)} has #{@subj_thread[Message.normalize_subj(root.subj)] ? 'a' : 'no'} thread"
         thread = (@subj_thread[Message.normalize_subj(root.subj)] ||= Thread.new)
         #thread = (@subj_thread[root.id] ||= Thread.new)
 
         thread << root
         root.thread = thread
-        #        puts "# (1) added #{root} to #{thread}"
+        # Redwood::log "[1] added #{root} to #{thread}"
       end
     else
       if oldroot.thread
         ## new root. need to drop old one and put this one in its place
-        #        puts "*** DROPPING #{oldroot} from #{oldroot.thread}"
         oldroot.thread.drop oldroot
         oldroot.thread = nil
       end
 
       if root.thread
-        #        puts "*** IGNORING cuz root already has a thread"
+        ## check to see if the subject is still the same (in the case
+        ## that we first added a child message with a different
+        ## subject)
+        s = Message.normalize_subj(root.subj)
+        unless @subj_thread[s] == root.thread
+          # Redwood::log "[2] moving thread to new subject #{root.subj}"
+          if @subj_thread[s]
+            @subj_thread[s] << root
+            root.thread = @subj_thread[s]
+          else
+            @subj_thread[s] = root.thread
+          end
+        end
+
       else
         ## to disable subject grouping, use the next line instead
         ## (and the same above)
-        #Redwood::log "[2] normalized subject for #{id} is #{Message.normalize_subj(root.subj)}"
+        
+        ## this code is duplicated above. sorry! TODO: refactor
+        # Redwood::log "[2] for #{root}, subject '#{Message.normalize_subj(root.subj)}' has #{@subj_thread[Message.normalize_subj(root.subj)] ? 'a' : 'no'} thread"
+
         thread = (@subj_thread[Message.normalize_subj(root.subj)] ||= Thread.new)
         #thread = (@subj_thread[root.id] ||= Thread.new)
 
         thread << root
         root.thread = thread
-        #        puts "# (2) added #{root} to #{thread}"
+        # Redwood::log "[2] added #{root} to #{thread}"
       end
     end
 

data/lib/sup/update.rb

@@ -11,10 +11,9 @@ class UpdateManager
   def register o; @targets[o] = true; end
   def unregister o; @targets.delete o; end
 
-  def relay type, *args
+  def relay sender, type, *args
     meth = "handle_#{type}_update".intern
-    @targets.keys.each { |o| o.send meth, *args if o.respond_to? meth }
-    BufferManager.draw_screen ## TODO: think about this
+    @targets.keys.each { |o| o.send meth, sender, *args unless o == sender if o.respond_to? meth }
   end
 end
 

data/lib/sup/util.rb

@@ -43,6 +43,19 @@ class Object
   ## i'm sure there's pithy comment i could make here about the
   ## superiority of lisp, but fuck lisp.
   def returning x; yield x; x; end
+
+  ## clone of java-style whole-method synchronization
+  ## assumes a @mutex variable
+  def synchronized *meth
+    meth.each do
+      class_eval <<-EOF
+        alias unsynchronized_#{meth} #{meth}
+        def #{meth}(*a, &b)
+          @mutex.synchronize { unsynchronized_#{meth}(*a, &b) }
+        end
+      EOF
+    end
+  end
 end
 
 class String
@@ -176,19 +189,6 @@ class Array
   def rest; self[1..-1]; end
 
   def to_boolean_h; Hash[*map { |x| [x, true] }.flatten]; end
-  
-  ## apparently uniq doesn't use ==. wtf.
-  def remove_successive_dupes
-    ret = []
-    last = nil
-    each do |e|
-      unless e == last
-        ret << e
-        last = e
-      end
-    end
-    ret
-  end
 end
 
 class Time