sup 0.14.1.1 → 0.15.0

data/lib/sup/index.rb

@@ -168,6 +168,32 @@ EOS
     matchset.matches_estimated
   end
 
+  ## check if a message is part of a killed thread
+  ## (warning: duplicates code below)
+  ## NOTE: We can be more efficient if we assume every
+  ## killed message that hasn't been initially added
+  ## to the indexi s this way
+  def message_joining_killed? m
+    return false unless doc = find_doc(m.id)
+    queue = doc.value(THREAD_VALUENO).split(',')
+    seen_threads = Set.new
+    seen_messages = Set.new [m.id]
+    while not queue.empty?
+      thread_id = queue.pop
+      next if seen_threads.member? thread_id
+      return true if thread_killed?(thread_id)
+      seen_threads << thread_id
+      docs = term_docids(mkterm(:thread, thread_id)).map { |x| @xapian.document x }
+      docs.each do |doc|
+        msgid = doc.value MSGID_VALUENO
+        next if seen_messages.member? msgid
+        seen_messages << msgid
+        queue.concat doc.value(THREAD_VALUENO).split(',')
+      end
+    end
+    false
+  end
+
   ## yield all messages in the thread containing 'm' by repeatedly
   ## querying the index. yields pairs of message ids and
   ## message-building lambdas, so that building an unwanted message
@@ -248,11 +274,11 @@ EOS
 
   ## Yield each message-id matching query
   EACH_ID_PAGE = 100
-  def each_id query={}
+  def each_id query={}, ignore_neg_terms = true
     offset = 0
     page = EACH_ID_PAGE
 
-    xapian_query = build_xapian_query query
+    xapian_query = build_xapian_query query, ignore_neg_terms
     while true
       ids = run_query_ids xapian_query, offset, (offset+page)
       ids.each { |id| yield id }
@@ -262,8 +288,12 @@ EOS
   end
 
   ## Yield each message matching query
-  def each_message query={}, &b
-    each_id query do |id|
+  ## The ignore_neg_terms parameter is used to display result even if
+  ## it contains "forbidden" labels such as :deleted, it is used in
+  ## Poll#poll_from when we need to get the location of a message that
+  ## may contain these labels
+  def each_message query={}, ignore_neg_terms = true, &b
+    each_id query, ignore_neg_terms do |id|
       yield build_message(id)
     end
   end
@@ -313,9 +343,9 @@ EOS
   ## Yields (in lexicographical order) the source infos of all locations from
   ## the given source with the given source_info prefix
   def each_source_info source_id, prefix='', &b
-    prefix = mkterm :location, source_id, prefix
-    each_prefixed_term prefix do |x|
-      yield x[prefix.length..-1]
+    p = mkterm :location, source_id, prefix
+    each_prefixed_term p do |x|
+      yield prefix + x[p.length..-1]
     end
   end
 
@@ -492,7 +522,7 @@ EOS
       raise ParseError, "xapian query parser error: #{e}"
     end
 
-    debug "parsed xapian query: #{Util::Query.describe(xapian_query)}"
+    debug "parsed xapian query: #{Util::Query.describe(xapian_query, subs)}"
 
     raise ParseError if xapian_query.nil? or xapian_query.empty?
     query[:qobj] = xapian_query
@@ -500,14 +530,18 @@ EOS
     query
   end
 
+  def save_message m
+    if @sync_worker
+      @sync_queue << m
+    else
+      update_message_state m
+    end
+    m.clear_dirty
+  end
+
   def save_thread t
     t.each_dirty_message do |m|
-      if @sync_worker
-        @sync_queue << m
-      else
-        update_message_state m
-      end
-      m.clear_dirty
+      save_message m
     end
   end
 
@@ -614,7 +648,7 @@ EOS
   end
 
   Q = Xapian::Query
-  def build_xapian_query opts
+  def build_xapian_query opts, ignore_neg_terms = true
     labels = ([opts[:label]] + (opts[:labels] || [])).compact
     neglabels = [:spam, :deleted, :killed].reject { |l| (labels.include? l) || opts.member?("load_#{l}".intern) }
     pos_terms, neg_terms = [], []
@@ -630,7 +664,7 @@ EOS
       pos_terms << Q.new(Q::OP_OR, participant_terms)
     end
 
-    neg_terms.concat(neglabels.map { |l| mkterm(:label,l) })
+    neg_terms.concat(neglabels.map { |l| mkterm(:label,l) }) if ignore_neg_terms
 
     pos_query = Q.new(Q::OP_AND, pos_terms)
     neg_query = Q.new(Q::OP_OR, neg_terms)
@@ -643,6 +677,10 @@ EOS
   end
 
   def sync_message m, overwrite
+    ## TODO: we should not save the message if the sync_back failed
+    ## since it would overwrite the location field
+    m.sync_back
+
     doc = synchronize { find_doc(m.id) }
     existed = doc != nil
     doc ||= Xapian::Document.new

data/lib/sup/label.rb

@@ -7,10 +7,10 @@ class LabelManager
 
   ## labels that have special semantics. user will be unable to
   ## add/remove these via normal label mechanisms.
-  RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent, :deleted, :inbox, :attachment ]
+  RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent, :deleted, :inbox, :attachment, :forwarded, :replied ]
 
   ## labels that will typically be hidden from the user
-  HIDDEN_RESERVED_LABELS = [ :starred, :unread, :attachment ]
+  HIDDEN_RESERVED_LABELS = [ :starred, :unread, :attachment, :forwarded, :replied ]
 
   def initialize fn
     @fn = fn

data/lib/sup/maildir.rb

@@ -1,4 +1,5 @@
 require 'uri'
+require 'set'
 
 module Redwood
 
@@ -7,8 +8,8 @@ class Maildir < Source
   MYHOSTNAME = Socket.gethostname
 
   ## remind me never to use inheritance again.
-  yaml_properties :uri, :usual, :archived, :id, :labels
-  def initialize uri, usual=true, archived=false, id=nil, labels=[]
+  yaml_properties :uri, :usual, :archived, :sync_back, :id, :labels
+  def initialize uri, usual=true, archived=false, sync_back=true, id=nil, labels=[]
     super uri, usual, archived, id
     @expanded_uri = Source.expand_filesystem_uri(uri)
     uri = URI(@expanded_uri)
@@ -17,16 +18,28 @@ class Maildir < Source
     raise ArgumentError, "maildir URI cannot have a host: #{uri.host}" if uri.host
     raise ArgumentError, "maildir URI must have a path component" unless uri.path
 
+    @sync_back = sync_back
+    # sync by default if not specified
+    @sync_back = true if @sync_back.nil?
+
     @dir = uri.path
     @labels = Set.new(labels || [])
     @mutex = Mutex.new
-    @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
+    @ctimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
   end
 
   def file_path; @dir end
   def self.suggest_labels_for path; [] end
   def is_source_for? uri; super || (uri == @expanded_uri); end
 
+  def supported_labels?
+    [:draft, :starred, :forwarded, :replied, :unread, :deleted]
+  end
+
+  def sync_back_enabled?
+    @sync_back
+  end
+
   def store_message date, from_email, &block
     stored = false
     new_fn = new_maildir_basefn + ':2,S'
@@ -44,7 +57,7 @@ class Maildir < Source
             f.fsync
           end
 
-          File.link tmp_path, new_path
+          File.safe_link tmp_path, new_path
           stored = true
         ensure
           File.unlink tmp_path if File.exists? tmp_path
@@ -71,6 +84,14 @@ class Maildir < Source
     with_file_for(id) { |f| RMail::Parser.read f }
   end
 
+  def sync_back id, labels
+    synchronize do
+      debug "syncing back maildir message #{id} with flags #{labels.to_a}"
+      flags = maildir_reconcile_flags id, labels
+      maildir_mark_file id, flags
+    end
+  end
+
   def raw_header id
     ret = ""
     with_file_for(id) do |f|
@@ -87,41 +108,78 @@ class Maildir < Source
 
   ## XXX use less memory
   def poll
-    @mtimes.each do |d,prev_mtime|
+    added = []
+    deleted = []
+    updated = []
+    @ctimes.each do |d,prev_ctime|
       subdir = File.join @dir, d
       debug "polling maildir #{subdir}"
       raise FatalSourceError, "#{subdir} not a directory" unless File.directory? subdir
-      mtime = File.mtime subdir
-      next if prev_mtime >= mtime
-      @mtimes[d] = mtime
+      ctime = File.ctime subdir
+      next if prev_ctime >= ctime
+      @ctimes[d] = ctime
 
       old_ids = benchmark(:maildir_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id, "#{d}/").to_a }
-      new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.basename x }.sort }
-      added = new_ids - old_ids
-      deleted = old_ids - new_ids
+      new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.join(d,File.basename(x)) }.sort }
+      added += new_ids - old_ids
+      deleted += old_ids - new_ids
       debug "#{old_ids.size} in index, #{new_ids.size} in filesystem"
-      debug "#{added.size} added, #{deleted.size} deleted"
+    end
 
-      added.each_with_index do |id,i|
-        yield :add,
-          :info => File.join(d,id),
-          :labels => @labels + maildir_labels(id) + [:inbox],
-          :progress => i.to_f/(added.size+deleted.size)
-      end
+    ## find updated mails by checking if an id is in both added and
+    ## deleted arrays, meaning that its flags changed or that it has
+    ## been moved, these ids need to be removed from added and deleted
+    add_to_delete = del_to_delete = []
+    map = Hash.new { |hash, key| hash[key] = [] }
+    deleted.each do |id_del|
+        map[maildir_data(id_del)[0]].push id_del
+    end
+    added.each do |id_add|
+        map[maildir_data(id_add)[0]].each do |id_del|
+          updated.push [ id_del, id_add ]
+          add_to_delete.push id_add
+          del_to_delete.push id_del
+        end
+    end
+    added -= add_to_delete
+    deleted -= del_to_delete
+    debug "#{added.size} added, #{deleted.size} deleted, #{updated.size} updated"
+    total_size = added.size+deleted.size+updated.size
 
-      deleted.each_with_index do |id,i|
-        yield :delete,
-          :info => File.join(d,id),
-          :progress => (i.to_f+added.size)/(added.size+deleted.size)
-      end
+    added.each_with_index do |id,i|
+      yield :add,
+      :info => id,
+      :labels => @labels + maildir_labels(id) + [:inbox],
+      :progress => i.to_f/total_size
+    end
+
+    deleted.each_with_index do |id,i|
+      yield :delete,
+      :info => id,
+      :progress => (i.to_f+added.size)/total_size
+    end
+
+    updated.each_with_index do |id,i|
+      yield :update,
+      :old_info => id[0],
+      :new_info => id[1],
+      :labels => @labels + maildir_labels(id[1]),
+      :progress => (i.to_f+added.size+deleted.size)/total_size
     end
     nil
   end
 
+  def labels? id
+    maildir_labels id
+  end
+
   def maildir_labels id
     (seen?(id) ? [] : [:unread]) +
       (trashed?(id) ?  [:deleted] : []) +
-      (flagged?(id) ? [:starred] : [])
+      (flagged?(id) ? [:starred] : []) +
+      (passed?(id) ? [:forwarded] : []) +
+      (replied?(id) ? [:replied] : []) +
+      (draft?(id) ? [:draft] : [])
   end
 
   def draft? id; maildir_data(id)[2].include? "D"; end
@@ -131,13 +189,6 @@ class Maildir < Source
   def seen? id; maildir_data(id)[2].include? "S"; end
   def trashed? id; maildir_data(id)[2].include? "T"; end
 
-  def mark_draft id; maildir_mark_file id, "D" unless draft? id; end
-  def mark_flagged id; maildir_mark_file id, "F" unless flagged? id; end
-  def mark_passed id; maildir_mark_file id, "P" unless passed? id; end
-  def mark_replied id; maildir_mark_file id, "R" unless replied? id; end
-  def mark_seen id; maildir_mark_file id, "S" unless seen? id; end
-  def mark_trashed id; maildir_mark_file id, "T" unless trashed? id; end
-
   def valid? id
     File.exists? File.join(@dir, id)
   end
@@ -159,25 +210,47 @@ private
   end
 
   def maildir_data id
-    id =~ %r{^([^:]+):([12]),([DFPRST]*)$}
+    id = File.basename id
+    # Flags we recognize are DFPRST
+    id =~ %r{^([^:]+):([12]),([A-Za-z]*)$}
     [($1 || id), ($2 || "2"), ($3 || "")]
   end
 
-  ## not thread-safe on msg
-  def maildir_mark_file msg, flag
-    orig_path = @ids_to_fns[msg]
-    orig_base, orig_fn = File.split(orig_path)
-    new_base = orig_base.slice(0..-4) + 'cur'
-    tmp_base = orig_base.slice(0..-4) + 'tmp'
-    md_base, md_ver, md_flags = maildir_data msg
-    md_flags += flag; md_flags = md_flags.split(//).sort.join.squeeze
-    new_path = File.join new_base, "#{md_base}:#{md_ver},#{md_flags}"
-    tmp_path = File.join tmp_base, "#{md_base}:#{md_ver},#{md_flags}"
-    File.link orig_path, tmp_path
-    File.unlink orig_path
-    File.link tmp_path, new_path
-    File.unlink tmp_path
-    @ids_to_fns[msg] = new_path
+  def maildir_reconcile_flags id, labels
+      new_flags = Set.new( maildir_data(id)[2].each_char )
+
+      # Set flags based on labels for the six flags we recognize
+      if labels.member? :draft then new_flags.add?( "D" ) else new_flags.delete?( "D" ) end
+      if labels.member? :starred then new_flags.add?( "F" ) else new_flags.delete?( "F" ) end
+      if labels.member? :forwarded then new_flags.add?( "P" ) else new_flags.delete?( "P" ) end
+      if labels.member? :replied then new_flags.add?( "R" ) else new_flags.delete?( "R" ) end
+      if not labels.member? :unread then new_flags.add?( "S" ) else new_flags.delete?( "S" ) end
+      if labels.member? :deleted or labels.member? :killed then new_flags.add?( "T" ) else new_flags.delete?( "T" ) end
+
+      ## Flags must be stored in ASCII order according to Maildir
+      ## documentation
+      new_flags.to_a.sort.join
+  end
+
+  def maildir_mark_file orig_path, flags
+    @mutex.synchronize do
+      new_base = (flags.include?("S")) ? "cur" : "new"
+      md_base, md_ver, md_flags = maildir_data orig_path
+
+      return if md_flags == flags
+
+      new_loc = File.join new_base, "#{md_base}:#{md_ver},#{flags}"
+      orig_path = File.join @dir, orig_path
+      new_path  = File.join @dir, new_loc
+      tmp_path  = File.join @dir, "tmp", "#{md_base}:#{md_ver},#{flags}"
+
+      File.safe_link orig_path, tmp_path
+      File.unlink orig_path
+      File.safe_link tmp_path, new_path
+      File.unlink tmp_path
+
+      new_loc
+    end
   end
 end
 

data/lib/sup/mbox.rb

@@ -120,7 +120,7 @@ class MBox < Source
   ## into memory with raw_message.
   ##
   ## i hoped never to have to move shit around on disk but
-  ## sup-sync-back has to do it.
+  ## sup-sync-back-mbox has to do it.
   def each_raw_message_line offset
     @mutex.synchronize do
       ensure_open

data/lib/sup/message.rb

@@ -291,6 +291,32 @@ EOS
     location.each_raw_message_line &b
   end
 
+  def sync_back
+    @locations.map { |l| l.sync_back @labels, self }.any? do
+      UpdateManager.relay self, :updated, self
+    end
+  end
+
+  def merge_labels_from_locations merge_labels
+    ## Get all labels from all locations
+    location_labels = Set.new([])
+
+    @locations.each do |l|
+      if l.valid?
+        location_labels = location_labels.union(l.labels?)
+      end
+    end
+
+    ## Add to the message labels the intersection between all location
+    ## labels and those we want to merge
+    location_labels = location_labels.intersection(merge_labels.to_set)
+
+    if not location_labels.empty?
+      @labels = @labels.union(location_labels)
+      @dirty = true
+    end
+  end
+
   ## returns all the content from a message that will be indexed
   def indexable_content
     load_from_source!
@@ -545,10 +571,18 @@ private
   ## (and possible signed) inline GPG messages
   def inline_gpg_to_chunks body, encoding_to, encoding_from
     lines = body.split("\n")
+
+    # First case: Message is enclosed between
+    #
+    # -----BEGIN PGP SIGNED MESSAGE-----
+    # and
+    # -----END PGP SIGNED MESSAGE-----
+    #
+    # In some cases, END PGP SIGNED MESSAGE doesn't appear
     gpg = lines.between(GPG_SIGNED_START, GPG_SIGNED_END)
     # between does not check if GPG_END actually exists
     # Reference: http://permalink.gmane.org/gmane.mail.sup.devel/641
-    if !gpg.empty? && !lines.index(GPG_END).nil?
+    if !gpg.empty?
       msg = RMail::Message.new
       msg.body = gpg.join("\n")
 
@@ -560,14 +594,21 @@ private
       before = startidx != 0 ? lines[0 .. startidx-1] : []
       after = endidx ? lines[endidx+1 .. lines.size] : []
 
+      # sig contains BEGIN PGP SIGNED MESSAGE and END PGP SIGNATURE, so
+      # we ditch them. sig may also contain the hash used by PGP (with a
+      # newline), so we also skip them
+      sig_start = sig[1].match(/^Hash:/) ? 3 : 1
+      sig_end = sig.size-2
       payload = RMail::Message.new
-      payload.body = sig[1, sig.size-2].join("\n")
+      payload.body = sig[sig_start, sig_end].join("\n")
       return [text_to_chunks(before, false),
               CryptoManager.verify(nil, msg, false),
               message_to_chunks(payload),
               text_to_chunks(after, false)].flatten.compact
     end
 
+    # Second case: Message is encrypted
+
     gpg = lines.between(GPG_START, GPG_END)
     # between does not check if GPG_END actually exists
     if !gpg.empty? && !lines.index(GPG_END).nil?
@@ -704,6 +745,24 @@ class Location
     source.raw_message info
   end
 
+  def sync_back labels, message
+    synced = false
+    return synced unless sync_back_enabled? and valid?
+    source.synchronize do
+      new_info = source.sync_back(@info, labels)
+      if new_info
+        @info = new_info
+        Index.sync_message message, true
+        synced = true
+      end
+    end
+    synced
+  end
+
+  def sync_back_enabled?
+    source.respond_to? :sync_back and $config[:sync_back_to_maildir] and source.sync_back_enabled?
+  end
+
   ## much faster than raw_message
   def each_raw_message_line &b
     source.each_raw_message_line info, &b
@@ -717,6 +776,10 @@ class Location
     source.valid? info
   end
 
+  def labels?
+    source.labels? info
+  end
+
   def == o
     o.source.id == source.id and o.info == info
   end