sup 0.11 → 0.12

data/lib/sup/logger.rb

@@ -54,7 +54,7 @@ private
       when "error"; "ERROR: "
       else ""
     end
-    "[#{time.to_s}] #{prefix}#{msg}\n"
+    "[#{time.to_s}] #{prefix}#{msg.rstrip}\n"
   end
 
   ## actually distribute the message

data/lib/sup/maildir.rb

@@ -1,22 +1,15 @@
-require 'rmail'
 require 'uri'
 
 module Redwood
 
-## Maildir doesn't provide an ordered unique id, which is what Sup
-## requires to be really useful. So we must maintain, in memory, a
-## mapping between Sup "ids" (timestamps, essentially) and the
-## pathnames on disk.
-
 class Maildir < Source
   include SerializeLabelsNicely
-  SCAN_INTERVAL = 30 # seconds
   MYHOSTNAME = Socket.gethostname
 
   ## remind me never to use inheritance again.
-  yaml_properties :uri, :cur_offset, :usual, :archived, :id, :labels, :mtimes
-  def initialize uri, last_date=nil, usual=true, archived=false, id=nil, labels=[], mtimes={}
-    super uri, last_date, usual, archived, id
+  yaml_properties :uri, :usual, :archived, :id, :labels
+  def initialize uri, usual=true, archived=false, id=nil, labels=[]
+    super uri, usual, archived, id
     uri = URI(Source.expand_filesystem_uri(uri))
 
     raise ArgumentError, "not a maildir URI" unless uri.scheme == "maildir"
@@ -25,28 +18,14 @@ class Maildir < Source
 
     @dir = uri.path
     @labels = Set.new(labels || [])
-    @ids = []
-    @ids_to_fns = {}
-    @last_scan = nil
     @mutex = Mutex.new
-    #the mtime from the subdirs in the maildir with the unix epoch as default.
-    #these are used to determine whether scanning the directory for new mail
-    #is a worthwhile effort
-    @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }.merge(mtimes || {})
-    @dir_ids = { 'cur' => [], 'new' => [] }
+    @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
   end
 
   def file_path; @dir end
   def self.suggest_labels_for path; [] end
   def is_source_for? uri; super || (URI(Source.expand_filesystem_uri(uri)) == URI(self.uri)); end
 
-  def check
-    scan_mailbox
-    return unless start_offset
-
-    start = @ids.index(cur_offset || start_offset) or raise OutOfSyncSourceError, "Unknown message id #{cur_offset || start_offset}." # couldn't find the most recent email
-  end
-
   def store_message date, from_email, &block
     stored = false
     new_fn = new_maildir_basefn + ':2,S'
@@ -76,7 +55,6 @@ class Maildir < Source
   end
 
   def each_raw_message_line id
-    scan_mailbox
     with_file_for(id) do |f|
       until f.eof?
         yield f.gets
@@ -85,17 +63,14 @@ class Maildir < Source
   end
 
   def load_header id
-    scan_mailbox
     with_file_for(id) { |f| parse_raw_email_header f }
   end
 
   def load_message id
-    scan_mailbox
     with_file_for(id) { |f| RMail::Parser.read f }
   end
 
   def raw_header id
-    scan_mailbox
     ret = ""
     with_file_for(id) do |f|
       until f.eof? || (l = f.gets) =~ /^$/
@@ -106,106 +81,75 @@ class Maildir < Source
   end
 
   def raw_message id
-    scan_mailbox
     with_file_for(id) { |f| f.read }
   end
 
-  def scan_mailbox opts={}
-    return unless @ids.empty? || opts[:rescan]
-    return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
-
-    initial_poll = @ids.empty?
-
-    debug "scanning maildir #@dir..."
-    begin
-      @mtimes.each_key do |d|
-	subdir = File.join(@dir, d)
-	raise FatalSourceError, "#{subdir} not a directory" unless File.directory? subdir
-
-	mtime = File.mtime subdir
-
-	#only scan the dir if the mtime is more recent (or we haven't polled
-	#since startup)
-	if @mtimes[d] < mtime || initial_poll
-	  @mtimes[d] = mtime
-	  @dir_ids[d] = []
-	  Dir[File.join(subdir, '*')].map do |fn|
-	    id = make_id fn
-	    @dir_ids[d] << id
-	    @ids_to_fns[id] = fn
-	  end
-	else
-	  debug "no poll on #{d}.  mtime on indicates no new messages."
-	end
+  ## XXX use less memory
+  def poll
+    @mtimes.each do |d,prev_mtime|
+      subdir = File.join @dir, d
+      debug "polling maildir #{subdir}"
+      raise FatalSourceError, "#{subdir} not a directory" unless File.directory? subdir
+      mtime = File.mtime subdir
+      next if prev_mtime >= mtime
+      @mtimes[d] = mtime
+
+      old_ids = benchmark(:maildir_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id, "#{d}/").to_a }
+      new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.basename x }.sort }
+      added = new_ids - old_ids
+      deleted = old_ids - new_ids
+      debug "#{old_ids.size} in index, #{new_ids.size} in filesystem"
+      debug "#{added.size} added, #{deleted.size} deleted"
+
+      added.each_with_index do |id,i|
+        yield :add,
+          :info => File.join(d,id),
+          :labels => @labels + maildir_labels(id) + [:inbox],
+          :progress => i.to_f/(added.size+deleted.size)
       end
-      @ids = @dir_ids.values.flatten.uniq.sort!
-    rescue SystemCallError, IOError => e
-      raise FatalSourceError, "Problem scanning Maildir directories: #{e.message}."
-    end
-    
-    debug "done scanning maildir"
-    @last_scan = Time.now
-  end
-  synchronized :scan_mailbox
 
-  def each
-    scan_mailbox
-    return unless start_offset
-
-    start = @ids.index(cur_offset || start_offset) or raise OutOfSyncSourceError, "Unknown message id #{cur_offset || start_offset}." # couldn't find the most recent email
-
-    start.upto(@ids.length - 1) do |i|         
-      id = @ids[i]
-      self.cur_offset = id
-      yield id, @labels + (seen?(id) ? [] : [:unread]) + (trashed?(id) ? [:deleted] : []) + (flagged?(id) ? [:starred] : [])
+      deleted.each_with_index do |id,i|
+        yield :delete,
+          :info => File.join(d,id),
+          :progress => (i.to_f+added.size)/(added.size+deleted.size)
+      end
     end
+    nil
   end
 
-  def start_offset
-    scan_mailbox
-    @ids.first
-  end
-
-  def end_offset
-    scan_mailbox :rescan => true
-    @ids.last + 1
+  def maildir_labels id
+    (seen?(id) ? [] : [:unread]) +
+      (trashed?(id) ?  [:deleted] : []) +
+      (flagged?(id) ? [:starred] : [])
   end
 
-  def pct_done; 100.0 * (@ids.index(cur_offset) || 0).to_f / (@ids.length - 1).to_f; end
+  def draft? id; maildir_data(id)[2].include? "D"; end
+  def flagged? id; maildir_data(id)[2].include? "F"; end
+  def passed? id; maildir_data(id)[2].include? "P"; end
+  def replied? id; maildir_data(id)[2].include? "R"; end
+  def seen? id; maildir_data(id)[2].include? "S"; end
+  def trashed? id; maildir_data(id)[2].include? "T"; end
 
-  def draft? msg; maildir_data(msg)[2].include? "D"; end
-  def flagged? msg; maildir_data(msg)[2].include? "F"; end
-  def passed? msg; maildir_data(msg)[2].include? "P"; end
-  def replied? msg; maildir_data(msg)[2].include? "R"; end
-  def seen? msg; maildir_data(msg)[2].include? "S"; end
-  def trashed? msg; maildir_data(msg)[2].include? "T"; end
+  def mark_draft id; maildir_mark_file id, "D" unless draft? id; end
+  def mark_flagged id; maildir_mark_file id, "F" unless flagged? id; end
+  def mark_passed id; maildir_mark_file id, "P" unless passed? id; end
+  def mark_replied id; maildir_mark_file id, "R" unless replied? id; end
+  def mark_seen id; maildir_mark_file id, "S" unless seen? id; end
+  def mark_trashed id; maildir_mark_file id, "T" unless trashed? id; end
 
-  def mark_draft msg; maildir_mark_file msg, "D" unless draft? msg; end
-  def mark_flagged msg; maildir_mark_file msg, "F" unless flagged? msg; end
-  def mark_passed msg; maildir_mark_file msg, "P" unless passed? msg; end
-  def mark_replied msg; maildir_mark_file msg, "R" unless replied? msg; end
-  def mark_seen msg; maildir_mark_file msg, "S" unless seen? msg; end
-  def mark_trashed msg; maildir_mark_file msg, "T" unless trashed? msg; end
-
-  def filename_for_id id; @ids_to_fns[id] end
+  def valid? id
+    File.exists? File.join(@dir, id)
+  end
 
 private
 
-  def make_id fn
-    #doing this means 1 syscall instead of 2 (File.mtime, File.size).
-    #makes a noticeable difference on nfs.
-    stat = File.stat(fn)
-    # use 7 digits for the size. why 7? seems nice.
-    sprintf("%d%07d", stat.mtime, stat.size % 10000000).to_i
-  end
-
   def new_maildir_basefn
     Kernel::srand()
     "#{Time.now.to_i.to_s}.#{$$}#{Kernel.rand(1000000)}.#{MYHOSTNAME}"
   end
 
   def with_file_for id
-    fn = @ids_to_fns[id] or raise OutOfSyncSourceError, "No such id: #{id.inspect}."
+    fn = File.join(@dir, id)
     begin
       File.open(fn, 'rb') { |f| yield f }
     rescue SystemCallError, IOError => e
@@ -213,10 +157,9 @@ private
     end
   end
 
-  def maildir_data msg
-    fn = File.basename @ids_to_fns[msg]
-    fn =~ %r{^([^:]+):([12]),([DFPRST]*)$}
-    [($1 || fn), ($2 || "2"), ($3 || "")]
+  def maildir_data id
+    id =~ %r{^([^:]+):([12]),([DFPRST]*)$}
+    [($1 || id), ($2 || "2"), ($3 || "")]
   end
 
   ## not thread-safe on msg

data/lib/sup/mbox.rb

@@ -1,13 +1,155 @@
-require "sup/mbox/loader"
-require "sup/mbox/ssh-file"
-require "sup/mbox/ssh-loader"
+require 'uri'
+require 'set'
 
 module Redwood
 
-module MBox
+class MBox < Source
   BREAK_RE = /^From \S+ (.+)$/
 
-  def is_break_line? l
+  include SerializeLabelsNicely
+  yaml_properties :uri, :usual, :archived, :id, :labels
+
+  attr_reader :labels
+
+  ## uri_or_fp is horrific. need to refactor.
+  def initialize uri_or_fp, usual=true, archived=false, id=nil, labels=nil
+    @mutex = Mutex.new
+    @labels = Set.new((labels || []) - LabelManager::RESERVED_LABELS)
+
+    case uri_or_fp
+    when String
+      uri = URI(Source.expand_filesystem_uri(uri_or_fp))
+      raise ArgumentError, "not an mbox uri" unless uri.scheme == "mbox"
+      raise ArgumentError, "mbox URI ('#{uri}') cannot have a host: #{uri.host}" if uri.host
+      raise ArgumentError, "mbox URI must have a path component" unless uri.path
+      @f = File.open uri.path, 'rb'
+      @path = uri.path
+    else
+      @f = uri_or_fp
+      @path = uri_or_fp.path
+    end
+
+    super uri_or_fp, usual, archived, id
+  end
+
+  def file_path; @path end
+  def is_source_for? uri; super || (self.uri.is_a?(String) && (URI(Source.expand_filesystem_uri(uri)) == URI(Source.expand_filesystem_uri(self.uri)))) end
+
+  def self.suggest_labels_for path
+    ## heuristic: use the filename as a label, unless the file
+    ## has a path that probably represents an inbox.
+    if File.dirname(path) =~ /\b(var|usr|spool)\b/
+      []
+    else
+      [File.basename(path).downcase.intern]
+    end
+  end
+
+  def load_header offset
+    header = nil
+    @mutex.synchronize do
+      @f.seek offset
+      header = parse_raw_email_header @f
+    end
+    header
+  end
+
+  def load_message offset
+    @mutex.synchronize do
+      @f.seek offset
+      begin
+        ## don't use RMail::Mailbox::MBoxReader because it doesn't properly ignore
+        ## "From" at the start of a message body line.
+        string = ""
+        until @f.eof? || MBox::is_break_line?(l = @f.gets)
+          string << l
+        end
+        RMail::Parser.read string
+      rescue RMail::Parser::Error => e
+        raise FatalSourceError, "error parsing mbox file: #{e.message}"
+      end
+    end
+  end
+
+  def raw_header offset
+    ret = ""
+    @mutex.synchronize do
+      @f.seek offset
+      until @f.eof? || (l = @f.gets) =~ /^\r*$/
+        ret << l
+      end
+    end
+    ret
+  end
+
+  def raw_message offset
+    ret = ""
+    each_raw_message_line(offset) { |l| ret << l }
+    ret
+  end
+
+  def store_message date, from_email, &block
+    need_blank = File.exists?(@path) && !File.zero?(@path)
+    File.open(@path, "ab") do |f|
+      f.puts if need_blank
+      f.puts "From #{from_email} #{date.asctime}"
+      yield f
+    end
+  end
+
+  ## apparently it's a million times faster to call this directly if
+  ## we're just moving messages around on disk, than reading things
+  ## into memory with raw_message.
+  ##
+  ## i hoped never to have to move shit around on disk but
+  ## sup-sync-back has to do it.
+  def each_raw_message_line offset
+    @mutex.synchronize do
+      @f.seek offset
+      until @f.eof? || MBox::is_break_line?(l = @f.gets)
+        yield l
+      end
+    end
+  end
+
+  def default_labels
+    [:inbox, :unread]
+  end
+
+  def poll
+    first_offset = first_new_message
+		offset = first_offset
+    end_offset = File.size @f
+    while offset and offset < end_offset
+      yield :add,
+        :info => offset,
+        :labels => (labels + default_labels),
+        :progress => (offset - first_offset).to_f/end_offset
+      offset = next_offset offset
+    end
+  end
+
+  def next_offset offset
+    @mutex.synchronize do
+      @f.seek offset
+      nil while line = @f.gets and not MBox::is_break_line? line
+      offset = @f.tell
+      offset != File.size(@f) ? offset : nil
+    end
+  end
+
+  ## TODO optimize this by iterating over allterms list backwards or
+  ## storing source_info negated
+  def last_indexed_message
+    benchmark(:mbox_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id).map(&:to_i).max }
+  end
+
+  ## offset of first new message or nil
+  def first_new_message
+    next_offset(last_indexed_message || 0)
+  end
+
+  def self.is_break_line? l
     l =~ BREAK_RE or return false
     time = $1
     begin
@@ -19,6 +161,9 @@ module MBox
       false
     end
   end
-  module_function :is_break_line?
+
+  class Loader < self
+    yaml_properties :uri, :usual, :archived, :id, :labels
+  end
 end
 end

data/lib/sup/message-chunks.rb

@@ -32,9 +32,25 @@ require 'tempfile'
 ## attachments are quotable; Signatures are not.
 
 ## monkey-patch time: make temp files have the right extension
-class Tempfile
-  def make_tmpname basename, n
-    sprintf '%d-%d-%s', $$, n, basename
+## Backport from Ruby 1.9.2 for versions lower than 1.8.7
+if RUBY_VERSION < '1.8.7'
+  class Tempfile
+    def make_tmpname(prefix_suffix, n)
+      case prefix_suffix
+      when String
+        prefix = prefix_suffix
+        suffix = ""
+      when Array
+        prefix = prefix_suffix[0]
+        suffix = prefix_suffix[1]
+      else
+        raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
+      end
+      t = Time.now.strftime("%Y%m%d")
+      path = "#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
+      path << "-#{n}" if n
+      path << suffix
+    end
   end
 end
 
@@ -149,7 +165,7 @@ EOS
     end
 
     def write_to_disk
-      file = Tempfile.new(@filename || "sup-attachment")
+      file = Tempfile.new(["sup", @filename.gsub("/", "_") || "sup-attachment"])
       file.print @raw_content
       file.close
       file.path