sup 0.0.2 → 0.0.3

data/lib/sup/label.rb

@@ -3,7 +3,7 @@ module Redwood
 class LabelManager
   include Singleton
 
-  ## all labels that have special meaning. user will be unable to
+  ## labels that have special semantics. user will be unable to
   ## add/remove these via normal label mechanisms.
   RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent ]
 
@@ -28,11 +28,8 @@ class LabelManager
   end
 
   def user_labels; @labels.keys; end
-
   def << t; @labels[t] = true unless @labels.member?(t) || RESERVED_LABELS.member?(t); end
-
   def delete t; @labels.delete t; end
-
   def save
     File.open(@fn, "w") { |f| f.puts @labels.keys }
   end

data/lib/sup/logger.rb

@@ -24,8 +24,9 @@ class Logger
 
   def log s
 #    $stderr.puts s
-    @mode << "#{Time.now}: #{s.chomp}\n"
     make_buf
+    @mode << "#{Time.now}: #{s.chomp}\n"
+    $stderr.puts "[#{Time.now}] #{s.chomp}" unless @mode.buffer
   end
   
   def self.method_missing m, *a

data/lib/sup/mbox.rb

@@ -1,4 +1,6 @@
 require "sup/mbox/loader"
+require "sup/mbox/ssh-file"
+require "sup/mbox/ssh-loader"
 
 module Redwood
 

data/lib/sup/mbox/loader.rb

@@ -1,27 +1,35 @@
-require 'thread'
 require 'rmail'
 
 module Redwood
 module MBox
 
 class Loader < Source
-  attr_reader :labels
+  attr_reader_cloned :labels
 
-  def initialize uri, start_offset=nil, usual=true, archived=false, id=nil
-    raise ArgumentError, "not an mbox uri" unless uri =~ %r!mbox://!
+  def initialize uri_or_fp, start_offset=nil, usual=true, archived=false, id=nil
     super
 
     @mutex = Mutex.new
-    @filename = uri.sub(%r!^mbox://!, "")
-    @f = File.open @filename
-    ## heuristic: use the filename as a label, unless the file
-    ## has a path that probably represents an inbox.
     @labels = [:unread]
-    @labels << File.basename(@filename).intern unless File.dirname(@filename) =~ /\b(var|usr|spool)\b/
+    @labels << :inbox unless archived?
+
+    case uri_or_fp
+    when String
+      raise ArgumentError, "not an mbox uri" unless uri_or_fp =~ %r!mbox://!
+
+      fn = uri_or_fp.sub(%r!^mbox://!, "")
+      ## heuristic: use the filename as a label, unless the file
+      ## has a path that probably represents an inbox.
+      @labels << File.basename(fn).intern unless File.dirname(fn) =~ /\b(var|usr|spool)\b/
+      @f = File.open fn
+    else
+      @f = uri_or_fp
+    end
   end
 
   def start_offset; 0; end
   def end_offset; File.size @f; end
+  def total; end_offset; end
 
   def load_header offset
     header = nil
@@ -38,6 +46,7 @@ class Loader < Source
   end
 
   def load_message offset
+    raise SourceError, self.broken_msg if broken?
     @mutex.synchronize do
       @f.seek offset
       begin
@@ -51,6 +60,7 @@ class Loader < Source
   end
 
   def raw_header offset
+    raise SourceError, self.broken_msg if broken?
     ret = ""
     @mutex.synchronize do
       @f.seek offset
@@ -62,6 +72,7 @@ class Loader < Source
   end
 
   def raw_full_message offset
+    raise SourceError, self.broken_msg if broken?
     ret = ""
     @mutex.synchronize do
       @f.seek offset
@@ -74,6 +85,7 @@ class Loader < Source
   end
 
   def next
+    raise SourceError, self.broken_msg if broken?
     returned_offset = nil
     next_offset = cur_offset
 

data/lib/sup/mbox/ssh-file.rb

@@ -0,0 +1,239 @@
+require 'net/ssh'
+
+module Redwood
+module MBox
+
+class SSHFileError < StandardError; end
+
+## this is a file-like interface to a file that actually lives on the
+## other end of an ssh connection. it works by using wc, head and tail
+## to simulate (buffered) random access. on a fast connection, this
+## can have a good bandwidth, but the latency is pretty terrible:
+## about 1 second (!) per request.  luckily, we're either just reading
+## straight through the mbox (an import) or we're reading a few
+## messages at a time (viewing messages) so the latency is not a problem.
+
+## all of the methods here can throw SSHFileErrors, SocketErrors,
+## Net::SSH::Exceptions and Errno::ENOENTs.
+
+## debugging TODO: remove me
+def debug s
+  Redwood::log s
+end
+module_function :debug
+
+## a simple buffer of contiguous data
+class Buffer
+  def initialize
+    clear!
+  end
+
+  def clear!
+    @start = nil
+    @buf = ""
+  end
+
+  def empty?; @start.nil?; end
+  def start; @start; end
+  def endd; @start + @buf.length; end
+
+  def add data, offset=endd
+    #MBox::debug "+ adding #{data.length} bytes; size will be #{size + data.length}; limit #{SSHFile::MAX_BUF_SIZE}"
+
+    if start.nil?
+      @buf = data
+      @start = offset
+      return
+    end
+
+    raise "non-continguous data added to buffer (data #{offset}:#{offset + data.length}, buf range #{start}:#{endd})" if offset + data.length < start || offset > endd
+
+    if offset < start
+      @buf = data[0 ... (start - offset)] + @buf
+      @start = offset
+    else
+      return if offset + data.length < endd
+      @buf += data[(endd - offset) .. -1]
+    end
+  end
+
+  def [](o)
+    raise "only ranges supported due to programmer's laziness" unless o.is_a? Range
+    @buf[Range.new(o.first - @start, o.last - @start, o.exclude_end?)]
+  end
+
+  def index what, start=0
+    x = @buf.index(what, start - @start)
+    x.nil? ? nil : x + @start
+  end
+  def rindex what, start=0
+    x = @buf.rindex(what, start - @start)
+    x.nil? ? nil : x + @start
+  end
+
+  def size; empty? ? 0 : @buf.size; end
+  def to_s; empty? ? "<empty>" : "[#{start}, #{endd})"; end # for debugging
+end
+
+## the file-like interface to a remote file
+class SSHFile
+  MAX_BUF_SIZE = 1024 * 1024 # bytes
+  MAX_TRANSFER_SIZE = 1024 * 128
+  REASONABLE_TRANSFER_SIZE = 1024 * 32
+  SIZE_CHECK_INTERVAL = 60 * 1 # seconds
+
+  @@shells = {}
+  @@shells_mutex = Mutex.new
+
+  def initialize host, fn, ssh_opts={}
+    @buf = Buffer.new
+    @host = host
+    @fn = fn
+    @ssh_opts = ssh_opts
+    @file_size = nil
+    @offset = 0
+    @say_id = nil
+    @broken_msg = nil
+  end
+
+  def broken?; !@broken_msg.nil?; end
+
+  ## TODO: share this code with imap
+  def say s
+    @say_id = BufferManager.say s, @say_id if BufferManager.instantiated?
+    Redwood::log s
+  end
+  def shutup
+    BufferManager.clear @say_id if BufferManager.instantiated?
+    @say_id = nil
+  end
+  private :say, :shutup
+
+  def connect
+    raise SSHFileError, @broken_msg if broken?
+    return if @shell
+
+    key = [@host, @ssh_opts[:username]]
+    @shell = 
+      @@shells_mutex.synchronize do
+      if @@shells.member? key
+        returning(@@shells[key]) do |shell|
+          say "Checking for #@fn..."
+          begin
+            raise Errno::ENOENT, @fn unless shell.test("-e #@fn").status == 0
+          ensure
+            shutup
+          end
+        end
+      else
+        say "Opening SSH connection to #{@host}..."
+        begin
+          #raise SSHFileError, "simulated SSH file error"
+          session = Net::SSH.start @host, @ssh_opts
+          say "Starting SSH shell..."
+          shell = session.shell.sync
+          say "Checking for #@fn..."
+          raise Errno::ENOENT, @fn unless shell.test("-e #@fn").status == 0
+          @@shells[key] = shell
+        ensure
+          shutup
+        end
+      end
+    end
+  end
+
+  def eof?; @offset >= size; end
+  def eof; eof?; end # lame but IO's method is named this and rmail calls that
+  def seek loc; @offset = loc; end
+  def tell; @offset; end
+  def total; size; end
+
+  def size
+    if @file_size.nil? || (Time.now - @last_size_check) > SIZE_CHECK_INTERVAL
+      @last_size_check = Time.now
+      @file_size = do_remote("wc -c #@fn").split.first.to_i
+    end
+    @file_size
+  end
+
+  def gets
+    return nil if eof?
+    make_buf_include @offset
+    expand_buf_forward while @buf.index("\n", @offset).nil? && @buf.endd < size
+    returning(@buf[@offset .. (@buf.index("\n", @offset) || -1)]) { |line| @offset += line.length }
+  end
+
+  def read n
+    return nil if eof?
+    make_buf_include @offset, n
+    @buf[@offset ... (@offset += n)]
+  end
+
+private
+
+  def do_remote cmd, expected_size=0
+    begin
+      retries = 0
+      connect
+      # MBox::debug "sending command: #{cmd.inspect}"
+      begin
+        result = @shell.send_command cmd
+        raise SSHFileError, "Failure during remote command #{cmd.inspect}: #{result.stderr[0 .. 100]}" unless result.status == 0
+      rescue Net::SSH::Exception # these happen occasionally for no apparent reason. gotta love that nondeterminism!
+        retry if (retries += 1) < 3
+        raise
+      end
+    rescue Net::SSH::Exception, SSHFileError, Errno::ENOENT => e
+      @broken_msg = e.message
+      raise
+    end
+    result.stdout
+  end
+
+  def get_bytes offset, size
+    do_remote "tail -c +#{offset + 1} #@fn | head -c #{size}", size
+  end
+
+  def expand_buf_forward n=REASONABLE_TRANSFER_SIZE
+    @buf.add get_bytes(@buf.endd, n)
+  end
+
+  ## try our best to transfer somewhere between
+  ## REASONABLE_TRANSFER_SIZE and MAX_TRANSFER_SIZE bytes
+  def make_buf_include offset, size=0
+    good_size = [size, REASONABLE_TRANSFER_SIZE].max
+
+    trans_start, trans_size = 
+      if @buf.empty?
+        [offset, good_size]
+      elsif offset < @buf.start
+        if @buf.start - offset <= good_size
+          start = [@buf.start - good_size, 0].max
+          [start, @buf.start - start]
+        elsif @buf.start - offset < MAX_TRANSFER_SIZE
+          [offset, @buf.start - offset]
+        else
+          MBox::debug "clearing SSH buffer because buf.start #{@buf.start} - offset #{offset} >= #{MAX_TRANSFER_SIZE}"
+          @buf.clear!
+          [offset, good_size]
+        end
+      else
+        return if [offset + size, self.size].min <= @buf.endd # whoohoo!
+        if offset - @buf.endd <= good_size
+          [@buf.endd, good_size]
+        elsif offset - @buf.endd < MAX_TRANSFER_SIZE
+          [@buf.endd, offset - @buf.endd]
+        else
+          MBox::debug "clearing SSH buffer because offset #{offset} - buf.end #{@buf.endd} >= #{MAX_TRANSFER_SIZE}"
+          @buf.clear!
+          [offset, good_size]
+        end
+      end          
+
+    @buf.clear! if @buf.size > MAX_BUF_SIZE
+    @buf.add get_bytes(trans_start, trans_size), trans_start
+  end
+end
+
+end
+end

data/lib/sup/mbox/ssh-loader.rb

@@ -0,0 +1,88 @@
+require 'net/ssh'
+
+module Redwood
+module MBox
+
+## this is slightly complicated because SSHFile (and thus @f or
+## @loader) can throw a variety of exceptions, and we need to catch
+## those, reraise them as SourceErrors, and set ourselves as broken.
+
+class SSHLoader < Source
+  attr_reader_cloned :labels
+  attr_accessor :username, :password
+
+  def initialize uri, username=nil, password=nil, start_offset=nil, usual=true, archived=false, id=nil
+    raise ArgumentError, "not an mbox+ssh uri: #{uri.inspect}" unless uri =~ %r!^mbox\+ssh://!
+
+    super uri, start_offset, usual, archived, id
+
+    @parsed_uri = URI(uri)
+    @username = username
+    @password = password
+    @uri = uri
+    @cur_offset = start_offset
+
+    opts = {}
+    opts[:username] = @username if @username
+    opts[:password] = @password if @password
+    
+    @f = SSHFile.new host, filename, opts
+    @loader = Loader.new @f, start_offset, usual, archived, id
+    
+    ## heuristic: use the filename as a label, unless the file
+    ## has a path that probably represents an inbox.
+    @labels = [:unread]
+    @labels << :inbox unless archived?
+    @labels << File.basename(filename).intern unless File.dirname(filename) =~ /\b(var|usr|spool)\b/
+  end
+
+  def host; @parsed_uri.host; end
+  def filename; @parsed_uri.path[1..-1] end
+
+  def next
+    return if broken?
+    begin
+      offset, labels = @loader.next
+      self.cur_offset = @loader.cur_offset # superclass keeps @cur_offset which is used by yaml
+      [offset, (labels + @labels).uniq] # add our labels
+    rescue Net::SSH::Exception, SocketError, SSHFileError, Errno::ENOENT => e
+      recover_from e
+    end
+  end
+
+  def end_offset
+    begin
+      @f.size
+    rescue Net::SSH::Exception, SocketError, SSHFileError, Errno::ENOENT => e
+      recover_from e
+    end
+  end
+
+  def cur_offset= o; @cur_offset = @loader.cur_offset = o; @dirty = true; end
+  def id; @loader.id; end
+  def id= o; @id = @loader.id = o; end
+  # def cur_offset; @loader.cur_offset; end # think we'll be ok without this
+  def to_s; @parsed_uri.to_s; end
+
+  def recover_from e
+    m = "error communicating with SSH server #{host} (#{e.class.name}): #{e.message}"
+    Redwood::log m
+    self.broken_msg = @loader.broken_msg = m
+    raise SourceError, m
+  end
+
+  [:start_offset, :load_header, :load_message, :raw_header, :raw_full_message].each do |meth|
+    define_method meth do |*a|
+      begin
+        @loader.send meth, *a
+      rescue Net::SSH::Exception, SocketError, SSHFileError, Errno::ENOENT => e
+        recover_from e
+      end
+    end
+  end
+end
+
+Redwood::register_yaml(SSHLoader, %w(uri username password cur_offset usual archived id))
+
+end
+end

data/lib/sup/message.rb

@@ -54,7 +54,7 @@ class Message
     attr_reader :lines
     def initialize lines
       ## do some wrapping
-      @lines = lines.map { |l| l.wrap 80 }.flatten
+      @lines = lines.map { |l| l.chomp.wrap 80 }.flatten
     end
   end
 
@@ -86,28 +86,20 @@ class Message
 
   bool_reader :dirty
 
-  ## if index_entry is specified, will fill in values from that,
+  ## if you specify a :header, will use values from that. otherwise, will try and
+  ## load the header from the source.
   def initialize opts
-    if opts[:source]
-      @source = opts[:source]
-      @source_info = opts[:source_info] or raise ArgumentError, ":source but no :source_info"
-      @body = nil
-    else
-      @source = @source_info = nil
-      @body = opts[:body] or raise ArgumentError, "one of :body or :source must be specified"
-    end
+    @source = opts[:source] or raise ArgumentError, "source can't be nil"
+    @source_info = opts[:source_info] or raise ArgumentError, "source_info can't be nil"
     @snippet = opts[:snippet] || ""
     @labels = opts[:labels] || []
     @dirty = false
 
-    header = 
-      if opts[:header]
-        opts[:header]
-      else
-        header = @source.load_header @source_info
-        header.each { |k, v| header[k.downcase] = v }
-        header
-      end
+    read_header(opts[:header] || @source.load_header(@source_info))
+  end
+
+  def read_header header
+    header.each { |k, v| header[k.downcase] = v }
 
     %w(message-id date).each do |f|
       raise MessageFormatError, "no #{f} field in header #{header.inspect} (source #@source offset #@source_info)" unless header.include? f
@@ -116,22 +108,18 @@ class Message
 
     begin
       date = header["date"]
-      @date = (Time === date ? date : Time.parse(header["date"]))
+      @date = Time === date ? date : Time.parse(header["date"])
     rescue ArgumentError => e
       raise MessageFormatError, "unparsable date #{header['date']}: #{e.message}"
     end
 
-    if(@subj = header["subject"])
-      @subj = @subj.gsub(/\s+/, " ").gsub(/\s+$/, "")
-    else
-      @subj = DEFAULT_SUBJECT
-    end
+    @subj = header.member?("subject") ? header["subject"].gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT
     @from = Person.for header["from"]
     @to = Person.for_several header["to"]
     @cc = Person.for_several header["cc"]
     @bcc = Person.for_several header["bcc"]
     @id = header["message-id"]
-    @refs = (header["references"] || "").scan(/<(.*?)>/).flatten
+    @refs = (header["references"] || "").gsub(/[<>]/, "").split(/\s+/).flatten
     @replytos = (header["in-reply-to"] || "").scan(/<(.*?)>/).flatten
     @replyto = Person.for header["reply-to"]
     @list_address =
@@ -144,7 +132,9 @@ class Message
     @recipient_email = header["delivered-to"]
     @status = header["status"]
   end
+  private :read_header
 
+  def broken?; @source.broken?; end
   def snippet; @snippet || to_chunks && @snippet; end
   def is_list_message?; !@list_address.nil?; end
   def is_draft?; DraftLoader === @source; end
@@ -154,6 +144,7 @@ class Message
   end
 
   def save index
+    return if broken?
     index.update_message self if @dirty
     @dirty = false
   end
@@ -179,30 +170,59 @@ class Message
     @dirty = true
   end
 
+  ## this is called when the message body needs to actually be loaded.
   def to_chunks
-    if @body
-      [Text.new(@body.split("\n"))]
-    else
-      message_to_chunks @source.load_message(@source_info)
-    end
+    @chunks ||=
+      if @source.broken?
+        [Text.new(error_message(@source.broken_msg.split("\n")))]
+      else
+        begin
+          read_header @source.load_header(@source_info)
+          message_to_chunks @source.load_message(@source_info)
+        rescue SourceError, SocketError => e
+          [Text.new(error_message(e.message))]
+        end
+      end
+  end
+
+  def error_message msg
+    <<EOS
+#@snippet...
+
+***********************************************************************
+* An error occurred while loading this message. It is possible that   *
+* the source has changed, or (in the case of remote sources) is down. *
+***********************************************************************
+
+The error message was:
+  #{msg}
+EOS
   end
 
   def raw_header
-    @source.raw_header @source_info
+    begin
+      @source.raw_header @source_info
+    rescue SourceError => e
+      error_message e.message
+    end
   end
 
   def raw_full_message
-    @source.raw_full_message @source_info
+    begin
+      @source.raw_full_message @source_info
+    rescue SourceError => e
+      error_message(e.message)
+    end
   end
 
   def content
     [
-      from && from.longname,
-      to.map { |p| p.longname },
-      cc.map { |p| p.longname },
-      bcc.map { |p| p.longname },
+      from && "#{from.name} #{from.email}",
+      to.map { |p| "#{p.name} #{p.email}" },
+      cc.map { |p| "#{p.name} #{p.email}" },
+      bcc.map { |p| "#{p.name} #{p.email}" },
       to_chunks.select { |c| c.is_a? Text }.map { |c| c.lines },
-      subj,
+      Message.normalize_subj(subj),
     ].flatten.compact.join " "
   end
 
@@ -226,10 +246,10 @@ private
     ret = [] <<
       case m.header.content_type
       when "text/plain", nil
-        raise MessageFormatError, "no message body before decode" unless
+        raise MessageFormatError, "no message body before decode (source #@source info #@source_info)" unless
           m.body
         body = m.decode or raise MessageFormatError, "no message body"
-        text_to_chunks body.gsub(/\t/, "    ").gsub(/\r/, "").split("\n")
+        text_to_chunks body.normalize_whitespace.split("\n")
       when /^multipart\//
         nil
       else
@@ -244,7 +264,6 @@ private
   ## parse the lines of text into chunk objects.  the heuristics here
   ## need tweaking in some nice manner. TODO: move these heuristics
   ## into the classes themselves.
-
   def text_to_chunks lines
     state = :text # one of :text, :quote, or :sig
     chunks = []
@@ -252,9 +271,11 @@ private
 
     lines.each_with_index do |line, i|
       nextline = lines[(i + 1) ... lines.length].find { |l| l !~ /^\s*$/ } # skip blank lines
+
       case state
       when :text
         newstate = nil
+
         if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && (nextline =~ QUOTE_PATTERN || nextline =~ QUOTE_START_PATTERN))
           newstate = :quote
         elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
@@ -262,6 +283,7 @@ private
         elsif line =~ BLOCK_QUOTE_PATTERN
           newstate = :block_quote
         end
+
         if newstate
           chunks << Text.new(chunk_lines) unless chunk_lines.empty?
           chunk_lines = [line]
@@ -269,8 +291,10 @@ private
         else
           chunk_lines << line
         end
+
       when :quote
         newstate = nil
+
         if line =~ QUOTE_PATTERN || line =~ QUOTE_START_PATTERN || line =~ /^\s*$/
           chunk_lines << line
         elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
@@ -278,6 +302,7 @@ private
         else
           newstate = :text
         end
+
         if newstate
           if chunk_lines.empty?
             # nothing
@@ -289,21 +314,20 @@ private
           chunk_lines = [line]
           state = newstate
         end
+
       when :block_quote
         chunk_lines << line
+
       when :sig
         chunk_lines << line
       end
  
       if state == :text && (@snippet.nil? || @snippet.length < SNIPPET_LEN) &&
           line !~ /[=\*#_-]{3,}/ && line !~ /^\s*$/
-        @snippet = (@snippet ? @snippet + " " : "") + line.gsub(/^\s+/, "").gsub(/[\r\n]/, "").gsub(/\s+/, " ")
-        @snippet = @snippet[0 ... SNIPPET_LEN]
+        @snippet += " " unless @snippet.empty?
+        @snippet += line.gsub(/^\s+/, "").gsub(/[\r\n]/, "").gsub(/\s+/, " ")
+        @snippet = @snippet[0 ... SNIPPET_LEN].chomp
       end
-#      if @snippet.nil? && state == :text && (line.length > 40 ||
-#                                             line =~ /\S+.*[^,!:]\s*$/)
-#        @snippet = line.gsub(/^\s+/, "").gsub(/[\r\n]/, "")[0 .. 80]
-#      end
     end
 
     ## final object