sup 0.0.7 → 0.0.8

data/lib/sup/index.rb

@@ -30,6 +30,7 @@ class Index
   end
 
   def save
+    Redwood::log "saving index and sources..."
     FileUtils.mkdir_p @dir unless File.exists? @dir
     save_sources
     save_index
@@ -72,25 +73,46 @@ class Index
     end
   end
 
-  ## Update the message state on disk, by deleting and re-adding it.
-  ## The message must exist in the index. docid and entry are found
-  ## unless given.
+  ## Syncs the message to the index: deleting if it's already there,
+  ## and adding either way. Index state will be determined by m.labels.
   ##
-  ## Overwrites the labels on disk with the new labels in 'm', so that
-  ## we can actually change message state.
-  def update_message m, docid=nil, entry=nil
-    unless docid && entry
-      docid, entry = load_entry_for_id m.id
-      raise ArgumentError, "cannot find #{m.id} in the index" unless entry
-    end
+  ## docid and entry can be specified if they're already known.
+  def sync_message m, docid=nil, entry=nil
+    docid, entry = load_entry_for_id m.id unless docid && entry
 
-    raise "no entry and no source info for message #{m.id}" unless m.source && m.source_info
+    raise "no source info for message #{m.id}" unless m.source && m.source_info
+    raise "trying deleting non-corresponding entry #{docid}" if docid && @index[docid][:message_id] != m.id
 
-    raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+    source_id = 
+      if m.source.is_a? Integer
+        raise "Debugging: integer source set"
+        m.source
+      else
+        m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
+      end
 
-    @index.delete docid
-    add_message m
+    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
+    d = {
+      :message_id => m.id,
+      :source_id => source_id,
+      :source_info => m.source_info,
+      :date => m.date.to_indexable_s,
+      :body => m.content,
+      :snippet => m.snippet,
+      :label => m.labels.join(" "),
+      :from => m.from ? m.from.email : "",
+      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
+      :subject => wrap_subj(Message.normalize_subj(m.subj)),
+      :refs => (m.refs + m.replytos).uniq.join(" "),
+    }
+
+    @index.delete docid if docid
+    @index.add_document d
+    
     docid, entry = load_entry_for_id m.id
+    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
+    raise "just added message #{m.id} but couldn't find it in a search" unless docid
+    true
   end
 
   def save_index fn=File.join(@dir, "ferret")
@@ -210,41 +232,6 @@ class Index
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
   def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
-  ## Adds a message to the index. The message cannot already exist in
-  ## the index.
-  def add_message m
-    raise ArgumentError, "index already contains #{m.id}" if contains? m
-
-    source_id = 
-      if m.source.is_a? Integer
-        m.source
-      else
-        m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
-      end
-
-    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
-    d = {
-      :message_id => m.id,
-      :source_id => source_id,
-      :source_info => m.source_info,
-      :date => m.date.to_indexable_s,
-      :body => m.content,
-      :snippet => m.snippet,
-      :label => m.labels.join(" "),
-      :from => m.from ? m.from.email : "",
-      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
-      :subject => wrap_subj(Message.normalize_subj(m.subj)),
-      :refs => (m.refs + m.replytos).uniq.join(" "),
-    }
-
-    @index.add_document d
-    
-    docid, entry = load_entry_for_id m.id
-    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
-    raise "just added message #{m.id} but couldn't find it in a search" unless docid
-    true
-  end
-
   def drop_entry docno; @index.delete docno; end
 
   def load_entry_for_id mid
@@ -285,6 +272,12 @@ class Index
     contacts.keys.compact
   end
 
+  def load_sources fn=Redwood::SOURCE_FN
+    source_array = (Redwood::load_yaml_obj(fn) || []).map { |o| Recoverable.new o }
+    @sources = Hash[*(source_array).map { |s| [s.id, s] }.flatten]
+    @sources_dirty = false
+  end
+
 protected
 
   def parse_user_query_string str; @qparser.parse str; end
@@ -308,11 +301,6 @@ protected
     query
   end
 
-  def load_sources fn=Redwood::SOURCE_FN
-    @sources = Hash[*(Redwood::load_yaml_obj(fn) || []).map { |s| [s.id, s] }.flatten]
-    @sources_dirty = false
-  end
-
   def save_sources fn=Redwood::SOURCE_FN
     if @sources_dirty || @sources.any? { |id, s| s.dirty? }
       bakfn = fn + ".bak"

data/lib/sup/logger.rb

@@ -26,7 +26,7 @@ class Logger
 #    $stderr.puts s
     make_buf
     @mode << "#{Time.now}: #{s.chomp}\n"
-    $stderr.puts "[#{Time.now}] #{s.chomp}" unless @mode.buffer
+    $stderr.puts "[#{Time.now}] #{s.chomp}" unless BufferManager.instantiated? && @mode.buffer
   end
   
   def self.method_missing m, *a

data/lib/sup/maildir.rb

@@ -0,0 +1,127 @@
+require 'rmail'
+require 'uri'
+
+module Redwood
+
+## Maildir doesn't provide an ordered unique id, which is what Sup
+## requires to be really useful. So we must maintain, in memory, a
+## mapping between Sup "ids" (timestamps, essentially) and the
+## pathnames on disk.
+
+class Maildir < Source
+  SCAN_INTERVAL = 30 # seconds
+
+  def initialize uri, last_date=nil, usual=true, archived=false, id=nil
+    super
+    uri = URI(uri)
+
+    raise ArgumentError, "not a maildir URI" unless uri.scheme == "maildir"
+    raise ArgumentError, "maildir URI cannot have a host: #{uri.host}" if uri.host
+
+    @dir = uri.path
+    @ids = []
+    @ids_to_fns = {}
+    @last_scan = nil
+    @mutex = Mutex.new
+  end
+
+  def check
+    scan_mailbox
+    start = @ids.index(cur_offset || start_offset) or raise OutOfSyncSourceError, "Unknown message id #{cur_offset || start_offset}." # couldn't find the most recent email
+  end
+
+  def load_header id
+    scan_mailbox
+    with_file_for(id) { |f| MBox::read_header f }
+  end
+
+  def load_message id
+    scan_mailbox
+    with_file_for(id) { |f| RMail::Parser.read f }
+  end
+
+  def raw_header id
+    scan_mailbox
+    ret = ""
+    with_file_for(id) do |f|
+      until f.eof? || (l = f.gets) =~ /^$/
+        ret += l
+      end
+    end
+    ret
+  end
+
+  def raw_full_message id
+    scan_mailbox
+    with_file_for(id) { |f| f.readlines.join }
+  end
+
+  def scan_mailbox
+    return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
+
+    cdir = File.join(@dir, 'cur')
+    ndir = File.join(@dir, 'new')
+    
+    raise FatalSourceError, "#{cdir} not a directory" unless File.directory? cdir
+    raise FatalSourceError, "#{ndir} not a directory" unless File.directory? ndir
+
+    begin
+      @ids, @ids_to_fns = @mutex.synchronize do
+        ids, ids_to_fns = [], {}
+        (Dir[File.join(cdir, "*")] + Dir[File.join(ndir, "*")]).map do |fn|
+          id = make_id fn
+          ids << id
+          ids_to_fns[id] = fn
+        end
+        [ids.sort, ids_to_fns]
+      end
+    rescue SystemCallError, IOError => e
+      raise FatalSourceError, "Problem scanning Maildir directories: #{e.message}."
+    end
+    
+    @last_scan = Time.now
+  end
+
+  def each
+    scan_mailbox
+    start = @ids.index(cur_offset || start_offset) or raise OutOfSyncSourceError, "Unknown message id #{cur_offset || start_offset}." # couldn't find the most recent email
+
+    start.upto(@ids.length - 1) do |i|         
+      id = @ids[i]
+      self.cur_offset = id
+      yield id, (@ids_to_fns[id] =~ /,.*R.*$/ ? [] : [:unread])
+    end
+  end
+
+  def start_offset
+    scan_mailbox
+    @ids.first
+  end
+
+  def end_offset
+    scan_mailbox
+    @ids.last
+  end
+
+  def pct_done; 100.0 * (@ids.index(cur_offset) || 0).to_f / (@ids.length - 1).to_f; end
+
+private
+
+  def make_id fn
+    # use 7 digits for the size. why 7? seems nice.
+    sprintf("%d%07d", File.mtime(fn), File.size(fn)).to_i
+  end
+
+  def with_file_for id
+    fn = @ids_to_fns[id] or raise OutOfSyncSourceError, "No such id: #{id.inspect}."
+    begin
+      File.open(fn) { |f| yield f }
+    rescue SystemCallError, IOError => e
+      raise FatalSourceError, "Problem reading file for id #{id.inspect}: #{fn.inspect}: #{e.message}."
+    end
+  end
+end
+
+Redwood::register_yaml(Maildir, %w(uri cur_offset usual archived id))
+
+end

data/lib/sup/mbox/loader.rb

@@ -1,32 +1,36 @@
 require 'rmail'
+require 'uri'
 
 module Redwood
 module MBox
 
 class Loader < Source
-  attr_reader_cloned :labels
-
   def initialize uri_or_fp, start_offset=nil, usual=true, archived=false, id=nil
     super
 
     @mutex = Mutex.new
     @labels = [:unread]
-    @labels << :inbox unless archived?
 
     case uri_or_fp
     when String
-      raise ArgumentError, "not an mbox uri" unless uri_or_fp =~ %r!mbox://!
-
-      fn = uri_or_fp.sub(%r!^mbox://!, "")
+      uri = URI(uri_or_fp)
+      raise ArgumentError, "not an mbox uri" unless uri.scheme == "mbox"
+      raise ArgumentError, "mbox uri ('#{uri}') cannot have a host: #{uri.host}" if uri.host
       ## heuristic: use the filename as a label, unless the file
       ## has a path that probably represents an inbox.
-      @labels << File.basename(fn).intern unless File.dirname(fn) =~ /\b(var|usr|spool)\b/
-      @f = File.open fn
+      @labels << File.basename(uri.path).intern unless File.dirname(uri.path) =~ /\b(var|usr|spool)\b/
+      @f = File.open uri.path
     else
       @f = uri_or_fp
     end
   end
 
+  def check
+    if (cur_offset ||= start_offset) > end_offset
+      raise OutOfSyncSourceError, "mbox file is smaller than last recorded message offset. Messages have probably been deleted by another client."
+    end
+  end
+    
   def start_offset; 0; end
   def end_offset; File.size @f; end
 
@@ -36,9 +40,7 @@ class Loader < Source
       @f.seek offset
       l = @f.gets
       unless l =~ BREAK_RE
-        Redwood::log "#{to_s}: offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}"
-        self.broken_msg = "offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}. Run 'sup-import --rebuild #{to_s}' to correct this." 
-        raise SourceError, self.broken_msg
+        raise OutOfSyncSourceError, "mismatch in mbox file offset #{offset.inspect}: #{l.inspect}." 
       end
       header = MBox::read_header @f
     end
@@ -46,7 +48,6 @@ class Loader < Source
   end
 
   def load_message offset
-    raise SourceError, self.broken_msg if broken?
     @mutex.synchronize do
       @f.seek offset
       begin
@@ -54,13 +55,12 @@ class Loader < Source
           return RMail::Parser.read(input)
         end
       rescue RMail::Parser::Error => e
-        raise SourceError, "error parsing message with rmail: #{e.message}"
+        raise FatalSourceError, "error parsing mbox file: #{e.message}"
       end
     end
   end
 
   def raw_header offset
-    raise SourceError, self.broken_msg if broken?
     ret = ""
     @mutex.synchronize do
       @f.seek offset
@@ -72,7 +72,6 @@ class Loader < Source
   end
 
   def raw_full_message offset
-    raise SourceError, self.broken_msg if broken?
     ret = ""
     @mutex.synchronize do
       @f.seek offset
@@ -85,37 +84,40 @@ class Loader < Source
   end
 
   def next
-    raise SourceError, self.broken_msg if broken?
     returned_offset = nil
     next_offset = cur_offset
 
-    @mutex.synchronize do
-      @f.seek cur_offset
-
-      ## cur_offset could be at one of two places here:
-
-      ## 1. before a \n and a mbox separator, if it was previously at
-      ##    EOF and a new message was added; or,
-      ## 2. at the beginning of an mbox separator (in all other
-      ##    cases).
-
-      l = @f.gets or raise "next while at EOF"
-      if l =~ /^\s*$/ # case 1
-        returned_offset = @f.tell
-        @f.gets # now we're at a BREAK_RE, so skip past it
-      else # case 2
-        returned_offset = cur_offset
-        ## we've already skipped past the BREAK_RE, so just go
-      end
+    begin
+      @mutex.synchronize do
+        @f.seek cur_offset
 
-      while(line = @f.gets)
-        break if line =~ BREAK_RE
-        next_offset = @f.tell
+        ## cur_offset could be at one of two places here:
+
+        ## 1. before a \n and a mbox separator, if it was previously at
+        ##    EOF and a new message was added; or,
+        ## 2. at the beginning of an mbox separator (in all other
+        ##    cases).
+
+        l = @f.gets or raise "next while at EOF"
+        if l =~ /^\s*$/ # case 1
+          returned_offset = @f.tell
+          @f.gets # now we're at a BREAK_RE, so skip past it
+        else # case 2
+          returned_offset = cur_offset
+          ## we've already skipped past the BREAK_RE, so just go
+        end
+
+        while(line = @f.gets)
+          break if line =~ BREAK_RE
+          next_offset = @f.tell
+        end
       end
+    rescue SystemCallError, IOError => e
+      raise FatalSourceError, "Error reading #{@f.path}: #{e.message}"
     end
 
     self.cur_offset = next_offset
-    [returned_offset, labels]
+    [returned_offset, @labels.clone]
   end
 end
 

data/lib/sup/mbox/ssh-file.rb

@@ -106,14 +106,12 @@ class SSHFile
     @file_size = nil
     @offset = 0
     @say_id = nil
-    @broken_msg = nil
     @shell = nil
     @shell_mutex = nil
     @buf_mutex = Mutex.new
   end
 
   def to_s; "mbox+ssh://#@host/#@fn"; end ## TODO: remove this EVILness
-  def broken?; !@broken_msg.nil?; end
 
   def connect
     do_remote nil
@@ -164,7 +162,6 @@ private
   end
 
   def unsafe_connect
-    raise SSHFileError, @broken_msg if broken?
     return if @shell
 
     @key = [@host, @ssh_opts[:username]]
@@ -172,14 +169,13 @@ private
       @shell, @shell_mutex = @@shells_mutex.synchronize do
         unless @@shells.member? @key
           say "Opening SSH connection to #{@host} for #@fn..."
-          #raise SSHFileError, "simulated SSH file error"
           session = Net::SSH.start @host, @ssh_opts
           say "Starting SSH shell..."
           @@shells[@key] = [session.shell.sync, Mutex.new]
         end
         @@shells[@key]
       end
-
+      
       say "Checking for #@fn..."
       @shell_mutex.synchronize { raise Errno::ENOENT, @fn unless @shell.test("-e #@fn").status == 0 }
     ensure
@@ -190,29 +186,24 @@ private
   def do_remote cmd, expected_size=0
     retries = 0
     result = nil
-    begin
-      begin
-        unsafe_connect
-        if cmd
-          # MBox::debug "sending command: #{cmd.inspect}"
-          result = @shell_mutex.synchronize { x = @shell.send_command cmd; sleep 0.25; x }
-          raise SSHFileError, "Failure during remote command #{cmd.inspect}: #{(result.stderr || result.stdout || "")[0 .. 100]}" unless result.status == 0
-        end
 
-        ## Net::SSH::Exceptions seem to happen every once in a while for
-        ## no good reason.
-      rescue Net::SSH::Exception, *RECOVERABLE_ERRORS
-        if (retries += 1) <= 3
-          @@shells_mutex.synchronize do
-            @shell = nil
-            @@shells[@key] = nil
-          end
-          retry
+    begin
+      unsafe_connect
+      if cmd
+        # MBox::debug "sending command: #{cmd.inspect}"
+        result = @shell_mutex.synchronize { x = @shell.send_command cmd; sleep 0.25; x }
+        raise SSHFileError, "Failure during remote command #{cmd.inspect}: #{(result.stderr || result.stdout || "")[0 .. 100]}" unless result.status == 0
+      end
+      ## Net::SSH::Exceptions seem to happen every once in a while for
+      ## no good reason.
+    rescue Net::SSH::Exception, *RECOVERABLE_ERRORS
+      if (retries += 1) <= 3
+        @@shells_mutex.synchronize do
+          @shell = nil
+          @@shells[@key] = nil
         end
-        raise
+        retry
       end
-    rescue Net::SSH::Exception, SSHFileError, SystemCallError => e
-      @broken_msg = e.message
       raise
     end