sup 0.13.2.1 → 0.14.0

data/lib/sup/draft.rb

@@ -32,14 +32,17 @@ class DraftLoader < Source
   attr_accessor :dir
   yaml_properties
 
-  def initialize
-    dir = Redwood::DRAFT_DIR
+  def initialize dir=Redwood::DRAFT_DIR
     Dir.mkdir dir unless File.exists? dir
     super DraftManager.source_name, true, false
     @dir = dir
     @cur_offset = 0
   end
 
+  def properly_initialized?
+    !!(@dir && @cur_offset)
+  end
+
   def id; DraftManager.source_id; end
   def to_s; DraftManager.source_name; end
   def uri; DraftManager.source_name; end

data/lib/sup/index.rb

@@ -1,4 +1,5 @@
 ENV["XAPIAN_FLUSH_THRESHOLD"] = "1000"
+ENV["XAPIAN_CJK_NGRAM"] = "1"
 
 require 'xapian'
 require 'set'
@@ -6,13 +7,21 @@ require 'fileutils'
 require 'monitor'
 require 'chronic'
 
+require "sup/util/query"
 require "sup/interactive_lock"
 require "sup/hook"
 require "sup/logger/singleton"
 
 
-if ([Xapian.major_version, Xapian.minor_version, Xapian.revision] <=> [1,2,1]) < 0
-	fail "Xapian version 1.2.1 or higher required"
+if ([Xapian.major_version, Xapian.minor_version, Xapian.revision] <=> [1,2,15]) < 0
+  fail <<-EOF
+\n
+Xapian version 1.2.15 or higher required.
+If you have xapian-full-alaveteli installed,
+Please remove it by running `gem uninstall xapian-full-alaveteli`
+since it's been replaced by the xapian-ruby gem.
+
+  EOF
 end
 
 module Redwood
@@ -312,6 +321,48 @@ EOS
 
   class ParseError < StandardError; end
 
+  # Stemmed
+  NORMAL_PREFIX = {
+    'subject' => {:prefix => 'S', :exclusive => false},
+    'body' => {:prefix => 'B', :exclusive => false},
+    'from_name' => {:prefix => 'FN', :exclusive => false},
+    'to_name' => {:prefix => 'TN', :exclusive => false},
+    'name' => {:prefix => %w(FN TN), :exclusive => false},
+    'attachment' => {:prefix => 'A', :exclusive => false},
+    'email_text' => {:prefix => 'E', :exclusive => false},
+    '' => {:prefix => %w(S B FN TN A E), :exclusive => false},
+  }
+
+  # Unstemmed
+  BOOLEAN_PREFIX = {
+    'type' => {:prefix => 'K', :exclusive => true},
+    'from_email' => {:prefix => 'FE', :exclusive => false},
+    'to_email' => {:prefix => 'TE', :exclusive => false},
+    'email' => {:prefix => %w(FE TE), :exclusive => false},
+    'date' => {:prefix => 'D', :exclusive => true},
+    'label' => {:prefix => 'L', :exclusive => false},
+    'source_id' => {:prefix => 'I', :exclusive => true},
+    'attachment_extension' => {:prefix => 'O', :exclusive => false},
+    'msgid' => {:prefix => 'Q', :exclusive => true},
+    'id' => {:prefix => 'Q', :exclusive => true},
+    'thread' => {:prefix => 'H', :exclusive => false},
+    'ref' => {:prefix => 'R', :exclusive => false},
+    'location' => {:prefix => 'J', :exclusive => false},
+  }
+
+  PREFIX = NORMAL_PREFIX.merge BOOLEAN_PREFIX
+
+  COMPL_OPERATORS = %w[AND OR NOT]
+  COMPL_PREFIXES = (
+    %w[
+      from to
+      is has label
+      filename filetypem
+      before on in during after
+      limit
+    ] + NORMAL_PREFIX.keys + BOOLEAN_PREFIX.keys
+  ).map{|p|"#{p}:"} + COMPL_OPERATORS
+
   ## parse a query string from the user. returns a query object
   ## that can be passed to any index method with a 'query'
   ## argument.
@@ -441,7 +492,7 @@ EOS
       raise ParseError, "xapian query parser error: #{e}"
     end
 
-    debug "parsed xapian query: #{xapian_query.description}"
+    debug "parsed xapian query: #{Util::Query.describe(xapian_query)}"
 
     raise ParseError if xapian_query.nil? or xapian_query.empty?
     query[:qobj] = xapian_query
@@ -482,37 +533,6 @@ EOS
 
   private
 
-  # Stemmed
-  NORMAL_PREFIX = {
-    'subject' => {:prefix => 'S', :exclusive => false},
-    'body' => {:prefix => 'B', :exclusive => false},
-    'from_name' => {:prefix => 'FN', :exclusive => false},
-    'to_name' => {:prefix => 'TN', :exclusive => false},
-    'name' => {:prefix => %w(FN TN), :exclusive => false},
-    'attachment' => {:prefix => 'A', :exclusive => false},
-    'email_text' => {:prefix => 'E', :exclusive => false},
-    '' => {:prefix => %w(S B FN TN A E), :exclusive => false},
-  }
-
-  # Unstemmed
-  BOOLEAN_PREFIX = {
-    'type' => {:prefix => 'K', :exclusive => true},
-    'from_email' => {:prefix => 'FE', :exclusive => false},
-    'to_email' => {:prefix => 'TE', :exclusive => false},
-    'email' => {:prefix => %w(FE TE), :exclusive => false},
-    'date' => {:prefix => 'D', :exclusive => true},
-    'label' => {:prefix => 'L', :exclusive => false},
-    'source_id' => {:prefix => 'I', :exclusive => true},
-    'attachment_extension' => {:prefix => 'O', :exclusive => false},
-    'msgid' => {:prefix => 'Q', :exclusive => true},
-    'id' => {:prefix => 'Q', :exclusive => true},
-    'thread' => {:prefix => 'H', :exclusive => false},
-    'ref' => {:prefix => 'R', :exclusive => false},
-    'location' => {:prefix => 'J', :exclusive => false},
-  }
-
-  PREFIX = NORMAL_PREFIX.merge BOOLEAN_PREFIX
-
   MSGID_VALUENO = 0
   THREAD_VALUENO = 1
   DATE_VALUENO = 2

data/lib/sup/label.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Redwood
 
 class LabelManager
@@ -77,7 +79,7 @@ class LabelManager
 
   def save
     return unless @modified
-    File.open(@fn, "w") { |f| f.puts @labels.keys.sort_by { |l| l.to_s } }
+    File.open(@fn, "w:UTF-8") { |f| f.puts @labels.keys.sort_by { |l| l.to_s } }
     @new_labels = {}
   end
 end

data/lib/sup/message.rb

@@ -69,7 +69,9 @@ class Message
     return unless v
     return v unless v.is_a? String
     return unless v.size < MAX_HEADER_VALUE_SIZE # avoid regex blowup on spam
-    Rfc2047.decode_to $encoding, Iconv.easy_decode($encoding, 'ASCII', v)
+    d = v.dup
+    d = d.transcode($encoding, 'ASCII')
+    Rfc2047.decode_to $encoding, d
   end
 
   def parse_header encoded_header
@@ -109,7 +111,9 @@ class Message
       Time.now
     end
 
-    @subj = header["subject"] ? header["subject"].gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT
+    subj = header["subject"]
+    subj = subj ? subj.fix_encoding : nil
+    @subj = subj ? subj.gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT
     @to = Person.from_address_list header["to"]
     @cc = Person.from_address_list header["cc"]
     @bcc = Person.from_address_list header["bcc"]
@@ -260,6 +264,8 @@ class Message
       rescue SourceError, SocketError, RMail::EncodingUnsupportedError => e
         warn "problem reading message #{id}"
         [Chunk::Text.new(error_message.split("\n"))]
+
+        debug "could not load message: #{location.inspect}, exception: #{e.inspect}"
       end
   end
 
@@ -524,7 +530,7 @@ private
           ## if there's no charset, use the current encoding as the charset.
           ## this ensures that the body is normalized to avoid non-displayable
           ## characters
-          body = Iconv.easy_decode($encoding, m.charset || $encoding, m.decode)
+          body = m.decode.transcode($encoding, m.charset)
         else
           body = ""
         end
@@ -546,7 +552,7 @@ private
       msg = RMail::Message.new
       msg.body = gpg.join("\n")
 
-      body = Iconv.easy_decode(encoding_to, encoding_from, body)
+      body = body.transcode(encoding_to, encoding_from)
       lines = body.split("\n")
       sig = lines.between(GPG_SIGNED_START, GPG_SIG_START)
       startidx = lines.index(GPG_SIGNED_START)

data/lib/sup/message_chunks.rb

@@ -1,6 +1,5 @@
 require 'tempfile'
 require 'rbconfig'
-require 'shellwords'
 
 ## Here we define all the "chunks" that a message is parsed
 ## into. Chunks are used by ThreadViewMode to render a message. Chunks
@@ -60,8 +59,6 @@ end
 module Redwood
 module Chunk
   class Attachment
-    ## please see note in write_to_disk on important usage
-    ## of quotes to avoid remote command injection.
     HookManager.register "mime-decode", <<EOS
 Decodes a MIME attachment into text form. The text will be displayed
 directly in Sup. For attachments that you wish to use a separate program
@@ -78,9 +75,6 @@ Return value:
   The decoded text of the attachment, or nil if not decoded.
 EOS
 
-
-    ## please see note in write_to_disk on important usage
-    ## of quotes to avoid remote command injection.
     HookManager.register "mime-view", <<EOS
 Views a non-text MIME attachment. This hook allows you to run
 third-party programs for attachments that require such a thing (e.g.
@@ -106,18 +100,8 @@ EOS
     attr_reader :content_type, :filename, :lines, :raw_content
     bool_reader :quotable
 
-    ## store tempfile objects as class variables so that they
-    ## are not removed when the viewing process returns. they
-    ## should be garbage collected when the class variable is removed.
-    @@view_tempfiles = []
-
     def initialize content_type, filename, encoded_content, sibling_types
       @content_type = content_type.downcase
-      if Shellwords.escape(@content_type) != @content_type
-        warn "content_type #{@content_type} is not safe, changed to application/octet-stream"
-        @content_type = 'application/octet-stream'
-      end
-
       @filename = filename
       @quotable = false # changed to true if we can parse it through the
                         # mime-decode hook, or if it's plain text
@@ -132,9 +116,7 @@ EOS
       when /^text\/plain\b/
         @raw_content
       else
-        ## please see note in write_to_disk on important usage
-        ## of quotes to avoid remote command injection.
-        HookManager.run "mime-decode", :content_type => @content_type,
+        HookManager.run "mime-decode", :content_type => content_type,
                         :filename => lambda { write_to_disk },
                         :charset => encoded_content.charset,
                         :sibling_types => sibling_types
@@ -142,7 +124,7 @@ EOS
 
       @lines = nil
       if text
-        text = text.transcode(encoded_content.charset || $encoding)
+        text = text.transcode(encoded_content.charset || $encoding, text.encoding)
         @lines = text.gsub("\r\n", "\n").gsub(/\t/, "        ").gsub(/\r/, "").split("\n")
         @quotable = true
       end
@@ -165,13 +147,11 @@ EOS
     def initial_state; :open end
     def viewable?; @lines.nil? end
     def view_default! path
-      ## please see note in write_to_disk on important usage
-      ## of quotes to avoid remote command injection.
       case RbConfig::CONFIG['arch']
         when /darwin/
-          cmd = "open #{path}"
+          cmd = "open '#{path}'"
         else
-          cmd = "/usr/bin/run-mailcap --action=view #{@content_type}:#{path}"
+          cmd = "/usr/bin/run-mailcap --action=view '#{@content_type}:#{path}'"
       end
       debug "running: #{cmd.inspect}"
       BufferManager.shell_out(cmd)
@@ -179,31 +159,17 @@ EOS
     end
 
     def view!
-      ## please see note in write_to_disk on important usage
-      ## of quotes to avoid remote command injection.
-      write_to_disk do |file|
-
-        @@view_tempfiles.push file # make sure the tempfile is not garbage collected before sup stops
-
-        ret = HookManager.run "mime-view", :content_type => @content_type,
-                                           :filename => file.path
-        ret || view_default!(file.path)
-      end
+      path = write_to_disk
+      ret = HookManager.run "mime-view", :content_type => @content_type,
+                                         :filename => path
+      ret || view_default!(path)
     end
 
-    ## note that the path returned from write_to_disk is
-    ## Shellwords.escaped and is intended to be used without single
-    ## or double quotes. the use of either opens sup up for remote
-    ## code injection through the file name.
     def write_to_disk
-      begin
-        file = Tempfile.new(["sup", Shellwords.escape(@filename.gsub("/", "_")) || "sup-attachment"])
-        file.print @raw_content
-        yield file if block_given?
-        return file.path
-      ensure
-        file.close
-      end
+      file = Tempfile.new(["sup", @filename.gsub("/", "_") || "sup-attachment"])
+      file.print @raw_content
+      file.close
+      file.path
     end
 
     ## used when viewing the attachment as text
@@ -263,7 +229,7 @@ EOS
   class EnclosedMessage
     attr_reader :lines
     def initialize from, to, cc, date, subj
-      @from = from ? "unknown sender" : from.full_address
+      @from = from ? "unknown sender" : from.full_adress
       @to = to ? "" : to.map { |p| p.full_address }.join(", ")
       @cc = cc ? "" : cc.map { |p| p.full_address }.join(", ")
       if date

data/lib/sup/modes/compose_mode.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Redwood
 
 class ComposeMode < EditMessageMode

data/lib/sup/modes/edit_message_mode.rb

@@ -484,10 +484,10 @@ protected
       m = build_message date
 
       if HookManager.enabled? "sendmail"
-    if not HookManager.run "sendmail", :message => m, :account => acct
-          warn "Sendmail hook was not successful"
-          return false
-    end
+        if not HookManager.run "sendmail", :message => m, :account => acct
+              warn "Sendmail hook was not successful"
+              return false
+        end
       else
         IO.popen(acct.sendmail, "w") { |p| p.puts m }
         raise SendmailCommandFailed, "Couldn't execute #{acct.sendmail}" unless $? == 0

data/lib/sup/modes/inbox_mode.rb

@@ -1,4 +1,4 @@
-require 'sup'
+require "sup/modes/thread_index_mode"
 
 module Redwood
 

data/lib/sup/modes/thread_index_mode.rb

@@ -856,14 +856,14 @@ protected
 
       abbrev =
         if cur_width + name.display_length > from_width
-          name[0 ... (from_width - cur_width - 1)] + "."
+          name.slice_by_display_length(from_width - cur_width - 1) + "."
         elsif cur_width + name.display_length == from_width
-          name[0 ... (from_width - cur_width)]
+          name.slice_by_display_length(from_width - cur_width)
         else
           if last
-            name[0 ... (from_width - cur_width)]
+            name.slice_by_display_length(from_width - cur_width)
           else
-            name[0 ... (from_width - cur_width - 1)] + ","
+            name.slice_by_display_length(from_width - cur_width - 1) + ","
           end
         end
 

data/lib/sup/modes/thread_view_mode.rb

@@ -846,6 +846,10 @@ private
       else
         width = buffer.content_width
       end
+      # lines can apparently be both String and Array, convert to Array for map.
+      if lines.kind_of? String
+        lines = lines.lines.to_a
+      end
       lines = lines.map { |l| l.chomp.wrap width if l }.flatten
     end
     return lines

data/lib/sup/poll.rb

@@ -171,7 +171,7 @@ EOS
   ## labels and locations set correctly. The Messages are saved to or removed
   ## from the index after being yielded.
   def poll_from source, opts={}
-    debug "trying to acquiring poll lock for: #{source}.."
+    debug "trying to acquire poll lock for: #{source}.."
     if source.poll_lock.try_lock
       debug "lock acquired for: #{source}."
       begin

data/lib/sup/rfc2047.rb

@@ -16,8 +16,6 @@
 #
 # This file is distributed under the same terms as Ruby.
 
-require 'iconv'
-
 module Rfc2047
   WORD = %r{=\?([!\#$%&'*+-/0-9A-Z\\^\`a-z{|}~]+)\?([BbQq])\?([!->@-~]+)\?=} # :nodoc: 'stupid ruby-mode
   WORDSEQ = %r{(#{WORD.source})\s+(?=#{WORD.source})}
@@ -52,7 +50,7 @@ module Rfc2047
         # WORD.
       end
 
-      Iconv.easy_decode(target, charset, text)
+      text.transcode(target, charset)
     end
   end
 end

data/lib/sup/sent.rb

@@ -25,8 +25,13 @@ class SentManager
   end
 
   def write_sent_message date, from_email, &block
-    @source.store_message date, from_email, &block
-    PollManager.poll_from @source
+    ::Thread.new do
+      debug "store the sent message (locking sent source..)"
+      @source.poll_lock.synchronize do
+        @source.store_message date, from_email, &block
+      end
+      PollManager.poll_from @source
+    end
   end
 end
 

data/lib/sup/source.rb

@@ -22,30 +22,23 @@ class Source
   ## read, delete them, or anything else. (Well, it's nice to be able
   ## to delete them, but that is optional.)
   ##
-  ## On the other hand, Sup assumes that you can assign each message a
-  ## unique integer id, such that newer messages have higher ids than
-  ## earlier ones, and that those ids stay constant across sessions
-  ## (in the absence of some other client going in and fucking
-  ## everything up). For example, for mboxes I use the file offset of
-  ## the start of the message. If a source does NOT have that
-  ## capability, e.g. IMAP, then you have to do a little more work to
-  ## simulate it.
+  ## Messages are identified internally based on the message id, and stored
+  ## with an unique document id. Along with the message, source information
+  ## that can contain arbitrary fields (set up by the source) is stored. This
+  ## information will be passed back to the source when a message in the
+  ## index (Sup database) needs to be identified to its source, e.g. when
+  ## re-reading or modifying a unique message.
   ##
   ## To write a new source, subclass this class, and implement:
   ##
-  ## - start_offset
-  ## - end_offset (exclusive!) (or, #done?)
+  ## - initialize
   ## - load_header offset
   ## - load_message offset
   ## - raw_header offset
   ## - raw_message offset
-  ## - check (optional)
+  ## - store_message (optional)
+  ## - poll (loads new messages)
   ## - go_idle (optional)
-  ## - next (or each, if you prefer): should return a message and an
-  ##   array of labels.
-  ##
-  ## ... where "offset" really means unique id. (You can tell I
-  ## started with mbox.)
   ##
   ## All exceptions relating to accessing the source must be caught
   ## and rethrown as FatalSourceErrors or OutOfSyncSourceErrors.
@@ -57,8 +50,7 @@ class Source
   ## Finally, be sure the source is thread-safe, since it WILL be
   ## pummelled from multiple threads at once.
   ##
-  ## Examples for you to look at: mbox/loader.rb, imap.rb, and
-  ## maildir.rb.
+  ## Examples for you to look at: mbox.rb and maildir.rb.
 
   bool_accessor :usual, :archived
   attr_reader :uri
@@ -211,9 +203,9 @@ class SourceManager
     end
   end
 
-  def save_sources fn=Redwood::SOURCE_FN
+  def save_sources fn=Redwood::SOURCE_FN, force=false
     @source_mutex.synchronize do
-      if @sources_dirty
+      if @sources_dirty || force
         Redwood::save_yaml_obj sources, fn, false, true
       end
       @sources_dirty = false