sup 0.11 → 0.12

data/lib/sup/client.rb

@@ -0,0 +1,92 @@
+require 'sup/protocol'
+
+module Redwood
+
+class Client < EM::P::RedwoodClient
+  def initialize *a
+    @next_tag = 1
+    @cbs = {}
+    super *a
+  end
+
+  def mktag &b
+    @next_tag.tap do |x|
+      @cbs[x] = b
+      @next_tag += 1
+    end
+  end
+
+  def rmtag tag
+    @cbs.delete tag
+  end
+
+  def query qstr, offset, limit, raw, &b
+    tag = mktag do |type,tag,args|
+      if type == 'message'
+        b.call args
+      else
+        fail unless type == 'done'
+        b.call nil
+        rmtag tag
+      end
+    end
+    send_message 'query', tag,
+                 'query' => qstr,
+                 'offset' => offset,
+                 'limit' => limit,
+                 'raw' => raw
+  end
+
+  def count qstr, &b
+    tag = mktag do |type,tag,args|
+      b.call args['count']
+      rmtag tag
+    end
+    send_message 'count', tag,
+                 'query' => qstr
+  end
+
+  def label qstr, add, remove, &b
+    tag = mktag do |type,tag,args|
+      b.call
+      rmtag tag
+    end
+    send_message 'label', tag,
+                 'query' => qstr,
+                 'add' => add,
+                 'remove' => remove
+  end
+
+  def add raw, labels, &b
+    tag = mktag do |type,tag,args|
+      b.call
+      rmtag tag
+    end
+    send_message 'add', tag,
+                 'raw' => raw,
+                 'labels' => labels
+  end
+
+  def thread msg_id, raw, &b
+    tag = mktag do |type,tag,args|
+      if type == 'message'
+        b.call args
+      else
+        fail unless type == 'done'
+        b.call nil
+        rmtag tag
+      end
+    end
+
+    send_message 'thread', tag,
+                 'message_id' => msg_id,
+                 'raw' => raw
+  end
+
+  def receive_message type, tag, args
+    cb = @cbs[tag] or fail "invalid tag #{tag.inspect}"
+    cb[type, tag, args]
+  end
+end
+
+end

data/lib/sup/crypto.rb

@@ -45,14 +45,15 @@ EOS
 
     sig_fn = Tempfile.new "redwood.signature"; sig_fn.close
 
-    message = run_gpg "--output #{sig_fn.path} --yes --armor --detach-sign --textmode --local-user '#{from}' #{payload_fn.path}", :interactive => true
+    sign_user_opts = gen_sign_user_opts from
+    message = run_gpg "--output #{sig_fn.path} --yes --armor --detach-sign --textmode --digest-algo sha256 #{sign_user_opts} #{payload_fn.path}", :interactive => true
     unless $?.success?
       info "Error while running gpg: #{message}"
       raise Error, "GPG command failed. See log for details."
     end
 
     envelope = RMail::Message.new
-    envelope.header["Content-Type"] = 'multipart/signed; protocol=application/pgp-signature; micalg=pgp-sha1'
+    envelope.header["Content-Type"] = 'multipart/signed; protocol=application/pgp-signature; micalg=pgp-sha256'
 
     envelope.add_part payload
     signature = RMail::Message.make_attachment IO.read(sig_fn.path), "application/pgp-signature", nil, "signature.asc"
@@ -68,7 +69,8 @@ EOS
     encrypted_fn = Tempfile.new "redwood.encrypted"; encrypted_fn.close
 
     recipient_opts = (to + [ from ] ).map { |r| "--recipient '<#{r}>'" }.join(" ")
-    sign_opts = sign ? "--sign --local-user '#{from}'" : ""
+    sign_opts = ""
+    sign_opts = "--sign --digest-algo sha256 " + gen_sign_user_opts(from) if sign
     message = run_gpg "--output #{encrypted_fn.path} --yes --armor --encrypt --textmode #{sign_opts} #{recipient_opts} #{payload_fn.path}", :interactive => true
     unless $?.success?
       info "Error while running gpg: #{message}"
@@ -86,7 +88,7 @@ EOS
     control.body = "Version: 1\n"
 
     envelope = RMail::Message.new
-    envelope.header["Content-Type"] = 'multipart/encrypted; protocol="application/pgp-encrypted"'
+    envelope.header["Content-Type"] = 'multipart/encrypted; protocol=application/pgp-encrypted'
 
     envelope.add_part control
     envelope.add_part encrypted_payload
@@ -97,43 +99,57 @@ EOS
     encrypt from, to, payload, true
   end
 
-  def verify payload, signature # both RubyMail::Message objects
-    return unknown_status(cant_find_binary) unless @cmd
-
-    payload_fn = Tempfile.new "redwood.payload"
-    payload_fn.write format_payload(payload)
-    payload_fn.close
-
-    signature_fn = Tempfile.new "redwood.signature"
-    signature_fn.write signature.decode
-    signature_fn.close
-
-    output = run_gpg "--verify #{signature_fn.path} #{payload_fn.path}"
+  def verified_ok? output, rc
     output_lines = output.split(/\n/)
 
     if output =~ /^gpg: (.* signature from .*$)/
-      if $? == 0
+      if rc == 0
         Chunk::CryptoNotice.new :valid, $1, output_lines
       else
         Chunk::CryptoNotice.new :invalid, $1, output_lines
       end
+    elsif output_lines.length == 0 && rc == 0
+      # the message wasn't signed
+      Chunk::CryptoNotice.new :valid, "Encrypted message wasn't signed", output_lines
     else
       unknown_status output_lines
     end
   end
 
+  def verify payload, signature, detached=true # both RubyMail::Message objects
+    return unknown_status(cant_find_binary) unless @cmd
+
+    if detached
+      payload_fn = Tempfile.new "redwood.payload"
+      payload_fn.write format_payload(payload)
+      payload_fn.close
+    end
+
+    signature_fn = Tempfile.new "redwood.signature"
+    signature_fn.write signature.decode
+    signature_fn.close
+
+    if detached
+      output = run_gpg "--verify #{signature_fn.path} #{payload_fn.path}"
+    else
+      output = run_gpg "--verify #{signature_fn.path}"
+    end
+
+    self.verified_ok? output, $?
+  end
+
   ## returns decrypted_message, status, desc, lines
-  def decrypt payload # a RubyMail::Message object
+  def decrypt payload, armor=false # a RubyMail::Message object
     return unknown_status(cant_find_binary) unless @cmd
 
-    payload_fn = Tempfile.new "redwood.payload"
+    payload_fn = Tempfile.new(["redwood.payload", ".asc"])
     payload_fn.write payload.to_s
     payload_fn.close
 
     output_fn = Tempfile.new "redwood.output"
     output_fn.close
 
-    message = run_gpg "--output #{output_fn.path} --yes --decrypt #{payload_fn.path}", :interactive => true
+    message = run_gpg "--output #{output_fn.path} --skip-verify --yes --decrypt #{payload_fn.path}", :interactive => true
 
     unless $?.success?
       info "Error while running gpg: #{message}"
@@ -143,34 +159,43 @@ EOS
     output = IO.read output_fn.path
     output.force_encoding Encoding::ASCII_8BIT if output.respond_to? :force_encoding
 
-    ## there's probably a better way to do this, but we're using the output to
-    ## look for a valid signature being present.
-
-    sig = case message
-    when /^gpg: (Good signature from .*$)/i
-      Chunk::CryptoNotice.new :valid, $1, message.split("\n")
-    when /^gpg: (Bad signature from .*$)/i
-      Chunk::CryptoNotice.new :invalid, $1, message.split("\n")
-    end
-
-    # This is gross. This decrypted payload could very well be a multipart
-    # element itself, as opposed to a simple payload. For example, a
-    # multipart/signed element, like those generated by Mutt when encrypting
-    # and signing a message (instead of just clearsigning the body).
-    # Supposedly, decrypted_payload being a multipart element ought to work
-    # out nicely because Message::multipart_encrypted_to_chunks() runs the
-    # decrypted message through message_to_chunks() again to get any
-    # children. However, it does not work as intended because these inner
-    # payloads need not carry a MIME-Version header, yet they are fed to
-    # RMail as a top-level message, for which the MIME-Version header is
-    # required. This causes for the part not to be detected as multipart,
-    # hence being shown as an attachment. If we detect this is happening,
-    # we force the decrypted payload to be interpreted as MIME.
-    msg = RMail::Parser.read output
-    if msg.header.content_type =~ %r{^multipart/} && !msg.multipart?
-      output = "MIME-Version: 1.0\n" + output
-      output.force_encoding Encoding::ASCII_8BIT if output.respond_to? :force_encoding
+    ## check for a valid signature in an extra run because gpg aborts if the
+    ## signature cannot be verified (but it is still able to decrypt)
+    sigoutput = run_gpg "#{payload_fn.path}"
+    sig = self.verified_ok? sigoutput, $?
+
+    if armor
+      msg = RMail::Message.new
+      # Look for Charset, they are put before the base64 crypted part
+      charsets = payload.body.split("\n").grep(/^Charset:/)
+      if !charsets.empty? and charsets[0] =~ /^Charset: (.+)$/
+        output = Iconv.easy_decode($encoding, $1, output)
+      end
+      msg.body = output
+    else
+      # It appears that some clients use Windows new lines - CRLF - but RMail
+      # splits the body and header on "\n\n". So to allow the parse below to 
+      # succeed, we will convert the newlines to what RMail expects
+      output = output.gsub(/\r\n/, "\n")
+      # This is gross. This decrypted payload could very well be a multipart
+      # element itself, as opposed to a simple payload. For example, a
+      # multipart/signed element, like those generated by Mutt when encrypting
+      # and signing a message (instead of just clearsigning the body).
+      # Supposedly, decrypted_payload being a multipart element ought to work
+      # out nicely because Message::multipart_encrypted_to_chunks() runs the
+      # decrypted message through message_to_chunks() again to get any
+      # children. However, it does not work as intended because these inner
+      # payloads need not carry a MIME-Version header, yet they are fed to
+      # RMail as a top-level message, for which the MIME-Version header is
+      # required. This causes for the part not to be detected as multipart,
+      # hence being shown as an attachment. If we detect this is happening,
+      # we force the decrypted payload to be interpreted as MIME.
       msg = RMail::Parser.read output
+      if msg.header.content_type =~ %r{^multipart/} && !msg.multipart?
+        output = "MIME-Version: 1.0\n" + output
+        output.force_encoding Encoding::ASCII_8BIT if output.respond_to? :force_encoding
+        msg = RMail::Parser.read output
+      end
     end
     notice = Chunk::CryptoNotice.new :valid, "This message has been decrypted for display"
     [notice, sig, msg]
@@ -189,12 +214,29 @@ private
   ## here's where we munge rmail output into the format that signed/encrypted
   ## PGP/GPG messages should be
   def format_payload payload
-    payload.to_s.gsub(/(^|[^\r])\n/, "\\1\r\n").gsub(/^MIME-Version: .*\r\n/, "")
+    payload.to_s.gsub(/(^|[^\r])\n/, "\\1\r\n")
+  end
+
+  # logic is:
+  # if    gpgkey set for this account, then use that
+  # elsif only one account,            then leave blank so gpg default will be user
+  # else                                    set --local-user from_email_address
+  def gen_sign_user_opts from
+    account = AccountManager.account_for from
+    if !account.gpgkey.nil?
+      opts = "--local-user '#{account.gpgkey}'"
+    elsif AccountManager.user_emails.length == 1
+      # only one account
+      opts = ""
+    else
+      opts = "--local-user '#{from}'" 
+    end
+    opts
   end
 
   def run_gpg args, opts={}
     args = HookManager.run("gpg-args", { :args => args }) || args
-    cmd = "#{@cmd} #{args}"
+    cmd = "LC_MESSAGES=C #{@cmd} #{args}"
     if opts[:interactive] && BufferManager.instantiated?
       output_fn = Tempfile.new "redwood.output"
       output_fn.close

data/lib/sup/draft.rb

@@ -11,20 +11,13 @@ class DraftManager
 
   def self.source_name; "sup://drafts"; end
   def self.source_id; 9999; end
-  def new_source; @source = Recoverable.new DraftLoader.new; end
+  def new_source; @source = DraftLoader.new; end
 
   def write_draft
     offset = @source.gen_offset
     fn = @source.fn_for_offset offset
     File.open(fn, "w") { |f| yield f }
-
-    my_message = nil
-    PollManager.each_message_from(@source) do |m|
-      PollManager.add_new_message m
-      my_message = m
-    end
-
-    my_message
+    PollManager.poll_from @source
   end
 
   def discard m
@@ -37,31 +30,35 @@ end
 
 class DraftLoader < Source
   attr_accessor :dir
-  yaml_properties :cur_offset
+  yaml_properties
 
-  def initialize cur_offset=0
+  def initialize
     dir = Redwood::DRAFT_DIR
     Dir.mkdir dir unless File.exists? dir
-    super DraftManager.source_name, cur_offset, true, false
+    super DraftManager.source_name, true, false
     @dir = dir
+    @cur_offset = 0
   end
 
   def id; DraftManager.source_id; end
   def to_s; DraftManager.source_name; end
   def uri; DraftManager.source_name; end
 
-  def each
+  def poll
     ids = get_ids
     ids.each do |id|
-      if id >= cur_offset
-        self.cur_offset = id + 1
-        yield [id, [:draft, :inbox]]
+      if id >= @cur_offset
+        @cur_offset = id + 1
+        yield :add,
+          :info => id,
+          :labels => [:draft, :inbox],
+          :progress => 0.0
       end
     end
   end
 
   def gen_offset
-    i = cur_offset
+    i = 0
     while File.exists? fn_for_offset(i)
       i += 1
     end

data/lib/sup/hook.rb

@@ -61,10 +61,15 @@ class HookManager
 
   include Singleton
 
+  @descs = {}
+
+  class << self
+    attr_reader :descs
+  end
+
   def initialize dir
     @dir = dir
     @hooks = {}
-    @descs = {}
     @contexts = {}
     @tags = {}
 
@@ -90,17 +95,17 @@ class HookManager
     result
   end
 
-  def register name, desc
+  def self.register name, desc
     @descs[name] = desc
   end
 
   def print_hooks f=$stdout
 puts <<EOS
-Have #{@descs.size} registered hooks:
+Have #{HookManager.descs.size} registered hooks:
 
 EOS
 
-    @descs.sort.each do |name, desc|
+    HookManager.descs.sort.each do |name, desc|
       f.puts <<EOS
 #{name}
 #{"-" * name.length}
@@ -112,7 +117,7 @@ EOS
 
   def enabled? name; !hook_for(name).nil? end
 
-  def clear; @hooks.clear; end
+  def clear; @hooks.clear; BufferManager.flash "Hooks cleared" end
   def clear_one k; @hooks.delete k; end
 
 private

data/lib/sup/index.rb

@@ -3,6 +3,7 @@ ENV["XAPIAN_FLUSH_THRESHOLD"] = "1000"
 require 'xapian'
 require 'set'
 require 'fileutils'
+require 'monitor'
 
 begin
   require 'chronic'
@@ -21,7 +22,7 @@ class Index
   include InteractiveLock
 
   STEM_LANGUAGE = "english"
-  INDEX_VERSION = '2'
+  INDEX_VERSION = '4'
 
   ## dates are converted to integers for xapian, and are used for document ids,
   ## so we must ensure they're reasonably valid. this typically only affect
@@ -48,6 +49,7 @@ EOS
 
   def initialize dir=BASE_DIR
     @dir = dir
+    FileUtils.mkdir_p @dir
     @lock = Lockfile.new lockfile, :retries => 0, :max_age => nil
     @sync_worker = nil
     @sync_queue = Queue.new
@@ -104,15 +106,16 @@ EOS
       @xapian = Xapian::WritableDatabase.new(path, Xapian::DB_OPEN)
       db_version = @xapian.get_metadata 'version'
       db_version = '0' if db_version.empty?
-      if db_version == '1'
-        info "Upgrading index format 1 to 2"
+      if false
+        info "Upgrading index format #{db_version} to #{INDEX_VERSION}"
         @xapian.set_metadata 'version', INDEX_VERSION
       elsif db_version != INDEX_VERSION
-        fail "This Sup version expects a v#{INDEX_VERSION} index, but you have an existing v#{db_version} index. Please downgrade to your previous version and dump your labels before upgrading to this version (then run sup-sync --restore)."
+        fail "This Sup version expects a v#{INDEX_VERSION} index, but you have an existing v#{db_version} index. Please run sup-dump to save your labels, move #{path} out of the way, and run sup-sync --restore."
       end
     else
       @xapian = Xapian::WritableDatabase.new(path, Xapian::DB_CREATE)
       @xapian.set_metadata 'version', INDEX_VERSION
+      @xapian.set_metadata 'rescue-version', '0'
     end
     @enquire = Xapian::Enquire.new @xapian
     @enquire.weighting_scheme = Xapian::BoolWeight.new
@@ -193,11 +196,15 @@ EOS
     entry = synchronize { get_entry id }
     return unless entry
 
-    source = SourceManager[entry[:source_id]]
-    raise "invalid source #{entry[:source_id]}" unless source
+    locations = entry[:locations].map do |source_id,source_info|
+      source = SourceManager[source_id]
+      raise "invalid source #{source_id}" unless source
+      Location.new source, source_info
+    end
 
-    m = Message.new :source => source, :source_info => entry[:source_info],
-                    :labels => entry[:labels], :snippet => entry[:snippet]
+    m = Message.new :locations => locations,
+                    :labels => entry[:labels],
+                    :snippet => entry[:snippet]
 
     mk_person = lambda { |x| Person.new(*x.reverse!) }
     entry[:from] = mk_person[entry[:from]]
@@ -260,6 +267,26 @@ EOS
     synchronize { get_entry(id)[:source_id] }
   end
 
+  ## Yields each tearm in the index that starts with prefix
+  def each_prefixed_term prefix
+    term = @xapian._dangerous_allterms_begin prefix
+    lastTerm = @xapian._dangerous_allterms_end prefix
+    until term.equals lastTerm
+      yield term.term
+      term.next
+    end
+    nil
+  end
+
+  ## Yields (in lexicographical order) the source infos of all locations from
+  ## the given source with the given source_info prefix
+  def each_source_info source_id, prefix='', &b
+    prefix = mkterm :location, source_id, prefix
+    each_prefixed_term prefix do |x|
+      yield x[prefix.length..-1]
+    end
+  end
+
   class ParseError < StandardError; end
 
   ## parse a query string from the user. returns a query object
@@ -288,22 +315,6 @@ EOS
       end
     end
 
-    ## if we see a label:deleted or a label:spam term anywhere in the query
-    ## string, we set the extra load_spam or load_deleted options to true.
-    ## bizarre? well, because the query allows arbitrary parenthesized boolean
-    ## expressions, without fully parsing the query, we can't tell whether
-    ## the user is explicitly directing us to search spam messages or not.
-    ## e.g. if the string is -(-(-(-(-label:spam)))), does the user want to
-    ## search spam messages or not?
-    ##
-    ## so, we rely on the fact that turning these extra options ON turns OFF
-    ## the adding of "-label:deleted" or "-label:spam" terms at the very
-    ## final stage of query processing. if the user wants to search spam
-    ## messages, not adding that is the right thing; if he doesn't want to
-    ## search spam messages, then not adding it won't have any effect.
-    query[:load_spam] = true if subs =~ /\blabel:spam\b/
-    query[:load_deleted] = true if subs =~ /\blabel:deleted\b/
-
     ## gmail style "is" operator
     subs = subs.gsub(/\b(is|has):(\S+)\b/) do
       field, label = $1, $2
@@ -321,6 +332,29 @@ EOS
       end
     end
 
+    ## labels are stored lower-case in the index
+    subs = subs.gsub(/\blabel:(\S+)\b/) do
+      label = $1
+      "label:#{label.downcase}"
+    end
+
+    ## if we see a label:deleted or a label:spam term anywhere in the query
+    ## string, we set the extra load_spam or load_deleted options to true.
+    ## bizarre? well, because the query allows arbitrary parenthesized boolean
+    ## expressions, without fully parsing the query, we can't tell whether
+    ## the user is explicitly directing us to search spam messages or not.
+    ## e.g. if the string is -(-(-(-(-label:spam)))), does the user want to
+    ## search spam messages or not?
+    ##
+    ## so, we rely on the fact that turning these extra options ON turns OFF
+    ## the adding of "-label:deleted" or "-label:spam" terms at the very
+    ## final stage of query processing. if the user wants to search spam
+    ## messages, not adding that is the right thing; if he doesn't want to
+    ## search spam messages, then not adding it won't have any effect.
+    query[:load_spam] = true if subs =~ /\blabel:spam\b/
+    query[:load_deleted] = true if subs =~ /\blabel:deleted\b/
+    query[:load_killed] = true if subs =~ /\blabel:killed\b/
+
     ## gmail style attachments "filename" and "filetype" searches
     subs = subs.gsub(/\b(filename|filetype):(\((.+?)\)\B|(\S+)\b)/) do
       field, name = $1, ($3 || $4)
@@ -453,6 +487,7 @@ EOS
     'id' => 'Q',
     'thread' => 'H',
     'ref' => 'R',
+    'location' => 'J',
   }
 
   PREFIX = NORMAL_PREFIX.merge BOOLEAN_PREFIX
@@ -514,7 +549,7 @@ EOS
 
   def get_entry id
     return unless doc = find_doc(id)
-    Marshal.load doc.data
+    doc.entry
   end
 
   def thread_killed? thread_id
@@ -547,6 +582,7 @@ EOS
     pos_terms.concat(labels.map { |l| mkterm(:label,l) })
     pos_terms << opts[:qobj] if opts[:qobj]
     pos_terms << mkterm(:source_id, opts[:source_id]) if opts[:source_id]
+    pos_terms << mkterm(:location, *opts[:location]) if opts[:location]
 
     if opts[:participants]
       participant_terms = opts[:participants].map { |p| [:from,:to].map { |d| mkterm(:email, d, (Redwood::Person === p) ? p.email : p) } }.flatten
@@ -575,8 +611,7 @@ EOS
 
     entry = {
       :message_id => m.id,
-      :source_id => m.source.id,
-      :source_info => m.source_info,
+      :locations => m.locations.map { |x| [x.source.id, x.info] },
       :date => truncate_date(m.date),
       :snippet => snippet,
       :labels => m.labels.to_a,
@@ -595,6 +630,7 @@ EOS
       index_message_static m, doc, entry
     end
 
+    index_message_locations doc, entry, old_entry
     index_message_threading doc, entry, old_entry
     index_message_labels doc, entry[:labels], (do_index_static ? [] : old_entry[:labels])
     doc.entry = entry
@@ -637,7 +673,6 @@ EOS
     doc.add_term mkterm(:date, m.date) if m.date
     doc.add_term mkterm(:type, 'mail')
     doc.add_term mkterm(:msgid, m.id)
-    doc.add_term mkterm(:source_id, m.source.id)
     m.attachments.each do |a|
       a =~ /\.(\w+)$/ or next
       doc.add_term mkterm(:attachment_extension, $1)
@@ -654,6 +689,13 @@ EOS
     doc.add_value DATE_VALUENO, date_value
   end
 
+  def index_message_locations doc, entry, old_entry
+    old_entry[:locations].map { |x| x[0] }.uniq.each { |x| doc.remove_term mkterm(:source_id, x) } if old_entry
+    entry[:locations].map { |x| x[0] }.uniq.each { |x| doc.add_term mkterm(:source_id, x) }
+    old_entry[:locations].each { |x| (doc.remove_term mkterm(:location, *x) rescue nil) } if old_entry
+    entry[:locations].each { |x| doc.add_term mkterm(:location, *x) }
+  end
+
   def index_message_labels doc, new_labels, old_labels
     return if new_labels == old_labels
     added = new_labels.to_a - old_labels.to_a
@@ -716,6 +758,8 @@ EOS
       end + args[1].to_s.downcase
     when :source_id
       PREFIX['source_id'] + args[0].to_s.downcase
+    when :location
+      PREFIX['location'] + [args[0]].pack('n') + args[1].to_s
     when :attachment_extension
       PREFIX['attachment_extension'] + args[0].to_s.downcase
     when :msgid, :ref, :thread