vmail 1.6.8 → 1.6.9

data/NOTES

@@ -0,0 +1,27 @@
+
+------------------------------------------------------------------------
+Sun Jun 26 18:07:59 EDT 2011
+
+Sqlite3 branch
+
+Use sqlite3 as a cache.
+fetch UIDS, and then check and cache.
+
+Also, log output to primary thread and let Vmail echo via Vim. Change vmail.vim
+How do make this output progressive?
+
+What if there is a queue for IMAP tasks instead of multiple untracked threads?
+
+Check for new messages can just reload the current mailbox
+  => simplify the code
+
+Build queue of stuff for Vmail to do
+ => this may lead to fewer errors than multithreaded delete (maybe too many
+ threads are spawned)
+
+TODO
+
+No number number limit inbox 100. NOTE IN DOCUMENTATION
+
+
+

data/db/create.sql

@@ -0,0 +1,25 @@
+drop table if exists version;
+create table version (
+  vmail_version text
+);
+create table if not exists messages (
+  message_id text PRIMARY KEY,
+  size integer,
+  flags text,
+  subject text,
+  sender text,
+  recipients text,
+  date text,
+  plaintext text,
+  rfc822 text
+);
+create table if not exists labelings (
+  label_id integer,
+  message_id text,
+  uid integer
+);
+create table if not exists labels (
+  label_id integer PRIMARY KEY,
+  name text UNIQUE
+);
+

data/lib/vmail.rb

@@ -58,20 +58,18 @@ module Vmail
     server.select_mailbox mailbox
 
     STDERR.puts "Mailbox: #{mailbox}"
-    STDERR.puts "Query: #{query.inspect} => #{query_string}"
+    STDERR.puts "Query: #{query.inspect}" 
+    STDERR.puts "Query String: #{String.shellescape(query_string)}"
     
     buffer_file = "vmailbuffer"
     # invoke vim
     vimscript = File.expand_path("../vmail.vim", __FILE__)
-    vim_command = "DRB_URI=#{drb_uri} VMAIL_CONTACTS_FILE=#{contacts_file} VMAIL_MAILBOX=#{String.shellescape(mailbox)} VMAIL_QUERY=#{String.shellescape(query_string)} #{vim} -S #{vimscript} #{buffer_file}"
+    vim_command = "DRB_URI=#{drb_uri} VMAIL_CONTACTS_FILE=#{contacts_file} VMAIL_MAILBOX=#{String.shellescape(mailbox)} VMAIL_QUERY=\"#{query_string}\" #{vim} -S #{vimscript} #{buffer_file}"
     STDERR.puts vim_command
     STDERR.puts "Using buffer file: #{buffer_file}"
     File.open(buffer_file, "w") do |file|
-      file.puts "Vmail starting with values:\n"
-      file.puts "- drb uri: #{drb_uri}"
-      file.puts "- mailbox: #{mailbox}"
-      file.puts "- query: #{query_string}\n"
-      file.puts "Fetching messages. please wait..."  
+      file.puts "\n\nVmail #{Vmail::VERSION}\n\n"
+      file.puts "Please wait while I fetch your messages.\n\n\n"
     end
 
     system(vim_command)
@@ -157,13 +155,15 @@ module Vmail
   end
 
   def parse_query
-    mailbox = if ARGV[0] =~ /^\d+/ 
-                "INBOX"
-              else 
-                ARGV.shift || 'INBOX' 
-              end
+    if ARGV[0] =~ /^\d+/ 
+      ARGV.shift
+    end
+    mailbox = ARGV.shift || 'INBOX' 
     query = Vmail::Query.parse(ARGV)
     [mailbox, query]
   end
 end
 
+if __FILE__ == $0
+  Vmail.start
+end

data/lib/vmail.vim

@@ -75,7 +75,7 @@ function! s:show_message(stay_in_message_list)
     call s:more_messages()
     return
   endif
-  let s:uid = matchstr(line, '\d\+$')
+  let s:uid = shellescape(matchstr(line, '\S\+$'))
   if s:uid == ""
     return
   end
@@ -219,7 +219,6 @@ func! s:close_message_window()
   endif
 endfunc
 
-
 " gets new messages since last update
 function! s:update()
   let command = s:update_command
@@ -227,13 +226,11 @@ function! s:update()
   let res = system(command)
   if len(split(res, "\n", '')) > 0
     setlocal modifiable
-    let line = line('$')
-    silent $put =res
+    call append(0, res)
     setlocal nomodifiable
     write!
     let num = len(split(res, '\n', ''))
-    call cursor(line + 1, 0)
-    normal z.
+    call cursor(0, 0)
     redraw
     echom "You have " . num . " new message" . (num == 1 ? '' : 's') . "!" 
   else
@@ -251,7 +248,7 @@ function! s:toggle_star() range
   if (match(getline(a:firstline), flag_symbol) != -1)
     let action = " -FLAGS"
   endif
-  let command = s:flag_command . join(uid_set, ',') . action . " Flagged" 
+  let command = s:flag_command . shellescape(join(uid_set, ',')) . action . " Flagged" 
   if nummsgs == 1
     echom "Toggling flag on message" 
   else
@@ -287,7 +284,7 @@ endfunction
 func! s:delete_messages(flag) range
   let uid_set = s:collect_uids(a:firstline, a:lastline)
   let nummsgs = len(uid_set)
-  let command = s:flag_command . join(uid_set, ',') . " +FLAGS " . a:flag
+  let command = s:flag_command . shellescape(join(uid_set, ',')) . " +FLAGS " . a:flag
   if nummsgs == 1
     echom "Deleting message" 
   else
@@ -305,7 +302,7 @@ endfunc
 func! s:archive_messages() range
   let uid_set = s:collect_uids(a:firstline, a:lastline)
   let nummsgs = len(uid_set)
-  let command = s:move_to_command . join(uid_set, ',') . ' ' . "all"
+  let command = s:move_to_command . shellescape(join(uid_set, ',')) . ' ' . "all"
   echo "Archiving message" . (nummsgs == 1 ? '' : 's')
   let res = system(command)
   setlocal modifiable
@@ -328,7 +325,7 @@ func! s:append_messages_to_file() range
     return
   endif
   let s:append_file = append_file
-  let command = s:append_to_file_command . join(uid_set, ',') . ' ' . s:append_file 
+  let command = s:append_to_file_command . shellescape(join(uid_set, ',')) . ' ' . s:append_file 
   echo "Appending " . nummsgs . " message" . (nummsgs == 1 ? '' : 's') . " to " . s:append_file . ". Please wait..."
   let res = system(command)
   echo res
@@ -341,7 +338,7 @@ function! s:move_to_mailbox(copy) range
   let s:copy_to_mailbox = a:copy
   let uid_set = s:collect_uids(a:firstline, a:lastline)
   let s:nummsgs = len(uid_set)
-  let s:uid_set = join(uid_set, ',')
+  let s:uid_set = shellescape(join(uid_set, ','))
   " now prompt use to select mailbox
   if !exists("s:mailboxes")
     call s:get_mailbox_list()
@@ -474,7 +471,7 @@ function! s:select_mailbox()
     return
   endif
   let s:mailbox = mailbox
-  let s:query = "100 all"
+  let s:query = "all"
   let command = s:select_mailbox_command . shellescape(s:mailbox)
   redraw
   echom "Selecting mailbox: ". s:mailbox . ". Please wait..."
@@ -483,16 +480,15 @@ function! s:select_mailbox()
   " now get latest 100 messages
   call s:focus_list_window()  
   setlocal modifiable
-  let command = s:search_command . shellescape("100 all")
+  let command = s:search_command . shellescape("all")
   echo "Loading messages..."
   let res = system(command)
   silent 1,$delete
   silent! put! =res
   execute "normal Gdd\<c-y>" 
-  normal G
   setlocal nomodifiable
   write
-  normal z.
+  normal gg
   redraw
   echom "Current mailbox: ". s:mailbox 
 endfunction
@@ -524,20 +520,19 @@ function! s:do_search()
   execute "silent normal Gdd\<c-y>" 
   setlocal nomodifiable
   write
-  normal z.
+  normal gg
 endfunction
 
 function! s:more_messages()
-  let line = getline(line('.'))
-  let seqno = get(split(matchstr(line, '\d\+:\d\+$'), ':'), 0)
-  let command = s:more_messages_command . seqno
+  let command = s:more_messages_command 
   echo "Fetching more messages. Please wait..."
   let res = system(command)
   setlocal modifiable
   let lines =  split(res, "\n")
-  call append(0, lines)
+  call append(line('$'), lines)
   " execute "normal Gdd\<c-y>" 
   setlocal nomodifiable
+  normal j
 endfunction
 
 " --------------------------------------------------------------------------------  
@@ -576,8 +571,9 @@ func! s:open_compose_window(command)
   setlocal modifiable
   " TODO maybe later save backups?
   setlocal buftype=nowrite
+  " 
   if winnr('$') > 1
-    wincmd p 
+    call s:focus_list_window()
     close!
   endif
   silent 1,$delete
@@ -736,7 +732,7 @@ function! s:collect_uids(startline, endline)
   let uid_set = []
   let lnum = a:startline
   while lnum <= a:endline
-    let uid = matchstr(getline(lnum), '\d\+$')
+    let uid = matchstr(getline(lnum), '\S\+$')
     call add(uid_set, uid)
     let lnum += 1
   endwhile
@@ -857,7 +853,7 @@ call system(s:set_window_width_command . winwidth(1))
 autocmd VimResized <buffer> call system(s:set_window_width_command . winwidth(1))
 
 autocmd bufreadpost *.txt call <SID>turn_into_compose_window()
-
+normal G
 call system(s:select_mailbox_command . shellescape(s:mailbox))
 call s:do_search()
 

data/lib/vmail/database.rb

@@ -0,0 +1,30 @@
+require 'sequel'
+
+DB = Sequel.connect 'sqlite://vmail.db'
+
+
+if !File.exists?("vmail.db")
+  create_table_script = File.expand_path("../db/create.sql", __FILE__)
+  DB.run create_table_script 
+end
+
+if DB[:version].count == 0
+  DB[:version].insert(:vmail_version => Vmail::VERSION)
+end
+
+class Vmail::Message < Sequel::Model
+  set_primary_key :message_id
+  one_to_many :labelings
+  many_to_many :labels, :join_table => 'labelings'
+end
+
+class Vmail::Label < Sequel::Model
+  set_primary_key :label_id
+  one_to_many :labelings
+  many_to_many :messages, :join_table => 'labelings'
+end
+
+class Vmail::Labeling < Sequel::Model
+end
+
+

data/lib/vmail/flagging_and_moving.rb

@@ -0,0 +1,91 @@
+module Vmail
+  module FlaggingAndMoving
+
+    def convert_to_message_ids(message_ids)
+      message_ids.split(',').map {|message_id|
+        labeling = Labeling[message_id: message_id, label_id: @label.label_id]
+        labeling.uid
+      }
+    end
+
+    # uid_set is a string comming from the vim client
+    # action is -FLAGS or +FLAGS
+    def flag(message_ids, action, flg)
+      uid_set = convert_to_message_ids(message_ids)
+      log "Flag #{uid_set} #{flg} #{action}"
+      if flg == 'Deleted'
+        log "Deleting uid_set: #{uid_set.inspect}"
+        decrement_max_seqno(uid_set.size)
+        # for delete, do in a separate thread because deletions are slow
+        spawn_thread_if_tty do 
+          unless @mailbox == mailbox_aliases['trash']
+            log "imap.uid_copy #{uid_set.inspect} to #{mailbox_aliases['trash']}"
+            log @imap.uid_copy(uid_set, mailbox_aliases['trash'])
+          end
+          log "imap.uid_store #{uid_set.inspect} #{action} [#{flg.to_sym}]"
+          log @imap.uid_store(uid_set, action, [flg.to_sym])
+          reload_mailbox
+          clear_cached_message
+        end
+      elsif flg == 'spam' || flg == mailbox_aliases['spam'] 
+        log "Marking as spam uid_set: #{uid_set.inspect}"
+        decrement_max_seqno(uid_set.size)
+        spawn_thread_if_tty do 
+          log "imap.uid_copy #{uid_set.inspect} to #{mailbox_aliases['spam']}"
+          log @imap.uid_copy(uid_set, mailbox_aliases['spam']) 
+          log "imap.uid_store #{uid_set.inspect} #{action} [:Deleted]"
+          log @imap.uid_store(uid_set, action, [:Deleted])
+          reload_mailbox
+          clear_cached_message
+        end
+      else
+        log "Flagging uid_set: #{uid_set.inspect}"
+        spawn_thread_if_tty do
+          log "imap.uid_store #{uid_set.inspect} #{action} [#{flg.to_sym}]"
+          log @imap.uid_store(uid_set, action, [flg.to_sym])
+        end
+      end
+    rescue
+      log $!
+    end
+
+    def move_to(message_ids, mailbox)
+      uid_set = convert_to_message_ids(message_ids)
+      decrement_max_seqno(uid_set.size)
+      log "Move #{uid_set.inspect} to #{mailbox}"
+      if mailbox == 'all'
+        log "Archiving messages"
+      end
+      if mailbox_aliases[mailbox]
+        mailbox = mailbox_aliases[mailbox]
+      end
+      create_if_necessary mailbox
+      log "Moving uid_set: #{uid_set.inspect} to #{mailbox}"
+      spawn_thread_if_tty do 
+        log @imap.uid_copy(uid_set, mailbox)
+        log @imap.uid_store(uid_set, '+FLAGS', [:Deleted])
+        reload_mailbox
+        clear_cached_message
+        log "Moved uid_set #{uid_set.inspect} to #{mailbox}"
+      end
+    rescue
+      log $!
+    end
+
+    def copy_to(message_ids, mailbox)
+      uid_set = convert_to_message_ids(message_ids)
+      if mailbox_aliases[mailbox]
+        mailbox = mailbox_aliases[mailbox]
+      end
+      create_if_necessary mailbox
+      log "Copying #{uid_set.inspect} to #{mailbox}"
+      spawn_thread_if_tty do 
+        log @imap.uid_copy(uid_set, mailbox)
+        log "Copied uid_set #{uid_set.inspect} to #{mailbox}"
+      end
+    rescue
+      log $!
+    end
+  end
+
+end

data/lib/vmail/imap_client.rb

@@ -6,13 +6,22 @@ require 'mail'
 require 'net/imap'
 require 'time'
 require 'logger'
+require 'vmail/helpers'
 require 'vmail/address_quoter'
+require 'vmail/database'
+require 'vmail/searching'
+require 'vmail/showing_headers'
+require 'vmail/showing_message'
+require 'vmail/flagging_and_moving'
 
 module Vmail
   class ImapClient
+    include Vmail::Helpers
     include Vmail::AddressQuoter
-
-    DIVIDER_WIDTH = 46
+    include Vmail::Searching
+    include Vmail::ShowingHeaders
+    include Vmail::ShowingMessage
+    include Vmail::FlaggingAndMoving
 
     attr_accessor :max_seqno # of current mailbox
 
@@ -26,22 +35,10 @@ module Vmail
       @logger.level = Logger::DEBUG
       @imap_server = config['server'] || 'imap.gmail.com'
       @imap_port = config['port'] || 993
-      @current_mail = nil
-      @current_message_uid = nil
-      @width = 140
-    end
-
-    # holds mail objects keyed by [mailbox, uid]
-    def message_cache
-      @message_cache ||= {}
-      size = @message_cache.values.reduce(0) {|sum, x| sum + x[:size]}
-      if size > 2_000_000 # TODO make this configurable
-        log "Pruning message cache; message cache is consuming #{number_to_human_size size}"
-        @message_cache.keys[0, @message_cache.size / 2].each {|k| @message_cache.delete(k)}
-      end
-      @message_cache
+      current_message = nil
     end
 
+
     def open
       @imap = Net::IMAP.new(@imap_server, @imap_port, true, nil, false)
       log @imap.login(@username, @password)
@@ -69,19 +66,23 @@ module Vmail
       if mailbox_aliases[mailbox]
         mailbox = mailbox_aliases[mailbox]
       end
-      if mailbox == @mailbox && !force
-        return
-      end
       log "Selecting mailbox #{mailbox.inspect}"
       reconnect_if_necessary(15) do 
         log @imap.select(mailbox)
       end
       log "Done"
+
       @mailbox = mailbox
+      @label = Label[name: @mailbox] || Label.create(name: @mailbox)
+
       log "Getting mailbox status"
       get_mailbox_status
       log "Getting highest message id"
       get_highest_message_id
+      if @next_window_width
+        @width = @next_window_width
+      end
+
       return "OK"
     end
 
@@ -90,12 +91,12 @@ module Vmail
       select_mailbox(@mailbox, true)
     end
 
+    # TODO no need for this if all shown messages are stored in SQLITE3 
+    # and keyed by UID.
     def clear_cached_message
       return unless STDIN.tty?
-      log "CLEARING CACHED MESSAGE"
-      @current_mail = nil
-      @current_message_uid = nil
-      @current_message = nil
+      log "Clearing cached message"
+      current_message = nil
     end
 
     def get_highest_message_id
@@ -103,7 +104,7 @@ module Vmail
       res = @imap.fetch([1,"*"], ["ENVELOPE"])
       if res 
         @num_messages = res[-1].seqno
-        log "HIGHEST ID: #@num_messages"
+        log "Highest seqno: #@num_messages"
       else
         @num_messages = 1
         log "NO HIGHEST ID: setting @num_messages to 1"
@@ -180,178 +181,18 @@ module Vmail
       @mailboxes
     end
 
-    # id_set may be a range, array, or string
-    def fetch_row_text(id_set, are_uids=false, is_update=false)
-      log "Fetch_row_text: #{id_set.inspect}"
-      if id_set.is_a?(String)
-        id_set = id_set.split(',')
-      end
-      if id_set.to_a.empty?
-        log "- empty set"
-        return ""
-      end
-      new_message_rows = fetch_envelopes(id_set, are_uids, is_update)
-      new_message_rows.map {|x| x[:row_text]}.join("\n")
-    rescue # Encoding::CompatibilityError (only in 1.9.2)
-      log "Error in fetch_row_text:\n#{$!}\n#{$!.backtrace}"
-      new_message_rows.map {|x| Iconv.conv('US-ASCII//TRANSLIT//IGNORE', 'UTF-8', x[:row_text])}.join("\n")
-    end
-
-    def fetch_envelopes(id_set, are_uids, is_update)
-      results = reconnect_if_necessary do 
-        if are_uids
-          @imap.uid_fetch(id_set, ["FLAGS", "ENVELOPE", "RFC822.SIZE", "UID" ])
-        else
-          @imap.fetch(id_set, ["FLAGS", "ENVELOPE", "RFC822.SIZE", "UID" ])
-        end
-      end
-      if results.nil?
-        error = "Expected fetch results but got nil"
-        log(error) && raise(error)
-      end
-      log "- extracting headers"
-      new_message_rows = results.map {|x| extract_row_data(x) }
-      log "- returning #{new_message_rows.size} new rows and caching result"  
-      new_message_rows
-    end
-
-    # TODO extract this to another class or module and write unit tests
-    def extract_row_data(fetch_data)
-      seqno = fetch_data.seqno
-      uid = fetch_data.attr['UID']
-      # log "fetched seqno #{seqno} uid #{uid}"
-      envelope = fetch_data.attr["ENVELOPE"]
-      size = fetch_data.attr["RFC822.SIZE"]
-      flags = fetch_data.attr["FLAGS"]
-      address_struct = if @mailbox == mailbox_aliases['sent'] 
-                         structs = envelope.to || envelope.cc
-                         structs.nil? ? nil : structs.first 
-                       else
-                         envelope.from.first
-                       end
-      address = if address_struct.nil?
-                  "Unknown"
-                elsif address_struct.name
-                  "#{Mail::Encodings.unquote_and_convert_to(address_struct.name, 'UTF-8')} <#{[address_struct.mailbox, address_struct.host].join('@')}>"
-                else
-                  [Mail::Encodings.unquote_and_convert_to(address_struct.mailbox, 'UTF-8'), Mail::Encodings.unquote_and_convert_to(address_struct.host, 'UTF-8')].join('@') 
-                end
-      if @mailbox == mailbox_aliases['sent'] && envelope.to && envelope.cc
-        total_recips = (envelope.to + envelope.cc).size
-        address += " + #{total_recips - 1}"
-      end
-      date = begin 
-               Time.parse(envelope.date).localtime
-             rescue ArgumentError
-               Time.now
-             end
-
-      date_formatted = if date.year != Time.now.year
-                         date.strftime "%b %d %Y" rescue envelope.date.to_s 
-                       else 
-                         date.strftime "%b %d %I:%M%P" rescue envelope.date.to_s 
-                       end
-      subject = envelope.subject || ''
-      subject = Mail::Encodings.unquote_and_convert_to(subject, 'UTF-8')
-      flags = format_flags(flags)
-      mid_width = @width - 38
-      address_col_width = (mid_width * 0.3).ceil
-      subject_col_width = (mid_width * 0.7).floor
-      identifier = [seqno.to_i, uid.to_i].join(':')
-      row_text = [ flags.col(2),
-                   (date_formatted || '').col(14),
-                   address.col(address_col_width),
-                   subject.col(subject_col_width), 
-                   number_to_human_size(size).rcol(7), 
-                   identifier.to_s
-      ].join(' | ')
-      {:uid => uid, :seqno => seqno, :row_text => row_text}
-    rescue 
-      log "Error extracting header for uid #{uid} seqno #{seqno}: #$!\n#{$!.backtrace}"
-      row_text = "#{seqno.to_s} : error extracting this header"
-      {:uid => uid, :seqno => seqno, :row_text => row_text}
-    end
-
-    UNITS = [:b, :kb, :mb, :gb].freeze
-
-    # borrowed from ActionView/Helpers
-    def number_to_human_size(number)
-      if number.to_i < 1024
-        "<1kb" # round up to 1kh
-      else
-        max_exp = UNITS.size - 1
-        exponent = (Math.log(number) / Math.log(1024)).to_i # Convert to base 1024
-        exponent = max_exp if exponent > max_exp # we need this to avoid overflow for the highest unit
-        number  /= 1024 ** exponent
-        unit = UNITS[exponent]
-        "#{number}#{unit}"
-      end
-    end
-
-    FLAGMAP = {:Flagged => '*'}
-    # flags is an array like [:Flagged, :Seen]
-    def format_flags(flags)
-      # other flags like "Old" should be hidden here
-      flags = flags.map {|flag| FLAGMAP[flag] || flag}
-      flags.delete("Old")
-      if flags.delete(:Seen).nil?
-        flags << '+' # unread
-      end
-      flags.join('')
-    end
-
-    def search(query)
-      query = Vmail::Query.parse(query)
-      @limit = query.shift.to_i
-      # a limit of zero is effectively no limit
-      if @limit == 0
-        @limit = @num_messages
-      end
-      if query.size == 1 && query[0].downcase == 'all'
-        # form a sequence range
-        query.unshift [[@num_messages - @limit + 1 , 1].max, @num_messages].join(':')
-        @all_search = true
-      else # this is a special query search
-        # set the target range to the whole set
-        query.unshift "1:#@num_messages"
-        @all_search = false
-      end
-      @query = query.map {|x| x.to_s.downcase}
-      query_string = Vmail::Query.args2string(@query)
-      log "Search query: #{@query} > #{query_string.inspect}"
-      log "- @all_search #{@all_search}"
-      @query = query
-      @ids = reconnect_if_necessary(180) do # increase timeout to 3 minutes
-        @imap.search(query_string)
-      end
-      # save ids in @ids, because filtered search relies on it
-      fetch_ids = if @all_search
-                    @ids
-                  else #filtered search
-                    @start_index = [@ids.length - @limit, 0].max
-                    @ids[@start_index..-1]
-                  end
-      self.max_seqno = @ids[-1]
-      log "- search query got #{@ids.size} results; max seqno: #{self.max_seqno}" 
-      clear_cached_message
-      res = fetch_row_text(fetch_ids)
-      if STDOUT.tty?
-        add_more_message_line(res, fetch_ids[0])
-      else
-        # non interactive mode
-        puts [@mailbox, res].join("\n")
-      end
-    rescue
-      log "ERROR:\n#{$!.inspect}\n#{$!.backtrace.join("\n")}"
-    end
-
     def decrement_max_seqno(num)
       return unless STDIN.tty?
       log "Decremented max seqno from #{self.max_seqno} to #{self.max_seqno - num}"
       self.max_seqno -= num
     end
 
+    # TODO why not just reload the current page?
     def update
+      if search_query?
+        log "Update aborted because query is search query: #{@query.inspect}"
+        return ""
+      end
       prime_connection
       old_num_messages = @num_messages
       # we need to re-select the mailbox to get the new highest id
@@ -371,7 +212,7 @@ module Vmail
       log "- update: new uids: #{new_ids.inspect}"
       if !new_ids.empty?
         self.max_seqno = new_ids[-1]
-        res = fetch_row_text(new_ids, false, true)
+        res = get_message_headers(new_ids.reverse, false, true)
         res
       else
         ''
@@ -379,202 +220,15 @@ module Vmail
     end
 
     # gets 100 messages prior to id
-    def more_messages(message_id, limit=100)
-      log "More_messages: message_id #{message_id}"
-      message_id = message_id.to_i
-      if @all_search 
-        x = [(message_id - limit), 0].max
-        y = [message_id - 1, 0].max
-
-        res = fetch_row_text((x..y))
-        add_more_message_line(res, x)
-      else # filter search query
-        log "@start_index #@start_index"
-        x = [(@start_index - limit), 0].max
-        y = [@start_index - 1, 0].max
-        @start_index = x
-        res = fetch_row_text(@ids[x..y]) 
-        add_more_message_line(res, @ids[x])
-      end
-    end
-
-    def add_more_message_line(res, start_seqno)
-      log "Add_more_message_line for start_seqno #{start_seqno}"
-      if @all_search
-        return res if start_seqno.nil?
-        remaining = start_seqno - 1
-      else # filter search
-        remaining = (@ids.index(start_seqno) || 1) - 1
-      end
-      if remaining < 1
-        log "None remaining"
-        return "Showing all matches\n" + res
-      end
-      log "Remaining messages: #{remaining}"
-      ">  Load #{[100, remaining].min} more messages. #{remaining} remaining.\n" + res
-    end
-
-    def show_message(uid, raw=false)
-      log "Show message: #{uid}"
-      return @current_mail.to_s if raw 
-      uid = uid.to_i
-      if uid == @current_message_uid 
-        return @current_message 
-      end
-
-      #prefetch_adjacent(index) # deprecated
-
-      # TODO keep state in vim buffers, instead of on Vmail Ruby client
-      # envelope_data[:row_text] = envelope_data[:row_text].gsub(/^\+ /, '  ').gsub(/^\*\+/, '* ') # mark as read in cache
-      #seqno = envelope_data[:seqno]
-
-      log "Showing message uid: #{uid}"
-      data = if x = message_cache[[@mailbox, uid]]
-               log "- message cache hit"
-               x
-             else 
-               log "- fetching and storing to message_cache[[#{@mailbox}, #{uid}]]"
-               fetch_and_cache(uid)
-             end
-      if data.nil?
-        # retry, though this is a hack!
-        log "- data is nil. retrying..."
-        return show_message(uid, raw)
-      end
-      # make this more DRY later by directly using a ref to the hash
-      mail = data[:mail]
-      size = data[:size] 
-      @current_message_uid = uid
-      log "- setting @current_mail"
-      @current_mail = mail # used later to show raw message or extract attachments if any
-      @current_message = data[:message_text]
-    rescue
-      log "Parsing error"
-      "Error encountered parsing this message:\n#{$!}\n#{$!.backtrace.join("\n")}"
-    end
-
-    def fetch_and_cache(uid)
-      if data = message_cache[[@mailbox, uid]] 
-        return data
-      end
-      fetch_data = reconnect_if_necessary do 
-        res = @imap.uid_fetch(uid, ["FLAGS", "RFC822", "RFC822.SIZE"])
-        if res.nil?
-          # retry one more time ( find a more elegant way to do this )
-          res = @imap.uid_fetch(uid, ["FLAGS", "RFC822", "RFC822.SIZE"])
-        end
-        res[0] 
-      end
-      # USE THIS
-      size = fetch_data.attr["RFC822.SIZE"]
-      flags = fetch_data.attr["FLAGS"]
-      mail = Mail.new(fetch_data.attr['RFC822'])
-      formatter = Vmail::MessageFormatter.new(mail)
-      message_text = <<-EOF
-#{@mailbox} uid:#{uid} #{number_to_human_size size} #{flags.inspect} #{format_parts_info(formatter.list_parts)}
-#{divider '-'}
-#{format_headers(formatter.extract_headers)}
-
-#{formatter.process_body}
-EOF
-      # log "Storing message_cache[[#{@mailbox}, #{uid}]]"
-      d = {:mail => mail, :size => size, :message_text => message_text, :seqno => fetch_data.seqno, :flags => flags}
-      message_cache[[@mailbox, uid]] = d
-    rescue
-      msg = "Error encountered parsing message uid  #{uid}:\n#{$!}\n#{$!.backtrace.join("\n")}" + 
-        "\n\nRaw message:\n\n" + mail.to_s
-      log msg
-      log message_text
-      {:message_text => msg}
-    end
-
-    # deprecated
-    def prefetch_adjacent(index)
-      Thread.new do 
-        [index + 1, index - 1].each do |idx|
-          fetch_and_cache(idx)
-        end
-      end
-    end
-
-    def format_parts_info(parts)
-      lines = parts.select {|part| part !~ %r{text/plain}}
-      if lines.size > 0
-        "\n#{lines.join("\n")}"
-      end
-    end
-
-    # id_set is a string comming from the vim client
-    # action is -FLAGS or +FLAGS
-    def flag(uid_set, action, flg)
-      log "Flag #{uid_set} #{flg} #{action}"
-      uid_set = uid_set.split(',').map(&:to_i)
-      if flg == 'Deleted'
-        log "Deleting uid_set: #{uid_set.inspect}"
-        decrement_max_seqno(uid_set.size)
-        # for delete, do in a separate thread because deletions are slow
-        spawn_thread_if_tty do 
-          unless @mailbox == mailbox_aliases['trash']
-            log "@imap.uid_copy #{uid_set.inspect} to #{mailbox_aliases['trash']}"
-            log @imap.uid_copy(uid_set, mailbox_aliases['trash'])
-          end
-          log "@imap.uid_store #{uid_set.inspect} #{action} [#{flg.to_sym}]"
-          log @imap.uid_store(uid_set, action, [flg.to_sym])
-          reload_mailbox
-          clear_cached_message
-        end
-      elsif flg == 'spam' || flg == mailbox_aliases['spam'] 
-        log "Marking as spam uid_set: #{uid_set.inspect}"
-        decrement_max_seqno(uid_set.size)
-        spawn_thread_if_tty do 
-          log "@imap.uid_copy #{uid_set.inspect} to #{mailbox_aliases['spam']}"
-          log @imap.uid_copy(uid_set, mailbox_aliases['spam']) 
-          log "@imap.uid_store #{uid_set.inspect} #{action} [:Deleted]"
-          log @imap.uid_store(uid_set, action, [:Deleted])
-          reload_mailbox
-          clear_cached_message
-        end
-      else
-        log "Flagging uid_set: #{uid_set.inspect}"
-        spawn_thread_if_tty do
-          log "@imap.uid_store #{uid_set.inspect} #{action} [#{flg.to_sym}]"
-          log @imap.uid_store(uid_set, action, [flg.to_sym])
-        end
-      end
-    end
-
-    def move_to(uid_set, mailbox)
-      uid_set = uid_set.split(',').map(&:to_i)
-      decrement_max_seqno(uid_set.size)
-      log "Move #{uid_set.inspect} to #{mailbox}"
-      if mailbox == 'all'
-        log "Archiving messages"
-      end
-      if mailbox_aliases[mailbox]
-        mailbox = mailbox_aliases[mailbox]
-      end
-      create_if_necessary mailbox
-      log "Moving uid_set: #{uid_set.inspect} to #{mailbox}"
-      spawn_thread_if_tty do 
-        log @imap.uid_copy(uid_set, mailbox)
-        log @imap.uid_store(uid_set, '+FLAGS', [:Deleted])
-        reload_mailbox
-        clear_cached_message
-        log "Moved uid_set #{uid_set.inspect} to #{mailbox}"
-      end
-    end
-
-    def copy_to(uid_set, mailbox)
-      uid_set = uid_set.split(',').map(&:to_i)
-      if mailbox_aliases[mailbox]
-        mailbox = mailbox_aliases[mailbox]
-      end
-      create_if_necessary mailbox
-      log "Copying #{uid_set.inspect} to #{mailbox}"
-      spawn_thread_if_tty do 
-        log @imap.uid_copy(uid_set, mailbox)
-        log "Copied uid_set #{uid_set.inspect} to #{mailbox}"
-      end
+    def more_messages
+      log "Getting more_messages"
+      x = [(@start_index - @limit), 0].max
+      y = [@start_index - 1, 0].max
+      @start_index = x
+      fetch_ids = search_query? ? @ids[x..y] : (x..y).to_a
+      message_ids = fetch_and_cache_headers(fetch_ids)
+      res = get_message_headers message_ids
+      with_more_message_line(res)
     end
 
     def spawn_thread_if_tty(&block) 
@@ -632,11 +286,7 @@ EOF
 
     def reply_template(replyall=false)
       log "Sending reply template"
-      if @current_mail.nil?
-        log "- missing @current mail!"
-        return nil
-      end
-      reply_headers = Vmail::ReplyTemplate.new(@current_mail, @username, @name, replyall, @always_cc).reply_headers
+      reply_headers = Vmail::ReplyTemplate.new(current_message.rfc822, @username, @name, replyall, @always_cc).reply_headers
       body = reply_headers.delete(:body)
       format_headers(reply_headers) + "\n\n\n" + body + signature
     end
@@ -647,8 +297,8 @@ EOF
     end
 
     def forward_template
-      original_body = @current_message.split(/\n-{20,}\n/, 2)[1]
-      formatter = Vmail::MessageFormatter.new(@current_mail)
+      original_body = current_message.split(/\n-{20,}\n/, 2)[1]
+      formatter = Vmail::MessageFormatter.new(current_mail)
       headers = formatter.extract_headers
       subject = headers['subject']
       if subject !~ /Fwd: /
@@ -660,12 +310,6 @@ EOF
         original_body + signature
     end
 
-    def divider(str)
-      str * DIVIDER_WIDTH
-    end
-
-    SENT_MESSAGES_FILE = "sent-messages.txt"
-
     def format_sent_message(mail)
       formatter = Vmail::MessageFormatter.new(mail)
       message_text = <<-EOF
@@ -673,14 +317,10 @@ Sent Message #{self.format_parts_info(formatter.list_parts)}
 
 #{format_headers(formatter.extract_headers)}
 
-#{formatter.process_body}
+#{formatter.plaintext_part}
 EOF
     end
 
-    def backup_sent_message(message)
-      File.open(SENT_MESSAGES_FILE, "a") {|f| f.write(divider('-') + "\n" + format_sent_message(message))}
-    end
-
     def deliver(text)
       # parse the text. The headers are yaml. The rest is text body.
       require 'net/smtp'
@@ -691,8 +331,7 @@ EOF
       log res.inspect
       log "\n"
       msg = if res.is_a?(Mail::Message)
-        backup_sent_message mail
-        "Message '#{mail.subject}' sent and saved to #{SENT_MESSAGES_FILE}"
+        "Message '#{mail.subject}' sent"
       else
         "Failed to deliver message '#{mail.subject}'!"
       end
@@ -746,11 +385,11 @@ EOF
 
     def save_attachments(dir)
       log "Save_attachments #{dir}"
-      if !@current_mail
+      if !current_mail
         log "Missing a current message"
       end
-      return unless dir && @current_mail
-      attachments = @current_mail.attachments
+      return unless dir && current_mail
+      attachments = current_mail.attachments
       `mkdir -p #{dir}`
       saved = attachments.map do |x|
         path = File.join(dir, x.filename)
@@ -763,14 +402,14 @@ EOF
 
     def open_html_part
       log "Open_html_part"
-      log @current_mail.parts.inspect
-      multipart = @current_mail.parts.detect {|part| part.multipart?}
+      log current_mail.parts.inspect
+      multipart = current_mail.parts.detect {|part| part.multipart?}
       html_part = if multipart 
                     multipart.parts.detect {|part| part.header["Content-Type"].to_s =~ /text\/html/}
-                  elsif ! @current_mail.parts.empty?
-                    @current_mail.parts.detect {|part| part.header["Content-Type"].to_s =~ /text\/html/}
+                  elsif ! current_mail.parts.empty?
+                    current_mail.parts.detect {|part| part.header["Content-Type"].to_s =~ /text\/html/}
                   else
-                    @current_mail.body
+                    current_mail.body
                   end
       return if html_part.nil?
       outfile = 'part.html'
@@ -780,8 +419,11 @@ EOF
     end
 
     def window_width=(width)
-      log "Setting window width to #{width}"
-      @width = width.to_i
+      @next_window_width = width.to_i
+      if @width.nil?
+        @width = @next_window_width
+      end
+      log "Setting next window width to #{width}"
     end
    
     def smtp_settings