sup 0.0.1

data/lib/sup/colormap.rb

@@ -0,0 +1,118 @@
+require "curses"
+
+module Redwood
+
+class Colormap
+  @@instance = nil
+
+  CURSES_COLORS = [Curses::COLOR_BLACK, Curses::COLOR_RED, Curses::COLOR_GREEN,
+                   Curses::COLOR_YELLOW, Curses::COLOR_BLUE,
+                   Curses::COLOR_MAGENTA, Curses::COLOR_CYAN,
+                   Curses::COLOR_WHITE]
+  NUM_COLORS = 15
+  
+  def initialize
+    raise "only one instance can be created" if @@instance
+    @@instance = self
+    @entries = {}
+    @color_pairs = {[Curses::COLOR_WHITE, Curses::COLOR_BLACK] => 0}
+    @users = []
+    @next_id = 0
+    yield self if block_given?
+    @entries[highlight_sym(:none)] = highlight_for(Curses::COLOR_WHITE,
+                                                   Curses::COLOR_BLACK,
+                                                   []) + [nil]
+  end
+
+  def add sym, fg, bg, *attrs
+    raise ArgumentError, "color for #{sym} already defined" if
+      @entries.member? sym
+    raise ArgumentError, "color '#{fg}' unknown" unless CURSES_COLORS.include? fg
+    raise ArgumentError, "color '#{bg}' unknown" unless CURSES_COLORS.include? bg
+
+    @entries[sym] = [fg, bg, attrs, nil]
+    @entries[highlight_sym(sym)] = highlight_for(fg, bg, attrs) + [nil]
+  end
+
+  def highlight_sym sym
+    "#{sym}_highlight".intern
+  end
+
+  def highlight_for fg, bg, attrs
+    hfg =
+      case fg
+      when Curses::COLOR_BLUE
+        Curses::COLOR_WHITE
+      when Curses::COLOR_YELLOW, Curses::COLOR_GREEN
+        fg
+      else
+        Curses::COLOR_BLACK
+      end
+
+    hbg = 
+      case bg
+      when Curses::COLOR_CYAN
+        Curses::COLOR_YELLOW
+      else
+        Curses::COLOR_CYAN
+      end
+
+    attrs =
+      if fg == Curses::COLOR_WHITE && attrs.include?(Curses::A_BOLD)
+        [Curses::A_BOLD]
+      else
+        case hfg
+        when Curses::COLOR_BLACK
+          []
+        else
+          [Curses::A_BOLD]
+        end
+      end
+    [hfg, hbg, attrs]
+  end
+
+  def color_for sym, highlight=false
+    sym = highlight_sym(sym) if highlight
+    return Curses::COLOR_BLACK if sym == :none
+    raise ArgumentError, "undefined color #{sym}" unless @entries.member? sym
+
+    ## if this color is cached, return it
+    fg, bg, attrs, color = @entries[sym]
+    return color if color
+
+    if(cp = @color_pairs[[fg, bg]])
+      ## nothing
+    else ## need to get a new colorpair
+      @next_id = (@next_id + 1) % NUM_COLORS
+      @next_id += 1 if @next_id == 0 # 0 is always white on black
+      id = @next_id
+      Redwood::log "colormap: for color #{sym}, using id #{id} -> #{fg}, #{bg}"
+      Curses.init_pair id, fg, bg or raise ArgumentError,
+        "couldn't initialize curses color pair #{fg}, #{bg} (key #{id})"
+
+      cp = @color_pairs[[fg, bg]] = Curses.color_pair(id)
+      ## delete the old mapping, if it exists
+      if @users[cp]
+        @users[cp].each do |usym|
+          Redwood::log "dropping color #{usym} (#{id})"
+          @entries[usym][3] = nil
+        end
+        @users[cp] = []
+      end
+    end
+
+    ## by now we have a color pair
+    color = attrs.inject(cp) { |color, attr| color | attr }
+    @entries[sym][3] = color # fill the cache
+    (@users[cp] ||= []) << sym # record entry as a user of that color pair
+    color
+  end
+
+  def self.instance; @@instance; end
+  def self.method_missing meth, *a
+    Colorcolors.new unless @@instance
+    @@instance.send meth, *a
+  end
+end
+
+end

data/lib/sup/contact.rb

@@ -0,0 +1,40 @@
+module Redwood
+
+class ContactManager
+  include Singleton
+
+  def initialize fn
+    @fn = fn
+    @people = {}
+
+    if File.exists? fn
+      IO.foreach(fn) do |l|
+        l =~ /^(\S+): (.*)$/ or raise "can't parse #{fn} line #{l.inspect}"
+        aalias, addr = $1, $2
+        @people[aalias] = Person.for addr
+      end
+    end
+
+    self.class.i_am_the_instance self
+  end
+
+  def contacts; @people; end
+  def set_contact person, aalias
+    oldentry = @people.find { |a, p| p == person }
+    @people.delete oldentry.first if oldentry
+    @people[aalias] = person
+  end
+  def drop_contact person; @people.delete person; end
+  def delete t; @people.delete t; end
+  def resolve aalias; @people[aalias]; end
+
+  def save
+    File.open(@fn, "w") do |f|
+      @people.keys.sort.each do |aalias|
+        f.puts "#{aalias}: #{@people[aalias].full_address}"
+      end
+    end
+  end
+end
+
+end

data/lib/sup/draft.rb

@@ -0,0 +1,105 @@
+module Redwood
+
+class DraftManager
+  include Singleton
+
+  attr_accessor :source
+  def initialize dir
+    @dir = dir
+    @source = nil
+    self.class.i_am_the_instance self
+  end
+
+  def self.source_name; "drafts"; end
+  def self.source_id; 9999; end
+  def new_source; @source = DraftLoader.new @dir; end
+
+  def write_draft
+    offset = @source.gen_offset
+    fn = @source.fn_for_offset offset
+    File.open(fn, "w") { |f| yield f }
+
+    @source.each do |offset, labels|
+      m = Message.new @source, offset, labels
+      Index.add_message m
+      UpdateManager.relay :add, m
+    end
+  end
+
+  def discard mid
+    docid, entry = Index.load_entry_for_id mid
+    raise ArgumentError, "can't find entry for draft: #{mid.inspect}" unless entry
+    raise ArgumentError, "not a draft: source id #{entry[:source_id].inspect}, should be #{DraftManager.source_id.inspect} for #{mid.inspect} / docno #{docid}" unless entry[:source_id].to_i == DraftManager.source_id
+    Index.drop_entry docid
+    File.delete @source.fn_for_offset(entry[:source_info])
+    UpdateManager.relay :delete, mid
+  end
+end
+
+class DraftLoader
+  attr_accessor :dir, :end_offset
+  bool_reader :dirty
+
+  def initialize dir, end_offset=0
+    Dir.mkdir dir unless File.exists? dir
+    @dir = dir
+    @end_offset = end_offset
+    @dirty = false
+  end
+
+  def done?; !File.exists? fn_for_offset(@end_offset); end
+  def usual?; true; end
+  def id; DraftManager.source_id; end
+  def to_s; DraftManager.source_name; end
+  def is_source_for? x; x == DraftManager.source_name; end
+
+  def gen_offset
+    i = @end_offset
+    while File.exists? fn_for_offset(i)
+      i += 1
+    end
+    i
+  end
+
+  def fn_for_offset o; File.join(@dir, o.to_s); end
+
+  def load_header offset
+    File.open fn_for_offset(offset) do |f|
+      return MBox::read_header(f)
+    end
+  end
+  
+  def load_message offset
+    File.open fn_for_offset(offset) do |f|
+      RMail::Mailbox::MBoxReader.new(f).each_message do |input|
+        return RMail::Parser.read(input)
+      end
+    end
+  end
+
+  ## load the full header text
+  def load_header_text offset
+    ret = ""
+    File.open fn_for_offset(offset) do |f|
+      until f.eof? || (l = f.gets) =~ /^$/
+        ret += l
+      end
+    end
+    ret
+  end
+
+  def each
+    while File.exists?(fn = File.join(@dir, @end_offset.to_s))
+      yield @end_offset, [:draft, :inbox]
+      @end_offset += 1
+      @dirty = true
+    end
+  end
+
+  def total; Dir[File.join(@dir, "*")].sort.last.to_i; end
+  def reset!; @end_offset = 0; @dirty = true; end
+end
+
+Redwood::register_yaml(DraftLoader, %w(dir end_offset))
+
+end

data/lib/sup/index.rb

@@ -0,0 +1,353 @@
+## the index structure for redwood. interacts with ferret.
+
+require 'thread'
+require 'fileutils'
+require_gem 'ferret', ">= 0.10.13"
+
+module Redwood
+
+class IndexError < StandardError
+  attr_reader :source
+
+  def initialize source, s
+    super s
+    @source = source
+  end
+end
+
+class Index
+  include Singleton
+
+  LOAD_THREAD_PETIT_DELAY = 0.1
+  LOAD_THREAD_GRAND_DELAY = 5
+
+  MESSAGES_AT_A_TIME = 10
+
+  attr_reader :index # debugging only
+  
+  def initialize dir=BASE_DIR
+    @dir = dir
+    @mutex = Mutex.new
+    @load_thread = nil # loads new messages
+    @sources = {}
+    @sources_dirty = false
+
+    self.class.i_am_the_instance self
+  end
+
+  def load
+    load_sources
+    load_index
+  end
+
+  def save
+    FileUtils.mkdir_p @dir unless File.exists? @dir
+    save_sources
+    save_index
+  end
+
+  def add_source source
+    raise "duplicate source!" if @sources.include? source
+    @sources_dirty = true
+    source.id ||= @sources.size
+    source.id += 1 while @sources.member? source.id
+    @sources[source.id] = source
+  end
+
+  def source_for name; @sources.values.find { |s| s.is_source_for? name }; end
+  def usual_sources; @sources.values.find_all { |s| s.usual? }; end
+
+  def load_index dir=File.join(@dir, "ferret")
+    wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false
+    sa = Ferret::Analysis::StandardAnalyzer.new
+    analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa
+    analyzer[:body] = sa
+
+    if File.exists? dir
+      Redwood::log "loading index"
+      @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer)
+    else
+      Redwood::log "creating index"
+      field_infos = Ferret::Index::FieldInfos.new :store => :yes
+      field_infos.add_field :message_id
+      field_infos.add_field :source_id
+      field_infos.add_field :source_info, :index => :no, :term_vector => :no
+      field_infos.add_field :date, :index => :untokenized
+      field_infos.add_field :body, :store => :no
+      field_infos.add_field :label
+      field_infos.add_field :subject
+      field_infos.add_field :from
+      field_infos.add_field :to
+      field_infos.add_field :refs
+      field_infos.add_field :snippet, :index => :no, :term_vector => :no
+      field_infos.create_index dir
+      @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer)
+    end
+  end
+
+  ## update the message by deleting and re-adding
+  def update_message m, source=nil, source_info=nil
+    docid, entry = load_entry_for_id m.id
+    if entry
+      source ||= entry[:source_id].to_i
+      source_info ||= entry[:source_info].to_i
+    end
+    raise "no entry and no source info for message #{m.id}" unless source && source_info
+
+    raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+    @index.delete docid
+    add_message m
+  end
+
+  def save_index fn=File.join(@dir, "ferret")
+    # don't have to do anything apparently
+  end
+
+  def contains_id? id
+    @index.search(Ferret::Search::TermQuery.new(:message_id, id)).total_hits > 0
+  end
+  def contains? m; contains_id? m.id; end
+  def size; @index.size; end
+
+  ## you should probably not call this on a block that doesn't break
+  ## rather quickly because the results will probably be, as we say
+  ## in scotland, frikkin' huuuge.
+  EACH_BY_DATE_NUM = 100
+  def each_id_by_date opts={}
+    return if @index.size == 0 # otherwise ferret barfs
+    query = build_query opts
+    offset = 0
+    while true
+      results = @index.search(query, :sort => "date DESC", :limit => EACH_BY_DATE_NUM, :offset => offset)
+      Redwood::log "got #{results.total_hits} results for query (offset #{offset}) #{query.inspect}"
+      results.hits.each { |hit| yield @index[hit.doc][:message_id], lambda { build_message hit.doc } }
+      break if offset >= results.total_hits - EACH_BY_DATE_NUM
+      offset += EACH_BY_DATE_NUM
+    end
+  end
+
+  def num_results_for opts={}
+    query = build_query opts
+    x = @index.search(query).total_hits
+    Redwood::log "num_results_for: have #{x} for query #{query}"
+    x
+  end
+
+  SAME_SUBJECT_DATE_LIMIT = 7
+  def each_message_in_thread_for m, opts={}
+    messages = {}
+    searched = {}
+    num_queries = 0
+
+    ## temporarily disabling subject searching because it's a
+    ## significant slowdown.
+    ##
+    ## TODO: make this configurable, i guess
+    if false
+      date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
+      date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
+
+      q = Ferret::Search::BooleanQuery.new true
+      sq = Ferret::Search::PhraseQuery.new(:subject)
+      wrap_subj(Message.normalize_subj(m.subj)).split(/\s+/).each do |t|
+        sq.add_term t
+      end
+      q.add_query sq, :must
+      q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
+
+      pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
+      Redwood::log "found #{pending.size} results for subject query #{q}"
+    else
+      pending = [m.id]
+    end
+
+    until pending.empty? || (opts[:limit] && messages.size >= opts[:limit])
+      id = pending.pop
+      next if searched.member? id
+      searched[id] = true
+      q = Ferret::Search::BooleanQuery.new true
+      q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should
+      q.add_query Ferret::Search::TermQuery.new(:refs, id), :should
+
+      num_queries += 1
+      @index.search_each(q, :limit => :all) do |docid, score|
+        break if opts[:limit] && messages.size >= opts[:limit]
+        mid = @index[docid][:message_id]
+        unless messages.member? mid
+          messages[mid] ||= lambda { build_message docid }
+          refs = @index[docid][:refs].split(" ")
+          pending += refs
+        end
+      end
+    end
+    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}"
+    messages.each { |mid, builder| yield mid, builder }
+  end
+
+  ## builds a message object from a ferret result
+  def build_message docid
+    doc = @index[docid]
+    source = @sources[doc[:source_id].to_i]
+    #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})"
+    raise "invalid source #{doc[:source_id]}" unless source
+    begin
+      raise "no snippet" unless doc[:snippet]
+      Message.new source, doc[:source_info].to_i, 
+                  doc[:label].split(" ").map { |s| s.intern },
+                  doc[:snippet]
+    rescue MessageFormatError => e
+      raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}")
+      nil
+    end
+  end
+
+  def start_load_thread
+    return if @load_thread
+    @load_thread = true
+    @load_thread = ::Thread.new do
+      while @load_thread
+        load_some_entries ENTRIES_AT_A_TIME, LOAD_THREAD_PETIT_DELAY, LOAD_THREAD_GRAND_DELAY
+      end
+    end
+  end
+
+  def end_load_thread; @load_thread = nil; end
+  def fresh_thread_id; @next_thread_id += 1; end
+
+  def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
+
+  def add_message m
+    return false if contains? m
+
+    source_id = 
+      if m.source.is_a? Integer
+        m.source
+      else
+        m.source.id or raise "unregistered source #{m.source}"
+      end
+
+    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
+    d = {
+      :message_id => m.id,
+      :source_id => source_id,
+      :source_info => m.source_info,
+      :date => m.date.to_indexable_s,
+      :body => m.content,
+      :snippet => m.snippet,
+      :label => m.labels.join(" "),
+      :from => m.from ? m.from.email : "",
+      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
+      :subject => wrap_subj(Message.normalize_subj(m.subj)),
+      :refs => (m.refs + m.replytos).join(" "),
+    }
+
+    @index.add_document d
+    
+    ## TODO: figure out why this is sometimes triggered
+    #docid, entry = load_entry_for_id m.id
+    #raise "just added message #{m.id} but couldn't find it in a search" unless docid
+    true
+  end
+
+  def drop_entry docno; @index.delete docno; end
+
+  def load_entry_for_id mid
+    results = @index.search(Ferret::Search::TermQuery.new(:message_id, mid))
+    return if results.total_hits == 0
+    docid = results.hits[0].doc
+    [docid, @index[docid]]
+  end
+
+  def load_contacts emails, h={}
+    q = Ferret::Search::BooleanQuery.new true
+    emails.each do |e|
+      qq = Ferret::Search::BooleanQuery.new true
+      qq.add_query Ferret::Search::TermQuery.new(:from, e), :should
+      qq.add_query Ferret::Search::TermQuery.new(:to, e), :should
+      q.add_query qq
+    end
+    q.add_query Ferret::Search::TermQuery.new(:label, "spam"), :must_not
+    
+    Redwood::log "contact search: #{q}"
+    contacts = {}
+    num = h[:num] || 20
+    @index.search_each(q, :sort => "date DESC", :limit => :all) do |docid, score|
+      break if contacts.size >= num
+      #Redwood::log "got message with to: #{@index[docid][:to].inspect} and from: #{@index[docid][:from].inspect}"
+      f = @index[docid][:from]
+      t = @index[docid][:to]
+
+      if AccountManager.is_account_email? f
+        t.split(" ").each { |e| #Redwood::log "adding #{e} because there's a message to him from account email #{f}"; 
+          contacts[Person.for(e)] = true }
+      else
+        #Redwood::log "adding from #{f} because there's a message from him to #{t}"
+        contacts[Person.for(f)] = true
+      end
+    end
+
+    contacts.keys.compact
+  end
+
+protected
+
+  ## TODO: convert this to query objects rather than strings
+  def build_query opts
+    query = ""
+    query += opts[:labels].map { |t| "+label:#{t}" }.join(" ") if opts[:labels]
+    query += " +label:#{opts[:label]}" if opts[:label]
+    query += " #{opts[:content]}" if opts[:content]
+    if opts[:participants]
+      query += "+(" + 
+        opts[:participants].map { |p| "from:#{p.email} OR to:#{p.email}" }.join(" OR ") + ")"
+    end
+        
+    query += " -label:spam" unless opts[:load_spam] || opts[:labels] == :spam || 
+      (opts[:labels] && opts[:labels].include?(:spam))
+    query += " -label:killed" unless opts[:load_killed] || opts[:labels] == :killed || 
+      (opts[:labels] && opts[:labels].include?(:killed))
+    query
+  end
+
+  def load_sources fn=Redwood::SOURCE_FN
+    @sources = Hash[*(Redwood::load_yaml_obj(fn) || []).map { |s| [s.id, s] }.flatten]
+    @sources_dirty = false
+  end
+
+  def save_sources fn=Redwood::SOURCE_FN
+    if @sources_dirty || @sources.any? { |id, s| s.dirty? }
+      FileUtils.mv fn, fn + ".bak", :force => true if File.exists? fn
+      Redwood::save_yaml_obj @sources.values, fn 
+    end
+    @sources_dirty = false
+  end
+
+  def load_some_entries max=ENTRIES_AT_A_TIME, delay1=nil, delay2=nil
+    num = 0
+    begin
+      @sources.each_with_index do |source, source_id|
+        next if source.done? || num >= max
+        source.each do |source_info, label|
+          begin
+            m = Message.new(source, source_info, label + [:inbox])
+            add_message m unless contains_id? m.id
+            puts m.content.inspect
+            num += 1
+          rescue MessageFormatError => e
+            $stderr.puts "ignoring erroneous message at #{source}##{source_info}: #{e.message}"
+          end
+          break if num >= max
+          sleep delay1 if delay1
+        end
+        Redwood::log "loaded #{num} entries from #{source}"
+        sleep delay2 if delay2
+      end
+    ensure
+      save_sources
+      save_index
+    end
+    num
+  end
+end
+
+end