sup 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d330a3455ed22952d8c45fce0f6a0d69346c40f2
-  data.tar.gz: 337b95be43bc301f6180b542b2395df025f4f1db
+  metadata.gz: e7b5ce3c73c1c6d194d8103bb25de84981db2529
+  data.tar.gz: a5d0868505c66e8c7974025d5131738d2a436ab6
 SHA512:
-  metadata.gz: f538e3054ba742a5fadd296bcb6d29b0295e3ca807a0dfe600d908b3cfa9602697bd4e422ac5bde8facb603fe12784add4709ccc4f5ca9b7b76bf348cec3a6f3
-  data.tar.gz: 1923d32e3cfc13998e52bc7c53b19482f6a4ed9b13d4e79880a1a6a16d926b26af2e6aaa73d1ea6556a6d2cdaf51aae1979afc51eff01697df9801dd787a45a3
+  metadata.gz: 8b44f362bd82be161b1d9451d805af1a9d73d962df4c54196703dec1bbca383fb7bd5f2c0bf6479c1fc57bc27b37ec17a65fc2b24abab838e7f2797bfb64687a
+  data.tar.gz: e3a2eaaebf74bec788b18c28be9d9a2bee2b8fa06487780f3199aa7446ec386ac5ffc731595245e40be6423dfc1708bed14fe74ada70eab06097655ccc4fd47a

data/.gitignore

@@ -4,6 +4,7 @@
 *~
 # artifact
 pkg/
+man/
 *.gem
 # i have accidently added this one one too many times
 sup-exception-log.txt
@@ -14,4 +15,5 @@ Gemfile.lock
 
 # generated file for gnupg test
 test/gnupg_test_home/random_seed
+test/gnupg_test_home/trustdb.gpg
 

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "doc/wiki"]
+	path = doc/wiki
+	url = https://github.com/sup-heliotrope/sup.wiki.git

data/.travis.yml

@@ -7,6 +7,7 @@ rvm:
 
 before_install:
   - sudo apt-get update -qq
-  - sudo apt-get install -qq uuid-dev uuid libncursesw5-dev libncursesw5 gnupg2
+  - sudo apt-get install -qq uuid-dev uuid libncursesw5-dev libncursesw5 gnupg2 pandoc
+  - git submodule update --init --recursive
 
 script: bundle exec rake travis

data/CONTRIBUTORS

@@ -13,12 +13,13 @@ Michael Stapelberg <michael at the stapelberg dot des>
 Eric Sherman <hyperbolist at the gmail dot coms>
 Tero Tilus <tero at the tilus dot nets>
 Ben Walton <bwalton at the artsci.utoronto dot cas>
+Scott Bonds <scott at the ggr dot coms>
 Mike Stipicevic <stipim at the rpi dot edus>
 Martin Bähr <mbaehr at the societyserver dot orgs>
+Matthieu Rakotojaona <matthieu.rakotojaona at the gmail dot coms>
 Clint Byrum <clint at the ubuntu dot coms>
 Wael M. Nasreddine <wael.nasreddine at the gmail dot coms>
 Marcus Williams <marcus-sup at the bar-coded dot nets>
-Matthieu Rakotojaona <matthieu.rakotojaona at the gmail dot coms>
 Lionel Ott <white.magic at the gmx dot des>
 Gaudenz Steinlin <gaudenz at the soziologie dot chs>
 Ingmar Vanhassel <ingmar at the exherbo dot orgs>
@@ -37,25 +38,27 @@ Markus Klinik <mkl at the lambdanaut dot nets>
 Bo Borgerson <gigabo at the gmail dot coms>
 Atte Kojo <atte.kojo at the reaktor dot fis>
 Michael Hamann <michael at the content-space dot des>
-William Erik Baxter <web at the superscript dot coms>
 Jonathan Lassoff <jof at the thejof dot coms>
+William Erik Baxter <web at the superscript dot coms>
 Grant Hollingworth <grant at the antiflux dot orgs>
 Ico Doornekamp <ico at the pruts dot nls>
 Adeodato Simó <dato at the net.com.org dot ess>
 Daniel Schoepe <daniel.schoepe at the googlemail dot coms>
 James Taylor <james at the jamestaylor dot orgs>
 Jason Petsod <jason at the petsod dot orgs>
-Robin Burchell <viroteck at the viroteck dot nets>
 Steve Goldman <sgoldman at the tower-research dot coms>
+Robin Burchell <viroteck at the viroteck dot nets>
 Peter Harkins <ph at the malaprop dot orgs>
 Decklin Foster <decklin at the red-bean dot coms>
 Cameron Matheson <cam+sup at the cammunism dot orgs>
 Carl Worth <cworth at the cworth dot orgs>
 Alex Vandiver <alex at the chmrr dot nets>
-Andrew Pimlott <andrew at the pimlott dot nets>
 Jeff Balogh <its.jeff.balogh at the gmail dot coms>
+Andrew Pimlott <andrew at the pimlott dot nets>
 Matías Aguirre <matiasaguirre at the gmail dot coms>
 PaulSmecker <paul.smecker at the gmail dot coms>
+Per Andersson <avtobiff at the gmail dot coms>
+Ruthard Baudach <rthrd at the web dot des>
 Kornilios Kourtis <kkourt at the cslab.ece.ntua dot grs>
 Lars Fischer <fischer at the wiwi.uni-siegen dot des>
 madhat2r <MaDhAt2r at the dukefoo dot coms>
@@ -67,18 +70,17 @@ Steven Lawrance <stl at the koffein dot nets>
 Jonah <Jonah at the GoodCoffee dot cas>
 ian <itaylor at the uark dot edus>
 Adam Lloyd <adam at the alloy-d dot nets>
-MichaelRevell <mikearevell at the gmail dot coms>
-Gregor Hoffleit <gregor at the sam.mediasupervision dot des>
 Todd Eisenberger <teisenbe at the andrew.cmu dot edus>
-Per Andersson <avtobiff at the gmail dot coms>
 0xACE <0xACE at the users.noreply.github dot coms>
+MichaelRevell <mikearevell at the gmail dot coms>
+Gregor Hoffleit <gregor at the sam.mediasupervision dot des>
 Steven Schmeiser <steven at the schmeiser dot orgs>
 Steven Walter <swalter at the monarch.(none)>
 Jon M. Dugan <jdugan at the es dot nets>
-akojo <atte.kojo at the gmail dot coms>
-Matthias Vallentin <vallentin at the icir dot orgs>
-William A. Kennington III <william at the wkennington dot coms>
 Horacio Sanson <horacio at the skillupjapan.co dot jps>
 Stefan Lundström <lundst at the snabb.(none)>
+William A. Kennington III <william at the wkennington dot coms>
+akojo <atte.kojo at the gmail dot coms>
+Matthias Vallentin <vallentin at the icir dot orgs>
 Johannes Larsen <johs.a.larsen at the gmail dot coms>
 Kirill Smelkov <kirr at the landau.phys.spbu dot rus>

data/Gemfile

@@ -1,3 +1,7 @@
 source 'https://rubygems.org/'
 
+if !RbConfig::CONFIG['arch'].include?('openbsd')
+  gem 'xapian-ruby', '~> 1.2.15'
+end
+
 gemspec

data/History.txt

@@ -1,3 +1,16 @@
+== 0.20.0 / 2014-10-06
+
+* add man-pages (generated from wiki) (Per Andersson)!
+* HTML messages or messages that are decoded with the mime-decode hook
+  are now indexed if the mime-decode hook is set up (Scott Bonds).
+* OpenBSD support (Scott Bonds)!
+* goto-hook for keybinding to open URLs.
+* support special charaters in source URIs (Scott Bonds).
+* output message id and locations on all load_from_source failures
+* fix long-standing getlocal bug
+* make new test GPG keys (old ones expired), valid for one year, script
+  now available in devel/ for making new ones.
+
 == 0.19.0 / 2014-07-05
 
 * new check-attachment hook

data/Rakefile

@@ -9,4 +9,45 @@ Rake::TestTask.new(:test) do |test|
 end
 task :default => :test
 
+task :build => [:man]
 task :travis => [:test, :build]
+
+def test_pandoc
+  return system("pandoc -v > /dev/null 2>&1")
+end
+
+task :man do
+  puts "building manpages from wiki.."
+  unless test_pandoc
+    puts "no pandoc installed, needed for manpage generation."
+    return
+  end
+
+  # test if wiki is cloned
+  unless Dir.exist? 'doc/wiki/man'
+    puts "wiki git repository is not cloned in doc/wiki, try: git submodule update --init."
+    return
+  end
+
+  unless Dir.exist? 'man'
+    Dir.mkdir 'man'
+  end
+
+  Dir.glob("doc/wiki/man/*.md").each do |md|
+    m = /^.*\/(?<manpage>[^\/]*)\.md$/.match(md)[:manpage]
+    puts "generating manpage for: #{m}.."
+    r = system "pandoc -s -f markdown -t man #{md} -o man/#{m}"
+
+    unless r
+      puts "failed to generate manpage: #{m}."
+      return
+    end
+  end
+end
+
+task :clean do
+  ['man', 'pkg'].each do |d|
+    puts "cleaning #{d}.."
+    FileUtils.rm_r d if Dir.exist? d
+  end
+end

data/ReleaseNotes

@@ -1,3 +1,13 @@
+Release 0.20.0:
+
+We've got man pages (Mr. Andersson)! We've got OpenBSD support (Scott Bonds)!
+It is now possible to get your HTML emails indexed by setting up a mime-decode
+hook before you index (Scott Bonds)! Scott Bonds also fixed up special character handing in source URIs. It is now possible to set up a goto hook
+for opening the URL below the cursor.
+
+Also a few long standing bugs have been fixed, and new GPG keys have been made
+for the tests.
+
 Release 0.19.0:
 
 New hook: check-attachment and a new option to shows dates in 24h format.

data/bin/sup

@@ -106,26 +106,28 @@ end
 ## ncurses.so that's been compiled against libncursesw. (note the w.) why
 ## this works, i have no idea. much like pretty much every aspect of
 ## dealing with curses.  cargo cult programming at its best.
-require 'dl/import'
 require 'rbconfig'
-module LibC
-  extend DL.const_defined?(:Importer) ? DL::Importer : DL::Importable
-  setlocale_lib = case RbConfig::CONFIG['arch']
-    when /darwin/; "libc.dylib"
-    when /cygwin/; "cygwin1.dll"
-    when /freebsd/; "libc.so.7"
-    else; "libc.so.6"
-  end
+unless RbConfig::CONFIG['arch'] =~ /openbsd/
+  require 'dl/import'
+  module LibC
+    extend DL.const_defined?(:Importer) ? DL::Importer : DL::Importable
+    setlocale_lib = case RbConfig::CONFIG['arch']
+      when /darwin/; "libc.dylib"
+      when /cygwin/; "cygwin1.dll"
+      when /freebsd/; "libc.so.7"
+      else; "libc.so.6"
+    end
 
-  debug "dynamically loading setlocale() from #{setlocale_lib}"
-  begin
-    dlload setlocale_lib
-    extern "void setlocale(int, const char *)"
-    debug "setting locale..."
-    LibC.setlocale(6, "")  # LC_ALL == 6
-  rescue RuntimeError => e
-    warn "cannot dlload setlocale(); ncurses wide character support probably broken."
-    warn "dlload error was #{e.class}: #{e.message}"
+    debug "dynamically loading setlocale() from #{setlocale_lib}"
+    begin
+      dlload setlocale_lib
+      extern "void setlocale(int, const char *)"
+      debug "setting locale..."
+      LibC.setlocale(6, "")  # LC_ALL == 6
+    rescue RuntimeError => e
+      warn "cannot dlload setlocale(); ncurses wide character support probably broken."
+      warn "dlload error was #{e.class}: #{e.message}"
+    end
   end
 end
 

data/ext/mkrf_conf_xapian.rb

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rubygems/command.rb'
+require 'rubygems/dependency_installer.rb'
+require 'rbconfig'
+
+begin
+  Gem::Command.build_args = ARGV
+rescue NoMethodError
+end
+
+puts "xapian: platform specific dependencies.."
+
+inst = Gem::DependencyInstaller.new
+begin
+
+  if !RbConfig::CONFIG['arch'].include?('openbsd')
+    # update version in Gemfile as well
+    name    = "xapian-ruby"
+    version = "~> 1.2.15"
+
+    begin
+      # try to load gem
+
+      gem name, version
+      STDERR.puts "xapian: already installed."
+
+    rescue Gem::LoadError
+
+      STDERR.puts "xapian: installing xapian-ruby.."
+      inst.install name, version
+
+    end
+  else
+    STDERR.puts "xapian: openbsd: you have to install xapian-core and xapian-bindings manually, have a look at: https://github.com/sup-heliotrope/sup/wiki/Installation%3A-OpenBSD"
+  end
+
+rescue
+
+  exit(1)
+
+end
+
+# create dummy rakefile to indicate success
+f = File.open(File.join(File.dirname(__FILE__), "Rakefile"), "w")
+f.write("task :default\n")
+f.close
+

data/lib/sup.rb

@@ -8,6 +8,7 @@ require 'fileutils'
 require 'locale'
 require 'ncursesw'
 require 'rmail'
+require 'uri'
 begin
   require 'fastthread'
 rescue LoadError
@@ -64,6 +65,7 @@ module Redwood
   LEGACY_YAML_DOMAIN = "masanjin.net"
   YAML_DATE = "2006-10-01"
   MAILDIR_SYNC_CHECK_SKIPPED = 'SKIPPED'
+  URI_ENCODE_CHARS = "!*'();:@&=+$,?#[] " # see https://en.wikipedia.org/wiki/Percent-encoding
 
   ## record exceptions thrown in threads nicely
   @exceptions = []

data/lib/sup/buffer.rb

@@ -396,6 +396,18 @@ EOS
     end
   end
 
+  ## ask* functions. these functions display a one-line text field with
+  ## a prompt at the bottom of the screen. answers typed or choosen by
+  ## tab-completion
+  ##
+  ## common arguments are:
+  ##
+  ## domain:      token used as key for @textfields, which seems to be a
+  ##              dictionary of input field objects
+  ## question:    string used as prompt
+  ## completions: array of possible answers, that can be completed by using
+  ##              the tab key
+  ## default:     default value to return
   def ask_with_completions domain, question, completions, default=nil
     ask domain, question, default do |s|
       s.fix_encoding!

data/lib/sup/maildir.rb

@@ -12,7 +12,15 @@ class Maildir < Source
   def initialize uri, usual=true, archived=false, sync_back=true, id=nil, labels=[]
     super uri, usual, archived, id
     @expanded_uri = Source.expand_filesystem_uri(uri)
-    uri = URI(@expanded_uri)
+    parts = @expanded_uri.match /^([a-zA-Z0-9]*:(\/\/)?)(.*)/
+    if parts
+      prefix = parts[1]
+      @path = parts[3]
+      uri = URI(prefix + URI.encode(@path, URI_ENCODE_CHARS))
+    else
+      uri = URI(URI.encode @expanded_uri, URI_ENCODE_CHARS)
+      @path = uri.path
+    end
 
     raise ArgumentError, "not a maildir URI" unless uri.scheme == "maildir"
     raise ArgumentError, "maildir URI cannot have a host: #{uri.host}" if uri.host
@@ -22,7 +30,7 @@ class Maildir < Source
     # sync by default if not specified
     @sync_back = true if @sync_back.nil?
 
-    @dir = uri.path
+    @dir = URI.decode uri.path
     @labels = Set.new(labels || [])
     @mutex = Mutex.new
     @ctimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
@@ -120,7 +128,10 @@ class Maildir < Source
       @ctimes[d] = ctime
 
       old_ids = benchmark(:maildir_read_index) { Index.instance.enum_for(:each_source_info, self.id, "#{d}/").to_a }
-      new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.join(d,File.basename(x)) }.sort }
+      new_ids = benchmark(:maildir_read_dir) {
+        Dir.open(subdir).select {
+          |f| !File.directory? f}.map {
+            |x| File.join(d,File.basename(x)) }.sort }
       added += new_ids - old_ids
       deleted += old_ids - new_ids
       debug "#{old_ids.size} in index, #{new_ids.size} in filesystem"

data/lib/sup/mbox.rb

@@ -19,16 +19,24 @@ class MBox < Source
     case uri_or_fp
     when String
       @expanded_uri = Source.expand_filesystem_uri(uri_or_fp)
-      uri = URI(@expanded_uri)
+      parts = @expanded_uri.match /^([a-zA-Z0-9]*:(\/\/)?)(.*)/
+      if parts
+        prefix = parts[1]
+        @path = parts[3]
+        uri = URI(prefix + URI.encode(@path, URI_ENCODE_CHARS))
+      else
+        uri = URI(URI.encode @expanded_uri, URI_ENCODE_CHARS)
+        @path = uri.path
+      end
+
       raise ArgumentError, "not an mbox uri" unless uri.scheme == "mbox"
       raise ArgumentError, "mbox URI ('#{uri}') cannot have a host: #{uri.host}" if uri.host
       raise ArgumentError, "mbox URI must have a path component" unless uri.path
       @f = nil
-      @path = uri.path
     else
       @f = uri_or_fp
       @path = uri_or_fp.path
-      @expanded_uri = "mbox://#{@path}"
+      @expanded_uri = "mbox://#{URI.encode @path, URI_ENCODE_CHARS}"
     end
 
     super uri_or_fp, usual, archived, id

data/lib/sup/message.rb

@@ -268,6 +268,14 @@ class Message
         debug "could not load message: #{location.inspect}, exception: #{e.inspect}"
 
         [Chunk::Text.new(error_message.split("\n"))]
+
+      rescue Exception => e
+
+        warn "problem reading message #{id}"
+        debug "could not load message: #{location.inspect}, exception: #{e.inspect}"
+
+        raise e
+
       end
   end
 
@@ -327,17 +335,17 @@ EOS
       to.map { |p| p.indexable_content },
       cc.map { |p| p.indexable_content },
       bcc.map { |p| p.indexable_content },
-      indexable_chunks.map { |c| c.lines },
+      indexable_chunks.map { |c| c.lines.map { |l| l.fix_encoding! } },
       indexable_subject,
     ].flatten.compact.join " "
   end
 
   def indexable_body
-    indexable_chunks.map { |c| c.lines }.flatten.compact.join " "
+    indexable_chunks.map { |c| c.lines }.flatten.compact.map { |l| l.fix_encoding! }.join " "
   end
 
   def indexable_chunks
-    chunks.select { |c| c.is_a? Chunk::Text } || []
+    chunks.select { |c| c.indexable? } || []
   end
 
   def indexable_subject

data/lib/sup/message_chunks.rb

@@ -164,6 +164,7 @@ EOS
     ## something we can display inline. otherwise, it's viewable.
     def inlineable?; false end
     def expandable?; !viewable? end
+    def indexable?; expandable? end
     def initial_state; :open end
     def viewable?; @lines.nil? end
     def view_default! path
@@ -229,6 +230,7 @@ EOS
     def inlineable?; true end
     def quotable?; true end
     def expandable?; false end
+    def indexable?; true end
     def viewable?; false end
     def color; :text_color end
   end
@@ -242,6 +244,7 @@ EOS
     def inlineable?; @lines.length == 1 end
     def quotable?; true end
     def expandable?; !inlineable? end
+    def indexable?; expandable? end
     def viewable?; false end
 
     def patina_color; :quote_patina_color end
@@ -258,6 +261,7 @@ EOS
     def inlineable?; @lines.length == 1 end
     def quotable?; false end
     def expandable?; !inlineable? end
+    def indexable?; expandable? end
     def viewable?; false end
 
     def patina_color; :sig_patina_color end
@@ -291,6 +295,7 @@ EOS
     def inlineable?; false end
     def quotable?; false end
     def expandable?; true end
+    def indexable?; true end
     def initial_state; :closed end
     def viewable?; false end
 
@@ -322,6 +327,7 @@ EOS
     def inlineable?; false end
     def quotable?; false end
     def expandable?; !@lines.empty? end
+    def indexable?; false end
     def viewable?; false end
   end
 end

data/lib/sup/modes/edit_message_mode.rb

@@ -20,6 +20,7 @@ Variables:
               to the raw headers for the message. E.g., header["From"],
               header["To"], etc.
   from_email: the email part of the From: line, or nil if empty
+  message_id: the unique message id of the message
 Return value:
   A string (multi-line ok) containing the text of the signature, or nil to
   use the default signature, or :none for no signature.
@@ -688,7 +689,7 @@ private
     from_email = p && p.email
 
     ## first run the hook
-    hook_sig = HookManager.run "signature", :header => @header, :from_email => from_email
+    hook_sig = HookManager.run "signature", :header => @header, :from_email => from_email, :message_id => @message_id
 
     return [] if hook_sig == :none
     return ["", "-- "] + hook_sig.split("\n") if hook_sig

data/lib/sup/modes/thread_index_mode.rb

@@ -856,6 +856,12 @@ protected
     need_update = false
 
     @mutex.synchronize do
+      # and certainly not sure why this happens..
+      #
+      # probably a race condition between thread modification and updating
+      # going on.
+      return if @threads[l].empty?
+
       @size_widgets[l] = size_widget_for_thread @threads[l]
       @date_widgets[l] = date_widget_for_thread @threads[l]
 

data/lib/sup/modes/thread_view_mode.rb

@@ -42,6 +42,14 @@ Return value:
   None.
 EOS
 
+  HookManager.register "goto", <<EOS
+Open the uri given as a parameter.
+Variables:
+      uri: The uri
+Return value:
+  None.
+EOS
+
   register_keymap do |k|
     k.add :toggle_detailed_header, "Toggle detailed header", 'h'
     k.add :show_header, "Show full message header", 'H'
@@ -80,6 +88,8 @@ EOS
     k.add :kill_and_next, "Kill this thread, kill buffer, and view next", '&'
     k.add :toggle_wrap, "Toggle wrapping of text", 'w'
 
+    k.add :goto_uri, "Goto uri under cursor", 'g'
+
     k.add_multi "(a)rchive/(d)elete/mark as (s)pam/mark as u(N)read:", '.' do |kk|
       kk.add :archive_and_kill, "Archive this thread and kill buffer", 'a'
       kk.add :delete_and_kill, "Delete this thread and kill buffer", 'd'
@@ -722,6 +732,48 @@ EOS
     [user_labels, super].join(" -- ")
   end
 
+  def goto_uri
+    unless (chunk = @chunk_lines[curpos])
+      BufferManager.flash "No URI found."
+      return
+    end
+    unless HookManager.enabled? "goto"
+      BufferManager.flash "You must add a goto.rb hook before you can goto a URI."
+      return
+    end
+
+    # @text is a list of lines with this format:
+    # [
+    #   [[:text_color, "Some text"]]
+    #   [[:text_color, " continued here"]]
+    # ]
+
+    linetext = @text.slice(curpos, @text.length).flatten(1)
+      .take_while{|d| d[0] == :text_color and d[1].strip != ""} # Only take up to the first "" alone on its line
+      .map{|d| d[1].strip}.join("").strip
+
+    found = false
+    (linetext || "").scan(URI::regexp).each do |matches|
+      begin
+        link = $& # ruby magic: $& is the whole regexp match
+        u = URI.parse(link)
+        next unless u.absolute?
+        next unless ["http", "https"].include?(u.scheme)
+
+        reallink = Shellwords.escape(u.to_s)
+        BufferManager.flash "Going to #{reallink} ..."
+        HookManager.run "goto", :uri => reallink
+        BufferManager.completely_redraw_screen
+        found = true
+
+      rescue URI::InvalidURIError => e
+        debug "not a uri: #{e}"
+        # Do nothing, this is an ok flow
+      end
+    end
+    BufferManager.flash "No URI found." unless found
+  end
+
 private
 
   def initial_state_for m

data/lib/sup/version.rb

@@ -1,3 +1,3 @@
 module Redwood
-  VERSION = "0.19.0"
+  VERSION = "0.20.0"
 end

data/sup.gemspec

@@ -25,16 +25,31 @@ DESC
 SUP: please note that our old mailing lists have been shut down,
      re-subscribe to supmua@googlegroups.com to discuss and follow
      updates on sup (send email to: supmua+subscribe@googlegroups.com).
+
+     OpenBSD users:
+     If your operating system is OpenBSD you have some
+     additional, manual steps to do before Sup will work, see:
+     https://github.com/sup-heliotrope/sup/wiki/Installation%3A-OpenBSD.
   EOF
 
   s.files         = `git ls-files -z`.split("\x0")
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ["lib"]
+  s.extra_rdoc_files = Dir.glob("man/*")
 
   s.required_ruby_version = '>= 1.9.3'
 
-  s.add_runtime_dependency "xapian-ruby", "~> 1.2.15"
+  # this is here to support skipping the xapian-ruby installation on OpenBSD
+  # because the xapian-ruby gem doesn't install on OpenBSD, you must install
+  # xapian-core and xapian-bindings manually on OpenBSD
+  # see https://github.com/sup-heliotrope/sup/wiki/Installation%3A-OpenBSD
+  # and https://en.wikibooks.org/wiki/Ruby_Programming/RubyGems#How_to_install_different_versions_of_gems_depending_on_which_version_of_ruby_the_installee_is_using
+  s.extensions = %w[ext/mkrf_conf_xapian.rb]
+
+  ## remember to update the xapian dependency in
+  ## ext/mkrf_conf_xapian.rb and Gemfile.
+
   s.add_runtime_dependency "ncursesw", "~> 1.4.0"
   s.add_runtime_dependency "rmail-sup", "~> 1.0.1"
   s.add_runtime_dependency "highline"
@@ -50,4 +65,5 @@ SUP: please note that our old mailing lists have been shut down,
   s.add_development_dependency "minitest", "~> 4.7"
   s.add_development_dependency "rr", "~> 1.0.5"
   s.add_development_dependency "gpgme", ">= 2.0.2"
+
 end

data/test/gnupg_test_home/gpg.conf

@@ -1 +1,2 @@
-default-key 789E7011
+trust-model always
+

data/test/gnupg_test_home/key1.gen

@@ -0,0 +1,15 @@
+ %echo Generating a standard key
+ Key-Type: DSA
+ Key-Length: 1024
+ Subkey-Type: ELG-E
+ Subkey-Length: 1024
+ Name-Real: Sup Test Sender 1 
+ Name-Comment: Test sender key
+ Name-Email: sup-test-1@foo.bar 
+ Expire-Date: 1y
+ %no-protection
+ %pubring pubring.gpg
+ %secring secring.gpg
+ # Do a commit here, so that we can later print "done" :-)
+ %commit
+ %echo done

data/test/gnupg_test_home/key2.gen

@@ -0,0 +1,15 @@
+ %echo Generating a standard key
+ Key-Type: DSA
+ Key-Length: 1024
+ Subkey-Type: ELG-E
+ Subkey-Length: 1024
+ Name-Real: Sup Test Receiver
+ Name-Comment: Test receiver for Sup
+ Name-Email: sup-test-2@foo.bar
+ Expire-Date: 1y
+ %no-protection
+ %pubring pubring.gpg
+ %secring secring.gpg
+ # Do a commit here, so that we can later print "done" :-)
+ %commit
+ %echo done

data/test/gnupg_test_home/regen_keys.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+#
+# re-generate test keys for the sup test base
+#
+# https://github.com/sup-heliotrope/sup/wiki/Development%3A-Crypto
+
+pushd $(dirname $0)
+
+export GNUPGHOME="$(pwd)"
+
+echo "genrating keys in: $GNUPGHOME.."
+
+rm *.gpg *.asc
+
+echo "generate receiver key.."
+gpg --batch --gen-key key2.gen
+
+echo "export receiver key.."
+
+gpg --output sup-test-2@foo.bar.asc --armor --export sup-test-2@foo.bar
+
+mv trustdb.gpg receiver_trustdb.gpg
+mv secring.gpg receiver_secring.gpg
+mv pubring.gpg receiver_pubring.gpg
+
+echo "generate sender key.."
+gpg --batch --gen-key key1.gen
+
+echo "import receiver key.."
+gpg --import sup-test-2@foo.bar.asc
+
+
+
+popd
+

data/test/gnupg_test_home/sup-test-2@foo.bar.asc

@@ -1,20 +1,25 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v2.0.20 (GNU/Linux)
+Version: GnuPG v2
 
-mI0EUgi0fAEEAOLAcQW96NEUSB7YE/la8X56jGW5BMX3aAixOF8LvOwMBbUK1T+U
-0H2PGIrXVcYyHcPqWRpRahbsIAldBqzffPlzMa+aqJaB1xKkNruxSoIzwPdidZMe
-l0Dxz2FDsoXD0KPyWnAYhGmQyz2MFpZxu2tlYqvwWVW//XGnk/KHvIXbABEBAAG0
-PlN1cCBUZXN0IFJlY2VpdmVyIChUZXN0IHJlY2VpdmVyIGZvciBTdXApIDxzdXAt
-dGVzdC0yQGZvby5iYXI+iL8EEwECACkFAlIItHwCGwMFCQHhM4AHCwkIBwMCAQYV
-CAIJCgsEFgIDAQIeAQIXgAAKCRAsABl+cWpykMMVBADHkQPgTz0CqKKp3k+z3dbm
-ocmI4tYNn1dOkDQqyfoBTfs6L3g4j5OE2UrguntRYyg5oon+uO5d18CQ5dY0sCw/
-o5IwyzTrxI8IocbtZvBdSb+XjLndynGuIQoqaJq9i6n1V4klFHVOna8Q9JstLfRX
-H1d4xPhnvKcaDDx/NV3X/biNBFIItHwBBADBpb43MpkrUWlg7HWJ1ZfOlxnOxrJ3
-Gz9WFNV06UbcZEuFKA/vHRjM6gWzUn5903FLuCWu3eBrq5xQfWipbp187PmocpoG
-skJ6gosLs1fMYRBjv2VbG9xJVKdKJMjqZw5FUpXKAaHr8P9jN6g2STQrbeQ8CVUK
-h7zOWRXAXSKUgwARAQABiKUEGAECAA8FAlIItHwCGwwFCQHhM4AACgkQLAAZfnFq
-cpDV1QQAzcxFXznEX92DjWxWRC7gRHgIsQk9WJnDzjtnDjSWCp3H85qeTZGZrn9W
-NoneV/S5Y7K3Mkceh4rFaANQ3zx4b05y1LFt5N/lPwIe5VB0vcPumtZum2fSGfpK
-nTXvzelcWcm2aGyUSaWvOkntWKEEt1kB5Oq6EtZoRZLMzAxLd7s=
-=aKsV
+mQGiBFP3VogRBADVBEkaZQXj728C1HUIaTRDCFoKzojwC79Z1BLsD72qQYE8z1ic
+5P9CJpJU5wbhQFDTGBjw+i1nNTWy01z4q5bfFqok+KorT3XNp5IJRcRIEOkj+Twq
+7ZaSODwXGsUmdzSoOVDYmtUpVzRQe0IM0rPQQV4vGzgw55FdJBe7a63nIwCg+WvR
+iQN09PlhpGG7SIEmx0psEqUEAL/t1c5oC9RC7L4a0GM+2AcgFRBMXvzpdnytrzgt
+73Ud6CcUplQp6WODrUYhX0RLzSJPO4zWDsBmkBad/iQCwbCKpFPfAFdBMArJpknx
+rc6vRED4a9dLfCNTT1g86CkiElge9t36juZgOoFT3xt/XP7BxhU1fCFshZNR6VK6
+tN9eA/9G4fUX6XvEGIrNiBYKyU4QvM1nyMXCBujm7vYF6KfSlYyAvVXxG4h+mvUy
+ZXQ/WHMQJSbPTY3dd4hmo0p0GUMlSvXU8JLf7qienW1IccD9Pv88J1XjkbFd+wgw
+feoSx1sAfc36gH+aE17lvsU+PPAP4Bc9CSiScNo0iQv7v/KZjrQ+U3VwIFRlc3Qg
+UmVjZWl2ZXIgKFRlc3QgcmVjZWl2ZXIgZm9yIFN1cCkgPHN1cC10ZXN0LTJAZm9v
+LmJhcj6IaQQTEQIAKQUCU/dWiAIbIwUJAeEzgAcLCQgHAwIBBhUIAgkKCwQWAgMB
+Ah4BAheAAAoJEKfs+g8ACvQGPxIAnj1CSZCzjwyIFLgNEQnIhntU+b28AKDsMEVN
+gf9mHqwhabN+UKgBwX0U3LkBDQRT91aIEAQAjQZEnDK++SKp/l2Oiku6H9IuCsi4
+lv+MhLQP0bMuD4DrPk3mauZNc8BB+U0wgAMh/kZoCKySEdMK1mcf2iOsd5yOCrK+
+sJQAMsALAnrYjCE9QA2xIQs8gHF4PrKopycF55iRHQMDNa1QWfs+j4WJaXderlGQ
+S0dGfLyoqtZsFusAAwUEAIi0+aDZlAVVIdDO2cvR0lu6eDW2Mr2ExZzuwTfAI6dS
+tJLoPzoA2OAVW7cFVVpCOHcVLiF2GOHvtJPw1MgpxaNjzpNdJPTiP2sYZg253dfR
+v66Cw9IuWKgZcElWXmIy5vFWqWWbLyTBOuwEQxCsFnjN9UUZauSADOJSPFy1sekf
+iE8EGBECAA8FAlP3VogCGwwFCQHhM4AACgkQp+z6DwAK9Ab/swCg8LWNwfMwNk+H
+gLgnS1LVsesZ8D4An2Ie2P0/oYuSmPPFV44kbWySX9wW
+=Jo82
 -----END PGP PUBLIC KEY BLOCK-----

data/test/integration/test_maildir.rb

@@ -0,0 +1,75 @@
+require "test_helper"
+
+class TestMaildir < MiniTest::Unit::TestCase
+
+  def setup
+    @path = Dir.mktmpdir
+
+    @test_message_1 = <<EOS
+From: Bob <bob@bob.com>
+To: a dear friend
+
+Hello there friend. How are you? Blah is blah blah.
+Wow. Maildir FTW, am I right?
+EOS
+
+  end
+
+  def teardown
+    ObjectSpace.each_object(Class).select {|a| a < Redwood::Singleton}.each do |klass|
+      klass.deinstantiate! unless klass == Redwood::Logger
+    end
+    FileUtils.rm_r @path
+  end
+
+  def create_a_maildir(extra='')
+    maildir = File.join @path, "test_maildir#{extra}"
+    ['', 'cur', 'new', 'tmp'].each do |dir|
+      Dir.mkdir(File.join maildir, dir)
+    end
+    maildir
+  end
+
+  def create_a_maildir_email(folder, content)
+    File.write(File.join(folder, "#{Time.now.to_f}.hostname:2,S"), content)
+  end
+
+  def start_sup_and_add_source(source)
+    start
+    Index.init @path
+    Index.load
+    SourceManager.instance.instance_eval '@sources = {}'
+    SourceManager.instance.add_source source
+    PollManager.poll_from source
+  end
+
+  # and now, let the tests begin!
+
+  def test_can_index_a_maildir_directory
+
+    maildir = create_a_maildir
+    create_a_maildir_email(File.join(maildir, 'cur'), @test_message_1)
+    start_sup_and_add_source Maildir.new "maildir:#{maildir}"
+
+    messages_in_index = []
+    Index.instance.each_message {|a| messages_in_index << a}
+    refute_empty messages_in_index, 'There are no messages in the index'
+    assert_equal(messages_in_index.first.raw_message, @test_message_1)
+
+  end
+
+  def test_can_index_a_maildir_directory_with_special_characters
+
+    maildir = create_a_maildir URI_ENCODE_CHARS
+    create_a_maildir_email(File.join(maildir, 'cur'), @test_message_1)
+    start_sup_and_add_source Maildir.new "maildir:#{maildir}"
+
+    messages_in_index = []
+    Index.instance.each_message {|a| messages_in_index << a}
+    refute_empty messages_in_index, 'There are no messages in the index'
+    assert_equal(messages_in_index.first.raw_message, @test_message_1)
+
+  end
+
+end
+

data/test/integration/test_mbox.rb

@@ -0,0 +1,69 @@
+require "test_helper"
+
+class TestMbox < MiniTest::Unit::TestCase
+
+  def setup
+    @path = Dir.mktmpdir
+
+    @test_message_1 = <<EOS
+From sup-talk-bounces@rubyforge.org Mon Apr 27 12:56:18 2009
+From: Bob <bob@bob.com>
+To: Joe <joe@joe.com>
+
+Hello there friend. How are you? Blah is blah blah.
+I like mboxes, don't you?
+EOS
+
+  end
+
+  def teardown
+    ObjectSpace.each_object(Class).select {|a| a < Redwood::Singleton}.each do |klass|
+      klass.deinstantiate! unless klass == Redwood::Logger
+    end
+    FileUtils.rm_r @path
+  end
+
+  def create_a_mbox(extra='')
+    mbox = File.join(@path, "test_mbox#{extra}.mbox")
+    File.write(mbox, @test_message_1)
+    mbox
+  end
+
+  def start_sup_and_add_source(source)
+    start
+    Index.init @path
+    Index.load
+    SourceManager.instance.instance_eval '@sources = {}'
+    SourceManager.instance.add_source source
+    PollManager.poll_from source
+  end
+
+  # and now, let the tests begin!
+
+  def test_can_index_a_mbox_directory
+
+    mbox = create_a_mbox
+    start_sup_and_add_source MBox.new "mbox:#{mbox}"
+
+    messages_in_index = []
+    Index.instance.each_message {|a| messages_in_index << a}
+    refute_empty messages_in_index, 'There are no messages in the index'
+    test_message_without_first_line = @test_message_1.sub(/^.*\n/,'')
+    assert_equal(messages_in_index.first.raw_message, test_message_without_first_line)
+
+  end
+
+  def test_can_index_a_mbox_directory_with_special_characters
+
+    mbox = create_a_mbox URI_ENCODE_CHARS
+    start_sup_and_add_source MBox.new "mbox:#{mbox}"
+
+    messages_in_index = []
+    Index.instance.each_message {|a| messages_in_index << a}
+    refute_empty messages_in_index, 'There are no messages in the index'
+    test_message_without_first_line = @test_message_1.sub(/^.*\n/,'')
+    assert_equal(messages_in_index.first.raw_message, test_message_without_first_line)
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sup
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
 platform: ruby
 authors:
 - William Morgan
@@ -11,22 +11,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-05 00:00:00.000000000 Z
+date: 2014-10-06 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: xapian-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.15
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.15
 - !ruby/object:Gem::Dependency
   name: ncursesw
   requirement: !ruby/object:Gem::Requirement
@@ -244,10 +230,22 @@ executables:
 - sup-sync
 - sup-sync-back-maildir
 - sup-tweak-labels
-extensions: []
-extra_rdoc_files: []
+extensions:
+- ext/mkrf_conf_xapian.rb
+extra_rdoc_files:
+- man/sup-tweak-labels.1
+- man/sup-sync.1
+- man/sup-psych-ify-config-files.1
+- man/sup-config.1
+- man/sup-import-dump.1
+- man/sup-dump.1
+- man/sup-sync-back-maildir.1
+- man/sup-add.1
+- man/sup-recover-sources.1
+- man/sup.1
 files:
 - ".gitignore"
+- ".gitmodules"
 - ".travis.yml"
 - CONTRIBUTORS
 - Gemfile
@@ -277,6 +275,7 @@ files:
 - doc/FAQ.txt
 - doc/Hooks.txt
 - doc/Philosophy.txt
+- ext/mkrf_conf_xapian.rb
 - lib/sup.rb
 - lib/sup/account.rb
 - lib/sup/buffer.rb
@@ -342,17 +341,30 @@ files:
 - lib/sup/util/query.rb
 - lib/sup/util/uri.rb
 - lib/sup/version.rb
+- man/sup-add.1
+- man/sup-config.1
+- man/sup-dump.1
+- man/sup-import-dump.1
+- man/sup-psych-ify-config-files.1
+- man/sup-recover-sources.1
+- man/sup-sync-back-maildir.1
+- man/sup-sync.1
+- man/sup-tweak-labels.1
+- man/sup.1
 - sup.gemspec
 - test/dummy_source.rb
 - test/gnupg_test_home/gpg.conf
+- test/gnupg_test_home/key1.gen
+- test/gnupg_test_home/key2.gen
 - test/gnupg_test_home/pubring.gpg
 - test/gnupg_test_home/receiver_pubring.gpg
 - test/gnupg_test_home/receiver_secring.gpg
-- test/gnupg_test_home/receiver_trustdb.gpg
+- test/gnupg_test_home/regen_keys.sh
 - test/gnupg_test_home/secring.gpg
 - test/gnupg_test_home/sup-test-2@foo.bar.asc
-- test/gnupg_test_home/trustdb.gpg
 - test/integration/test_label_service.rb
+- test/integration/test_maildir.rb
+- test/integration/test_mbox.rb
 - test/messages/bad-content-transfer-encoding-1.eml
 - test/messages/binary-content-transfer-encoding-2.eml
 - test/messages/missing-line.eml
@@ -376,6 +388,11 @@ post_install_message: |
   SUP: please note that our old mailing lists have been shut down,
        re-subscribe to supmua@googlegroups.com to discuss and follow
        updates on sup (send email to: supmua+subscribe@googlegroups.com).
+
+       OpenBSD users:
+       If your operating system is OpenBSD you have some
+       additional, manual steps to do before Sup will work, see:
+       https://github.com/sup-heliotrope/sup/wiki/Installation%3A-OpenBSD.
 rdoc_options: []
 require_paths:
 - lib
@@ -398,14 +415,17 @@ summary: A console-based email client with the best features of GMail, mutt and
 test_files:
 - test/dummy_source.rb
 - test/gnupg_test_home/gpg.conf
+- test/gnupg_test_home/key1.gen
+- test/gnupg_test_home/key2.gen
 - test/gnupg_test_home/pubring.gpg
 - test/gnupg_test_home/receiver_pubring.gpg
 - test/gnupg_test_home/receiver_secring.gpg
-- test/gnupg_test_home/receiver_trustdb.gpg
+- test/gnupg_test_home/regen_keys.sh
 - test/gnupg_test_home/secring.gpg
 - test/gnupg_test_home/sup-test-2@foo.bar.asc
-- test/gnupg_test_home/trustdb.gpg
 - test/integration/test_label_service.rb
+- test/integration/test_maildir.rb
+- test/integration/test_mbox.rb
 - test/messages/bad-content-transfer-encoding-1.eml
 - test/messages/binary-content-transfer-encoding-2.eml
 - test/messages/missing-line.eml