segfault-larch 1.0.2.3

data/bin/larch

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'highline/import' # optional dep: termios
+require 'trollop'
+
+require 'larch'
+
+module Larch
+
+  # Parse command-line options.
+  options = Trollop.options do
+    version "Larch #{APP_VERSION}\n" << APP_COPYRIGHT
+    banner <<-EOS
+Larch syncs messages from one IMAP server to another. Awesomely.
+
+Usage:
+  larch [config section] [options]
+  larch --from <uri> --to <uri> [options]
+
+Server Options:
+EOS
+    opt :from,        "URI of the source IMAP server.", :short => '-f', :type => :string
+    opt :from_folder, "Source folder to copy from", :short => '-F', :default => Config::DEFAULT['from-folder']
+    opt :from_pass,   "Source server password (default: prompt)", :short => '-p', :type => :string
+    opt :from_user,   "Source server username (default: prompt)", :short => '-u', :type => :string
+    opt :to,          "URI of the destination IMAP server.", :short => '-t', :type => :string
+    opt :to_folder,   "Destination folder to copy to", :short => '-T', :default => Config::DEFAULT['to-folder']
+    opt :to_pass,     "Destination server password (default: prompt)", :short => '-P', :type => :string
+    opt :to_user,     "Destination server username (default: prompt)", :short => '-U', :type => :string
+
+    text "\nSync Options:"
+    opt :all,              "Copy all folders recursively", :short => '-a'
+    opt :all_subscribed,   "Copy all subscribed folders recursively", :short => '-s'
+    opt :exclude,          "List of mailbox names/patterns that shouldn't be copied", :short => :none, :type => :strings, :multi => true
+    opt :exclude_file,     "Filename containing mailbox names/patterns that shouldn't be copied", :short => :none, :type => :string
+
+    text "\nGeneral Options:"
+    opt :config,           "Specify a non-default config file to use", :short => '-c', :default => Config::DEFAULT['config']
+    opt :database,         "Specify a non-default message database to use", :short => :none, :default => Config::DEFAULT['database']
+    opt :dry_run,          "Don't actually make any changes", :short => '-n'
+    opt :max_retries,      "Maximum number of times to retry after a recoverable error", :short => :none, :default => Config::DEFAULT['max-retries']
+    opt :no_create_folder, "Don't create destination folders that don't already exist", :short => :none
+    opt :ssl_certs,        "Path to a trusted certificate bundle to use to verify server SSL certificates", :short => :none, :type => :string
+    opt :ssl_verify,       "Verify server SSL certificates", :short => :none
+    opt :verbosity,        "Output verbosity: debug, info, warn, error, or fatal", :short => '-V', :default => Config::DEFAULT['verbosity']
+  end
+
+  # Load config.
+  config = Config.new(ARGV.shift || 'default', options[:config], options)
+  
+  if options[:config_given]
+    Trollop.die :config, ": file not found: #{options[:config]}" unless File.exist?(options[:config])
+  end
+
+  # Validate config.
+  begin
+    config.validate
+  rescue Config::Error => e
+    abort "Config error: #{e}"
+  end
+
+  # Create URIs.
+  uri_from = URI(config.from)
+  uri_to   = URI(config.to)
+
+  # Use --from-folder and --to-folder unless folders were specified in the URIs.
+  uri_from.path ||= '/' + CGI.escape(config.from_folder.gsub(/^\//, ''))
+  uri_to.path   ||= '/' + CGI.escape(config.to_folder.gsub(/^\//, ''))
+
+  # --all and --all-subscribed options override folders
+  if config.all || config.all_subscribed
+    uri_from.path = ''
+    uri_to.path   = ''
+  end
+
+  # Usernames and passwords specified as arguments override those in the URIs
+  uri_from.user     = CGI.escape(config.from_user) if config.from_user
+  uri_from.password = CGI.escape(config.from_pass) if config.from_pass
+  uri_to.user       = CGI.escape(config.to_user) if config.to_user
+  uri_to.password   = CGI.escape(config.to_pass) if config.to_pass
+
+  # If usernames/passwords aren't specified in either URIs or config, then prompt.
+  uri_from.user     ||= CGI.escape(ask("Source username (#{uri_from.host}): "))
+  uri_from.password ||= CGI.escape(ask("Source password (#{uri_from.host}): ") {|q| q.echo = false })
+  uri_to.user       ||= CGI.escape(ask("Destination username (#{uri_to.host}): "))
+  uri_to.password   ||= CGI.escape(ask("Destination password (#{uri_to.host}): ") {|q| q.echo = false })
+
+  # Go go go!
+  init(config)
+
+  imap_from = Larch::IMAP.new(uri_from,
+      :dry_run     => config[:dry_run],
+      :max_retries => config[:max_retries],
+      :ssl_certs   => config[:ssl_certs] || nil,
+      :ssl_verify  => config[:ssl_verify]
+  )
+
+  imap_to = Larch::IMAP.new(uri_to,
+      :create_mailbox => !config[:no_create_folder] && !config[:dry_run],
+      :dry_run        => config[:dry_run],
+      :max_retries    => config[:max_retries],
+      :ssl_certs      => config[:ssl_certs] || nil,
+      :ssl_verify     => config[:ssl_verify]
+  )
+
+  unless RUBY_PLATFORM =~ /mswin|mingw|bccwin|wince|java/
+    begin
+      for sig in [:SIGINT, :SIGQUIT, :SIGTERM]
+        trap(sig) { @log.fatal "Interrupted (#{sig})"; Kernel.exit }
+      end
+    rescue => e
+    end
+  end
+
+  if config.all
+    copy_all(imap_from, imap_to)
+  elsif config.all_subscribed
+    copy_all(imap_from, imap_to, true)
+  else
+    copy_folder(imap_from, imap_to)
+  end
+end

data/lib/larch.rb

@@ -0,0 +1,254 @@
+# Prepend this file's directory to the include path if it's not there already.
+$:.unshift(File.dirname(File.expand_path(__FILE__)))
+$:.uniq!
+
+require 'cgi'
+require 'digest/md5'
+require 'fileutils'
+require 'net/imap'
+require 'time'
+require 'uri'
+require 'yaml'
+
+require 'sequel'
+require 'sequel/extensions/migration'
+
+require 'larch/config'
+require 'larch/errors'
+require 'larch/imap'
+require 'larch/imap/mailbox'
+require 'larch/logger'
+require 'larch/version'
+
+module Larch
+
+  class << self
+    attr_reader :config, :db, :log, :exclude
+
+    EXCLUDE_COMMENT = /#.*$/
+    EXCLUDE_REGEX   = /^\s*\/(.*)\/\s*/
+    GLOB_PATTERNS   = {'*' => '.*', '?' => '.'}
+    LIB_DIR         = File.join(File.dirname(File.expand_path(__FILE__)), 'larch')
+
+    def init(config)
+      raise ArgumentError, "config must be a Larch::Config instance" unless config.is_a?(Config)
+
+      @config = config
+      @log    = Logger.new(@config[:verbosity])
+      @db     = open_db(@config[:database])
+
+      @exclude = @config[:exclude].map do |e|
+        if e =~ EXCLUDE_REGEX
+          Regexp.new($1, Regexp::IGNORECASE)
+        else
+          glob_to_regex(e.strip)
+        end
+      end
+
+      load_exclude_file(@config[:exclude_file]) if @config[:exclude_file]
+
+      Net::IMAP.debug = true if @log.level == :insane
+
+      # Stats
+      @copied = 0
+      @failed = 0
+      @total  = 0
+    end
+
+    # Recursively copies all messages in all folders from the source to the
+    # destination.
+    def copy_all(imap_from, imap_to, subscribed_only = false)
+      raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
+      raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
+
+      @copied = 0
+      @failed = 0
+      @total  = 0
+
+      imap_from.each_mailbox do |mailbox_from|
+        next if excluded?(mailbox_from.name)
+        next if subscribed_only && !mailbox_from.subscribed?
+
+        mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim)
+        mailbox_to.subscribe if mailbox_from.subscribed?
+
+        copy_messages(mailbox_from, mailbox_to)
+      end
+
+    rescue => e
+      @log.fatal e.message
+
+    ensure
+      summary
+    end
+
+    # Copies the messages in a single IMAP folder and all its subfolders
+    # (recursively) from the source to the destination.
+    def copy_folder(imap_from, imap_to)
+      raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
+      raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
+
+      @copied = 0
+      @failed = 0
+      @total  = 0
+
+      mailbox_from = imap_from.mailbox(imap_from.uri_mailbox || 'INBOX')
+      mailbox_to   = imap_to.mailbox(imap_to.uri_mailbox || 'INBOX')
+
+      copy_mailbox(mailbox_from, mailbox_to)
+
+      imap_from.disconnect
+      imap_to.disconnect
+
+    rescue => e
+      @log.fatal e.message
+
+    ensure
+      summary
+    end
+
+    # Opens a connection to the Larch message database, creating it if
+    # necessary.
+    def open_db(database)
+      filename  = File.expand_path(database)
+      directory = File.dirname(filename)
+
+      unless File.exist?(directory)
+        FileUtils.mkdir_p(directory)
+        File.chmod(0700, directory)
+      end
+
+      begin
+        db = Sequel.connect("sqlite://#{filename}")
+        db.test_connection
+      rescue => e
+        @log.fatal "unable to open message database: #{e}"
+        abort
+      end
+
+      # Ensure that the database schema is up to date.
+      migration_dir = File.join(LIB_DIR, 'db', 'migrate')
+
+      unless Sequel::Migrator.get_current_migration_version(db) ==
+          Sequel::Migrator.latest_migration_version(migration_dir)
+        begin
+          Sequel::Migrator.apply(db, migration_dir)
+        rescue => e
+          @log.fatal "unable to migrate message database: #{e}"
+          abort
+        end
+      end
+
+      require 'larch/db/message'
+      require 'larch/db/mailbox'
+      require 'larch/db/account'
+
+      db
+    end
+
+    def summary
+      @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@total - @copied - @failed} untouched out of #{@total} total"
+    end
+
+    private
+
+    def copy_mailbox(mailbox_from, mailbox_to)
+      raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(Larch::IMAP::Mailbox)
+      raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(Larch::IMAP::Mailbox)
+
+      return if excluded?(mailbox_from.name) || excluded?(mailbox_to.name)
+
+      mailbox_to.subscribe if mailbox_from.subscribed?
+      copy_messages(mailbox_from, mailbox_to)
+
+      mailbox_from.each_mailbox do |child_from|
+        next if excluded?(child_from.name)
+        child_to = mailbox_to.imap.mailbox(child_from.name, child_from.delim)
+        copy_mailbox(child_from, child_to)
+      end
+    end
+
+    def copy_messages(mailbox_from, mailbox_to)
+      raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(Larch::IMAP::Mailbox)
+      raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(Larch::IMAP::Mailbox)
+
+      return if excluded?(mailbox_from.name) || excluded?(mailbox_to.name)
+
+      imap_from = mailbox_from.imap
+      imap_to   = mailbox_to.imap
+
+      @log.info "copying messages from #{imap_from.host}/#{mailbox_from.name} to #{imap_to.host}/#{mailbox_to.name}"
+
+      @total += mailbox_from.length
+
+      mailbox_from.each_guid do |guid|
+        next if mailbox_to.has_guid?(guid)
+
+        begin
+          next unless msg = mailbox_from.peek(guid)
+
+          if msg.envelope.from
+            env_from = msg.envelope.from.first
+            from = "#{env_from.mailbox}@#{env_from.host}"
+          else
+            from = '?'
+          end
+
+          @log.info "copying message: #{from} - #{msg.envelope.subject}"
+
+          mailbox_to << msg
+          @copied += 1
+
+        rescue Larch::IMAP::Error => e
+          @failed += 1
+          @log.error e.message
+          next
+        end
+      end
+    end
+
+    def excluded?(name)
+      name = name.downcase
+
+      @exclude.each do |e|
+        return true if (e.is_a?(Regexp) ? !!(name =~ e) : File.fnmatch?(e, name))
+      end
+
+      return false
+    end
+
+    def glob_to_regex(str)
+      str.gsub!(/(.)/) {|c| GLOB_PATTERNS[$1] || Regexp.escape(c) }
+      Regexp.new("^#{str}$", Regexp::IGNORECASE)
+    end
+
+    def load_exclude_file(filename)
+      @exclude ||= []
+      lineno = 0
+
+      File.open(filename, 'rb') do |f|
+        f.each do |line|
+          lineno += 1
+
+          # Strip comments.
+          line.sub!(EXCLUDE_COMMENT, '')
+          line.strip!
+
+          # Skip empty lines.
+          next if line.empty?
+
+          if line =~ EXCLUDE_REGEX
+            @exclude << Regexp.new($1, Regexp::IGNORECASE)
+          else
+            @exclude << glob_to_regex(line)
+          end
+        end
+      end
+
+    rescue => e
+      raise Larch::IMAP::FatalError, "error in exclude file at line #{lineno}: #{e}"
+    end
+
+  end
+
+end

data/lib/larch/config.rb

@@ -0,0 +1,105 @@
+module Larch
+
+class Config
+  attr_reader :filename, :section
+
+  DEFAULT = {
+    'all'              => false,
+    'all-subscribed'   => false,
+    'config'           => File.join('~', '.larch', 'config.yaml'),
+    'database'         => File.join('~', '.larch', 'larch.db'),
+    'dry-run'          => false,
+    'exclude'          => [],
+    'exclude-file'     => nil,
+    'from'             => nil,
+    'from-folder'      => 'INBOX',
+    'from-pass'        => nil,
+    'from-user'        => nil,
+    'max-retries'      => 3,
+    'no-create-folder' => false,
+    'ssl-certs'        => nil,
+    'ssl-verify'       => false,
+    'to'               => nil,
+    'to-folder'        => 'INBOX',
+    'to-pass'          => nil,
+    'to-user'          => nil,
+    'verbosity'        => 'info'
+  }.freeze
+
+  def initialize(section = 'default', filename = DEFAULT['config'], override = {})
+    @section  = section.to_s
+    @override = {}
+
+    override.each do |k, v|
+      k = k.to_s.gsub('_', '-')
+      @override[k] = v if DEFAULT.has_key?(k) && v != DEFAULT[k]
+    end
+
+    load_file(filename)
+  end
+
+  def fetch(name)
+    (@cached || {})[name.to_s.gsub('_', '-')] || nil
+  end
+  alias [] fetch
+
+  def load_file(filename)
+    @filename = File.expand_path(filename)
+
+    config = {}
+
+    if File.exist?(@filename)
+      begin
+        config = YAML.load_file(@filename)
+      rescue => e
+        raise Larch::Config::Error, "config error in #{filename}: #{e}"
+      end
+    end
+
+    @lookup = [@override, config[@section] || {}, config['default'] || {}, DEFAULT]
+    cache_config
+  end
+
+  def method_missing(name)
+    fetch(name)
+  end
+
+  def validate
+    ['from', 'to'].each do |s|
+      raise Error, "'#{s}' must be a valid IMAP URI (e.g. imap://example.com)" unless fetch(s) =~ IMAP::REGEX_URI
+    end
+
+    unless Logger::LEVELS.has_key?(verbosity.to_sym)
+      raise Error, "'verbosity' must be one of: #{Logger::LEVELS.keys.join(', ')}"
+    end
+
+    if exclude_file
+      raise Error, "exclude file not found: #{exclude_file}" unless File.file?(exclude_file)
+      raise Error, "exclude file cannot be read: #{exclude_file}" unless File.readable?(exclude_file)
+    end
+  end
+
+  private
+
+  # Merges configs such that those earlier in the lookup chain override those
+  # later in the chain.
+  def cache_config
+    @cached = {}
+
+    @lookup.reverse.each do |c|
+      c.each {|k, v| @cached[k] = config_merge(@cached[k] || {}, v) }      
+    end
+  end
+
+  def config_merge(master, value)
+    if value.is_a?(Hash)
+      value.each {|k, v| master[k] = config_merge(master[k] || {}, v) }
+      return master
+    end
+
+    value
+  end
+
+end
+
+end