imap-backup 5.0.0 → 6.0.0.rc2

data/lib/imap/backup/serializer/imap.rb

@@ -0,0 +1,120 @@
+require "json"
+
+module Imap::Backup
+  class Serializer::Imap
+    CURRENT_VERSION = 2
+
+    attr_reader :folder_path
+    attr_reader :loaded
+
+    def initialize(folder_path)
+      @folder_path = folder_path
+      @loaded = false
+      @uid_validity = nil
+      @uids = nil
+    end
+
+    def append(uid)
+      uids << uid
+      save
+    end
+
+    def exist?
+      File.exist?(pathname)
+    end
+
+    def include?(uid)
+      uids.include?(uid)
+    end
+
+    def index(uid)
+      uids.find_index(uid)
+    end
+
+    def rename(new_path)
+      if exist?
+        old_pathname = pathname
+        @folder_path = new_path
+        File.rename(old_pathname, pathname)
+      else
+        @folder_path = new_path
+      end
+    end
+
+    def uid_validity
+      ensure_loaded
+      @uid_validity
+    end
+
+    def uid_validity=(value)
+      ensure_loaded
+      @uid_validity = value
+      @uids ||= []
+      save
+    end
+
+    def uids
+      ensure_loaded
+      @uids || []
+    end
+
+    def update_uid(old, new)
+      index = uids.find_index(old.to_i)
+      return if index.nil?
+
+      uids[index] = new.to_i
+      save
+    end
+
+    private
+
+    def pathname
+      "#{folder_path}.imap"
+    end
+
+    def ensure_loaded
+      return if loaded
+
+      data = load
+      if data
+        @uids = data[:uids].map(&:to_i)
+        @uid_validity = data[:uid_validity]
+      else
+        @uids = []
+        @uid_validity = nil
+      end
+      @loaded = true
+    end
+
+    def load
+      return nil if !exist?
+
+      data = nil
+      begin
+        content = File.read(pathname)
+        data = JSON.parse(content, symbolize_names: true)
+      rescue JSON::ParserError
+        return nil
+      end
+
+      return nil if !data.key?(:uids)
+      return nil if !data[:uids].is_a?(Array)
+
+      data
+    end
+
+    def save
+      ensure_loaded
+
+      raise "Cannot save metadata without a uid_validity" if !uid_validity
+
+      data = {
+        version: CURRENT_VERSION,
+        uid_validity: @uid_validity,
+        uids: @uids
+      }
+      content = data.to_json
+      File.open(pathname, "w") { |f| f.write content }
+    end
+  end
+end

data/lib/imap/backup/serializer/mbox.rb

@@ -1,115 +1,44 @@
-require "forwardable"
-
-require "imap/backup/serializer/mbox_store"
-
 module Imap::Backup
   class Serializer::Mbox
-    extend Forwardable
-    def_delegators :store, :mbox_pathname
+    attr_reader :folder_path
 
-    attr_reader :path
-    attr_reader :folder
-
-    def initialize(path, folder)
-      @path = path
-      @folder = folder
+    def initialize(folder_path)
+      @folder_path = folder_path
     end
 
-    def apply_uid_validity(value)
-      case
-      when store.uid_validity.nil?
-        store.uid_validity = value
-        nil
-      when store.uid_validity == value
-        # NOOP
-        nil
-      else
-        apply_new_uid_validity value
+    def append(message)
+      File.open(pathname, "ab") do |file|
+        file.write message
       end
     end
 
-    def force_uid_validity(value)
-      store.uid_validity = value
+    def exist?
+      File.exist?(pathname)
     end
 
-    def uids
-      store.uids
-    end
-
-    def load(uid)
-      store.load(uid)
-    end
-
-    def each_message(uids)
-      store.each_message(uids)
-    end
-
-    def save(uid, message)
-      store.add(uid, message)
-    end
-
-    def rename(new_name)
-      @folder = new_name
-      store.rename new_name
-    end
+    def length
+      return nil if !exist?
 
-    def update_uid(old, new)
-      store.update_uid old, new
+      File.stat(pathname).size
     end
 
-    private
-
-    def store
-      @store ||=
-        begin
-          create_containing_directory
-          Serializer::MboxStore.new(path, folder)
-        end
+    def pathname
+      "#{folder_path}.mbox"
     end
 
-    def apply_new_uid_validity(value)
-      digit = 0
-      new_name = nil
-      loop do
-        extra = digit.zero? ? "" : ".#{digit}"
-        new_name = "#{folder}.#{store.uid_validity}#{extra}"
-        test_store = Serializer::MboxStore.new(path, new_name)
-        break if !test_store.exist?
-
-        digit += 1
+    def rename(new_path)
+      if exist?
+        old_pathname = pathname
+        @folder_path = new_path
+        File.rename(old_pathname, pathname)
+      else
+        @folder_path = new_path
       end
-      rename_store new_name, value
-    end
-
-    def rename_store(new_name, value)
-      store.rename new_name
-      @store = nil
-      store.uid_validity = value
-      new_name
-    end
-
-    def relative_path
-      File.dirname(folder)
     end
 
-    def containing_directory
-      File.join(path, relative_path)
-    end
-
-    def full_path
-      File.expand_path(containing_directory)
-    end
-
-    def create_containing_directory
-      if !File.directory?(full_path)
-        Utils.make_folder(
-          path, relative_path, Serializer::DIRECTORY_PERMISSIONS
-        )
-      end
-
-      if Utils.mode(full_path) !=
-         Serializer::DIRECTORY_PERMISSIONS
-        FileUtils.chmod Serializer::DIRECTORY_PERMISSIONS, full_path
+    def rewind(length)
+      File.open(pathname, File::RDWR | File::CREAT, 0o644) do |f|
+        f.truncate(length)
       end
     end
   end

data/lib/imap/backup/serializer/mbox_enumerator.rb

@@ -1,4 +1,6 @@
 module Imap::Backup
+  class Serializer; end
+
   class Serializer::MboxEnumerator
     attr_reader :mbox_pathname
 

data/lib/imap/backup/serializer.rb

@@ -1,6 +1,183 @@
+require "forwardable"
+
+require "email/mboxrd/message"
+require "imap/backup/serializer/imap"
+require "imap/backup/serializer/mbox"
+require "imap/backup/serializer/mbox_enumerator"
+
 module Imap::Backup
-  module Serializer
-    DIRECTORY_PERMISSIONS = 0o700
-    FILE_PERMISSIONS      = 0o600
+  class Serializer
+    extend Forwardable
+
+    def_delegator :mbox, :pathname, :mbox_pathname
+    def_delegators :imap, :uid_validity, :uids, :update_uid
+
+    attr_reader :folder
+    attr_reader :path
+
+    def initialize(path, folder)
+      @path = path
+      @folder = folder
+    end
+
+    def apply_uid_validity(value)
+      case
+      when uid_validity.nil?
+        imap.uid_validity = value
+        nil
+      when uid_validity == value
+        # NOOP
+        nil
+      else
+        apply_new_uid_validity value
+      end
+    end
+
+    def force_uid_validity(value)
+      imap.uid_validity = value
+    end
+
+    def append(uid, message)
+      raise "Can't add messages without uid_validity" if !imap.uid_validity
+
+      uid = uid.to_i
+      if imap.include?(uid)
+        Logger.logger.debug(
+          "[#{folder}] message #{uid} already downloaded - skipping"
+        )
+        return
+      end
+
+      do_append uid, message
+    end
+
+    def load(uid_maybe_string)
+      uid = uid_maybe_string.to_i
+      message_index = imap.index(uid)
+      return nil if message_index.nil?
+
+      load_nth(message_index)
+    end
+
+    def load_nth(index)
+      enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
+      enumerator.each.with_index do |raw, i|
+        next if i != index
+
+        return Email::Mboxrd::Message.from_serialized(raw)
+      end
+      nil
+    end
+
+    def each_message(required_uids)
+      return enum_for(:each_message, required_uids) if !block_given?
+
+      indexes = required_uids.each.with_object({}) do |uid_maybe_string, acc|
+        uid = uid_maybe_string.to_i
+        index = imap.index(uid)
+        acc[index] = uid if index
+      end
+      enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
+      enumerator.each.with_index do |raw, i|
+        uid = indexes[i]
+        next if !uid
+
+        yield uid, Email::Mboxrd::Message.from_serialized(raw)
+      end
+    end
+
+    def rename(new_name)
+      # Initialize so we get memoized instances with the correct folder_path
+      mbox
+      imap
+      @folder = new_name
+      ensure_containing_directory
+      mbox.rename folder_path
+      imap.rename folder_path
+    end
+
+    private
+
+    def do_append(uid, message)
+      mboxrd_message = Email::Mboxrd::Message.new(message)
+      initial = mbox.length || 0
+      mbox_appended = false
+      begin
+        mbox.append mboxrd_message.to_serialized
+        mbox_appended = true
+        imap.append uid
+      rescue StandardError => e
+        mbox.rewind(initial) if mbox_appended
+
+        message = <<-ERROR.gsub(/^\s*/m, "")
+          [#{folder}] failed to append message #{uid}:
+          #{message}. #{e}:
+          #{e.backtrace.join("\n")}"
+        ERROR
+        Logger.logger.warn message
+      end
+    end
+
+    def mbox
+      @mbox ||=
+        begin
+          ensure_containing_directory
+          Serializer::Mbox.new(folder_path)
+        end
+    end
+
+    def imap
+      @imap ||=
+        begin
+          ensure_containing_directory
+          Serializer::Imap.new(folder_path)
+        end
+    end
+
+    def folder_path
+      folder_path_for(path, folder)
+    end
+
+    def folder_path_for(path, folder)
+      relative = File.join(path, folder)
+      File.expand_path(relative)
+    end
+
+    def ensure_containing_directory
+      relative = File.dirname(folder)
+      directory = Serializer::Directory.new(path, relative)
+      directory.ensure_exists
+    end
+
+    def apply_new_uid_validity(value)
+      new_name = rename_existing_folder
+      # Clear memoization so we get empty data
+      @mbox = nil
+      @imap = nil
+      imap.uid_validity = value
+
+      new_name
+    end
+
+    def rename_existing_folder
+      digit = 0
+      new_name = nil
+      loop do
+        extra = digit.zero? ? "" : "-#{digit}"
+        new_name = "#{folder}-#{imap.uid_validity}#{extra}"
+        new_folder_path = folder_path_for(path, new_name)
+        test_mbox = Serializer::Mbox.new(new_folder_path)
+        test_imap = Serializer::Imap.new(new_folder_path)
+        break if !test_mbox.exist? && !test_imap.exist?
+
+        digit += 1
+      end
+
+      previous = folder
+      rename(new_name)
+      @folder = previous
+
+      new_name
+    end
   end
 end

data/lib/imap/backup/setup/account.rb

@@ -26,6 +26,7 @@ module Imap::Backup
         modify_password menu
         modify_backup_path menu
         choose_folders menu
+        modify_multi_fetch_size menu
         modify_server menu
         modify_connection_options menu
         test_connection menu
@@ -37,21 +38,30 @@ module Imap::Backup
 
     def header(menu)
       modified = account.modified? ? "*" : ""
-      connection_options =
-        if account.connection_options
-          escaped =
-            JSON.generate(account.connection_options).
-            gsub('"', '\"')
-          "\n  connection options #{escaped}"
-        end
+
+      if account.multi_fetch_size > 1
+        multi_fetch_size = "\nmulti-fetch #{account.multi_fetch_size}"
+      end
+
+      if account.connection_options
+        escaped =
+          JSON.generate(account.connection_options)
+        connection_options =
+          "\nconnection options  '#{escaped}'"
+        space = " " * 12
+      else
+        connection_options = nil
+        space = " " * 4
+      end
+
       menu.header = <<~HEADER.chomp
         #{helpers.title_prefix} Account#{modified}
 
-        email      #{account.username}
-        password   #{masked_password}
-        path       #{account.local_path}
-        folders    #{folders.map { |f| f[:name] }.join(', ')}
-        server     #{account.server}#{connection_options}
+        email   #{space}#{account.username}
+        password#{space}#{masked_password}
+        path    #{space}#{account.local_path}
+        folders #{space}#{folders.map { |f| f[:name] }.join(', ')}#{multi_fetch_size}
+        server  #{space}#{account.server}#{connection_options}
 
         Choose an action
       HEADER
@@ -70,12 +80,10 @@ module Imap::Backup
           )
         else
           account.username = username
-          # rubocop:disable Style/IfUnlessModifier
           default = default_server(username)
           if default && (account.server.nil? || (account.server == ""))
             account.server = default
           end
-          # rubocop:enable Style/IfUnlessModifier
         end
       end
     end
@@ -88,20 +96,6 @@ module Imap::Backup
       end
     end
 
-    def modify_server(menu)
-      menu.choice("modify server") do
-        server = highline.ask("server: ")
-        account.server = server if !server.nil?
-      end
-    end
-
-    def modify_connection_options(menu)
-      menu.choice("modify connection options") do
-        connection_options = highline.ask("connections options (as JSON): ")
-        account.connection_options = connection_options if !connection_options.nil?
-      end
-    end
-
     def path_modification_validator(path)
       same = config.accounts.find do |a|
         a.username != account.username && a.local_path == path
@@ -130,6 +124,35 @@ module Imap::Backup
       end
     end
 
+    def modify_multi_fetch_size(menu)
+      menu.choice("modify multi-fetch size (number of emails to fetch at a time)") do
+        size = highline.ask("size: ")
+        int = size.to_i
+        account.multi_fetch_size = int if int.positive?
+      end
+    end
+
+    def modify_server(menu)
+      menu.choice("modify server") do
+        server = highline.ask("server: ")
+        account.server = server if !server.nil?
+      end
+    end
+
+    def modify_connection_options(menu)
+      menu.choice("modify connection options") do
+        connection_options = highline.ask("connections options (as JSON): ")
+        if !connection_options.nil?
+          begin
+            account.connection_options = connection_options
+          rescue JSON::ParserError
+            Kernel.puts "Malformed JSON, please try again"
+            highline.ask "Press a key "
+          end
+        end
+      end
+    end
+
     def test_connection(menu)
       menu.choice("test connection") do
         result = Setup::ConnectionTester.new(account).test
@@ -141,7 +164,7 @@ module Imap::Backup
     def delete_account(menu)
       menu.choice("delete") do
         if highline.agree("Are you sure? (y/n) ")
-          account.mark_for_deletion!
+          account.mark_for_deletion
           throw :done
         end
       end

data/lib/imap/backup/setup/helpers.rb

@@ -9,7 +9,7 @@ module Imap::Backup
     end
 
     def version
-      Version::VERSION
+      VERSION
     end
   end
 end

data/lib/imap/backup/thunderbird/mailbox_exporter.rb

@@ -19,7 +19,7 @@ module Imap::Backup
 
     def run
       local_folder_ok = local_folder.set_up
-      return if !local_folder_ok
+      return false if !local_folder_ok
 
       if local_folder.msf_exists?
         if force

data/lib/imap/backup/version.rb

@@ -1,9 +1,9 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 5
+  MAJOR    = 6
   MINOR    = 0
   REVISION = 0
-  PRE      = nil
+  PRE      = "rc2".freeze
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

data/lib/imap/backup.rb

@@ -8,7 +8,6 @@ require "imap/backup/downloader"
 require "imap/backup/logger"
 require "imap/backup/uploader"
 require "imap/backup/serializer"
-require "imap/backup/serializer/mbox"
 require "imap/backup/setup"
 require "imap/backup/setup/account"
 require "imap/backup/setup/asker"