imap-backup 5.0.0 → 6.0.0.rc2

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 6.0.0.rc2
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-06 00:00:00.000000000 Z
+date: 2022-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -81,13 +81,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: aruba
+  name: thunderbird
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.0
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -95,19 +95,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.0
 - !ruby/object:Gem::Dependency
-  name: codeclimate-test-reporter
+  name: aruba
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.8
+        version: 0.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.8
+        version: 0.0.0
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -136,20 +136,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.0
-- !ruby/object:Gem::Dependency
-  name: rspec_junit_formatter
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -193,6 +179,7 @@ files:
 - docs/restore.md
 - docs/setup.md
 - imap-backup.gemspec
+- lib/cli_coverage.rb
 - lib/email/mboxrd/message.rb
 - lib/email/provider.rb
 - lib/email/provider/apple_mail.rb
@@ -224,9 +211,10 @@ files:
 - lib/imap/backup/migrator.rb
 - lib/imap/backup/sanitizer.rb
 - lib/imap/backup/serializer.rb
+- lib/imap/backup/serializer/directory.rb
+- lib/imap/backup/serializer/imap.rb
 - lib/imap/backup/serializer/mbox.rb
 - lib/imap/backup/serializer/mbox_enumerator.rb
-- lib/imap/backup/serializer/mbox_store.rb
 - lib/imap/backup/setup.rb
 - lib/imap/backup/setup/account.rb
 - lib/imap/backup/setup/asker.rb
@@ -238,13 +226,6 @@ files:
 - lib/imap/backup/utils.rb
 - lib/imap/backup/version.rb
 - lib/retry_on_error.rb
-- lib/thunderbird.rb
-- lib/thunderbird/install.rb
-- lib/thunderbird/local_folder.rb
-- lib/thunderbird/profile.rb
-- lib/thunderbird/profiles.rb
-- lib/thunderbird/subdirectory.rb
-- lib/thunderbird/subdirectory_placeholder.rb
 - spec/features/backup_spec.rb
 - spec/features/configuration/minimal_configuration.rb
 - spec/features/configuration/missing_configuration.rb
@@ -264,7 +245,6 @@ files:
 - spec/features/support/shared/connection_context.rb
 - spec/features/support/shared/message_fixtures.rb
 - spec/fixtures/connection.yml
-- spec/gather_rspec_coverage.rb
 - spec/spec_helper.rb
 - spec/support/fixtures.rb
 - spec/support/higline_test_helpers.rb
@@ -277,25 +257,39 @@ files:
 - spec/unit/email/provider_spec.rb
 - spec/unit/imap/backup/account/connection_spec.rb
 - spec/unit/imap/backup/account/folder_spec.rb
+- spec/unit/imap/backup/account_spec.rb
 - spec/unit/imap/backup/cli/accounts_spec.rb
+- spec/unit/imap/backup/cli/backup_spec.rb
+- spec/unit/imap/backup/cli/folders_spec.rb
 - spec/unit/imap/backup/cli/helpers_spec.rb
 - spec/unit/imap/backup/cli/local_spec.rb
+- spec/unit/imap/backup/cli/migrate_spec.rb
+- spec/unit/imap/backup/cli/restore_spec.rb
+- spec/unit/imap/backup/cli/setup_spec.rb
 - spec/unit/imap/backup/cli/utils_spec.rb
+- spec/unit/imap/backup/cli_spec.rb
+- spec/unit/imap/backup/client/apple_mail_spec.rb
 - spec/unit/imap/backup/client/default_spec.rb
 - spec/unit/imap/backup/configuration_spec.rb
 - spec/unit/imap/backup/downloader_spec.rb
 - spec/unit/imap/backup/logger_spec.rb
 - spec/unit/imap/backup/migrator_spec.rb
+- spec/unit/imap/backup/sanitizer_spec.rb
+- spec/unit/imap/backup/serializer/directory_spec.rb
+- spec/unit/imap/backup/serializer/imap_spec.rb
 - spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
 - spec/unit/imap/backup/serializer/mbox_spec.rb
-- spec/unit/imap/backup/serializer/mbox_store_spec.rb
+- spec/unit/imap/backup/serializer_spec.rb
 - spec/unit/imap/backup/setup/account_spec.rb
 - spec/unit/imap/backup/setup/asker_spec.rb
 - spec/unit/imap/backup/setup/connection_tester_spec.rb
 - spec/unit/imap/backup/setup/folder_chooser_spec.rb
+- spec/unit/imap/backup/setup/helpers_spec.rb
 - spec/unit/imap/backup/setup_spec.rb
+- spec/unit/imap/backup/thunderbird/mailbox_exporter_spec.rb
 - spec/unit/imap/backup/uploader_spec.rb
 - spec/unit/imap/backup/utils_spec.rb
+- spec/unit/retry_on_error_spec.rb
 homepage: https://github.com/joeyates/imap-backup
 licenses:
 - MIT
@@ -311,9 +305,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.6
 signing_key:
@@ -321,18 +315,27 @@ specification_version: 4
 summary: Backup GMail (or other IMAP) accounts to disk
 test_files:
 - spec/spec_helper.rb
+- spec/unit/imap/backup/account_spec.rb
+- spec/unit/imap/backup/serializer/imap_spec.rb
+- spec/unit/imap/backup/serializer/directory_spec.rb
 - spec/unit/imap/backup/serializer/mbox_spec.rb
-- spec/unit/imap/backup/serializer/mbox_store_spec.rb
 - spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
+- spec/unit/imap/backup/cli_spec.rb
 - spec/unit/imap/backup/setup_spec.rb
 - spec/unit/imap/backup/logger_spec.rb
 - spec/unit/imap/backup/configuration_spec.rb
 - spec/unit/imap/backup/setup/account_spec.rb
 - spec/unit/imap/backup/setup/connection_tester_spec.rb
 - spec/unit/imap/backup/setup/asker_spec.rb
+- spec/unit/imap/backup/setup/helpers_spec.rb
 - spec/unit/imap/backup/setup/folder_chooser_spec.rb
+- spec/unit/imap/backup/cli/folders_spec.rb
+- spec/unit/imap/backup/cli/setup_spec.rb
+- spec/unit/imap/backup/cli/backup_spec.rb
+- spec/unit/imap/backup/cli/restore_spec.rb
 - spec/unit/imap/backup/cli/helpers_spec.rb
 - spec/unit/imap/backup/cli/accounts_spec.rb
+- spec/unit/imap/backup/cli/migrate_spec.rb
 - spec/unit/imap/backup/cli/utils_spec.rb
 - spec/unit/imap/backup/cli/local_spec.rb
 - spec/unit/imap/backup/downloader_spec.rb
@@ -341,7 +344,12 @@ test_files:
 - spec/unit/imap/backup/account/folder_spec.rb
 - spec/unit/imap/backup/account/connection_spec.rb
 - spec/unit/imap/backup/utils_spec.rb
+- spec/unit/imap/backup/serializer_spec.rb
 - spec/unit/imap/backup/client/default_spec.rb
+- spec/unit/imap/backup/client/apple_mail_spec.rb
+- spec/unit/imap/backup/thunderbird/mailbox_exporter_spec.rb
+- spec/unit/imap/backup/sanitizer_spec.rb
+- spec/unit/retry_on_error_spec.rb
 - spec/unit/email/mboxrd/message_spec.rb
 - spec/unit/email/provider/gmail_spec.rb
 - spec/unit/email/provider/apple_mail_spec.rb
@@ -370,4 +378,3 @@ test_files:
 - spec/support/silence_logging.rb
 - spec/support/higline_test_helpers.rb
 - spec/fixtures/connection.yml
-- spec/gather_rspec_coverage.rb

data/lib/imap/backup/serializer/mbox_store.rb

@@ -1,217 +0,0 @@
-require "json"
-
-require "email/mboxrd/message"
-require "imap/backup/serializer/mbox_enumerator"
-
-module Imap::Backup
-  class Serializer::MboxStore
-    CURRENT_VERSION = 2
-
-    attr_reader :folder
-    attr_reader :path
-    attr_reader :loaded
-
-    def initialize(path, folder)
-      @path = path
-      @folder = folder
-      @loaded = false
-      @uids = nil
-      @uid_validity = nil
-    end
-
-    def exist?
-      mbox_exist? && imap_exist?
-    end
-
-    def uid_validity
-      do_load if !loaded
-      @uid_validity
-    end
-
-    def uid_validity=(value)
-      do_load if !loaded
-      @uid_validity = value
-      @uids ||= []
-      write_imap_file
-    end
-
-    def uids
-      do_load if !loaded
-      @uids || []
-    end
-
-    def add(uid, body)
-      do_load if !loaded
-      raise "Can't add messages without uid_validity" if !uid_validity
-
-      uid = uid.to_i
-      if uids.include?(uid)
-        Imap::Backup::Logger.logger.debug(
-          "[#{folder}] message #{uid} already downloaded - skipping"
-        )
-        return
-      end
-
-      mboxrd_message = Email::Mboxrd::Message.new(body)
-      mbox = nil
-      begin
-        mbox = File.open(mbox_pathname, "ab")
-        mbox.write mboxrd_message.to_serialized
-        @uids << uid
-        write_imap_file
-      rescue StandardError => e
-        message = <<-ERROR.gsub(/^\s*/m, "")
-          [#{folder}] failed to save message #{uid}:
-          #{body}. #{e}:
-          #{e.backtrace.join("\n")}"
-        ERROR
-        Imap::Backup::Logger.logger.warn message
-      ensure
-        mbox&.close
-      end
-    end
-
-    def load(uid_maybe_string)
-      do_load if !loaded
-      uid = uid_maybe_string.to_i
-      message_index = uids.find_index(uid)
-      return nil if message_index.nil?
-
-      load_nth(message_index)
-    end
-
-    def each_message(required_uids)
-      return enum_for(:each_message, required_uids) if !block_given?
-
-      indexes = required_uids.each.with_object({}) do |uid_maybe_string, acc|
-        uid = uid_maybe_string.to_i
-        index = uids.find_index(uid)
-        acc[index] = uid if index
-      end
-      enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
-      enumerator.each.with_index do |raw, i|
-        uid = indexes[i]
-        next if !uid
-
-        yield uid, Email::Mboxrd::Message.from_serialized(raw)
-      end
-    end
-
-    def update_uid(old, new)
-      index = uids.find_index(old.to_i)
-      return if index.nil?
-
-      uids[index] = new.to_i
-      write_imap_file
-    end
-
-    def reset
-      @uids = nil
-      @uid_validity = nil
-      @loaded = false
-      delete_files
-      write_blank_mbox_file
-    end
-
-    def rename(new_name)
-      new_mbox_pathname = absolute_path("#{new_name}.mbox")
-      new_imap_pathname = absolute_path("#{new_name}.imap")
-      File.rename(mbox_pathname, new_mbox_pathname)
-      File.rename(imap_pathname, new_imap_pathname)
-      @folder = new_name
-    end
-
-    def mbox_pathname
-      absolute_path("#{folder}.mbox")
-    end
-
-    def imap_pathname
-      absolute_path("#{folder}.imap")
-    end
-
-    private
-
-    def do_load
-      data = imap_data
-      if data
-        @uids = data[:uids].map(&:to_i)
-        @uid_validity = data[:uid_validity]
-        @loaded = true
-      else
-        reset
-      end
-    end
-
-    def imap_data
-      return nil if !imap_ok?
-
-      imap_data = nil
-
-      begin
-        imap_data = JSON.parse(File.read(imap_pathname), symbolize_names: true)
-      rescue JSON::ParserError
-        return nil
-      end
-
-      return nil if !imap_data.key?(:uids)
-      return nil if !imap_data[:uids].is_a?(Array)
-
-      imap_data
-    end
-
-    def imap_ok?
-      return false if !exist?
-      return false if !imap_looks_like_json?
-
-      true
-    end
-
-    def load_nth(index)
-      enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
-      enumerator.each.with_index do |raw, i|
-        next if i != index
-
-        return Email::Mboxrd::Message.from_serialized(raw)
-      end
-      nil
-    end
-
-    def imap_looks_like_json?
-      return false if !imap_exist?
-
-      content = File.read(imap_pathname)
-      content.start_with?("{")
-    end
-
-    def write_imap_file
-      imap_data = {
-        version: CURRENT_VERSION,
-        uid_validity: @uid_validity,
-        uids: @uids
-      }
-      content = imap_data.to_json
-      File.open(imap_pathname, "w") { |f| f.write content }
-    end
-
-    def write_blank_mbox_file
-      File.open(mbox_pathname, "w") { |f| f.write "" }
-    end
-
-    def delete_files
-      File.unlink(imap_pathname) if imap_exist?
-      File.unlink(mbox_pathname) if mbox_exist?
-    end
-
-    def mbox_exist?
-      File.exist?(mbox_pathname)
-    end
-
-    def imap_exist?
-      File.exist?(imap_pathname)
-    end
-
-    def absolute_path(relative_path)
-      File.join(path, relative_path)
-    end
-  end
-end

data/lib/thunderbird/install.rb

@@ -1,16 +0,0 @@
-require "thunderbird/profiles"
-
-class Thunderbird::Install
-  attr_reader :title
-  attr_reader :entries
-
-  # entries are lines from profile.ini
-  def initialize(title, entries)
-    @title = title
-    @entries = entries
-  end
-
-  def default
-    Thunderbird::Profiles.new.profile_for_path(entries[:Default])
-  end
-end

data/lib/thunderbird/local_folder.rb

@@ -1,65 +0,0 @@
-require "thunderbird/profile"
-require "thunderbird/subdirectory"
-
-# A local folder is a file containing emails
-class Thunderbird::LocalFolder
-  attr_reader :path
-  attr_reader :profile
-
-  def initialize(profile, path)
-    @profile = profile
-    @path = path
-  end
-
-  def set_up
-    return if path_elements.empty?
-
-    return true if !in_subdirectory?
-
-    subdirectory.set_up
-  end
-
-  def full_path
-    if in_subdirectory?
-      File.join(subdirectory.full_path, folder_name)
-    else
-      folder_name
-    end
-  end
-
-  def exists?
-    File.exist?(full_path)
-  end
-
-  def msf_path
-    "#{path}.msf"
-  end
-
-  def msf_exists?
-    File.exist?(msf_path)
-  end
-
-  private
-
-  def in_subdirectory?
-    path_elements.count > 1
-  end
-
-  def subdirectory
-    return nil if !in_subdirectory?
-
-    Thunderbird::Subdirectory.new(profile, subdirectory_path)
-  end
-
-  def path_elements
-    path.split(File::SEPARATOR)
-  end
-
-  def subdirectory_path
-    File.join(path_elements[0..-2])
-  end
-
-  def folder_name
-    path_elements[-1]
-  end
-end

data/lib/thunderbird/profile.rb

@@ -1,30 +0,0 @@
-require "thunderbird"
-
-class Thunderbird::Profile
-  attr_reader :title
-  attr_reader :entries
-
-  # entries are lines from profile.ini
-  def initialize(title, entries)
-    @title = title
-    @entries = entries
-  end
-
-  def root
-    if relative?
-      File.join(Thunderbird.new.data_path, entries[:Path])
-    else
-      entries[:Path]
-    end
-  end
-
-  def local_folders_path
-    File.join(root, "Mail", "Local Folders")
-  end
-
-  private
-
-  def relative?
-    entries[:IsRelative] == "1"
-  end
-end

data/lib/thunderbird/profiles.rb

@@ -1,71 +0,0 @@
-require "thunderbird"
-require "thunderbird/install"
-require "thunderbird/profile"
-
-# http://kb.mozillazine.org/Profiles.ini_file
-class Thunderbird::Profiles
-  def profile_for_path(path)
-    title, entries = blocks.find { |_name, entries| entries[:Path] == path }
-
-    Thunderbird::Profile.new(title, entries) if title
-  end
-
-  def profile(name)
-    title, entries = blocks.find { |_name, entries| entries[:Name] == name }
-
-    Thunderbird::Profile.new(title, entries) if title
-  end
-
-  def installs
-    @installs ||= begin
-      pairs = blocks.filter { |name, _entries| name.start_with?("Install") }
-      pairs.map { |title, entries| Thunderbird::Install.new(title, entries) }
-    end
-  end
-
-  private
-
-  # Parse profiles.ini.
-  # Blocks start with a title, e.g. '[Abc]'
-  # and are followed by a number of lines
-  def blocks
-    @blocks ||= begin
-      blocks = {}
-      File.open(profiles_ini_path, "rb") do |f|
-        title = nil
-        entries = nil
-
-        loop do
-          line = f.gets
-          if !line
-            blocks[title] = entries if title
-            break
-          end
-
-          line.chomp!
-
-          # Is this line the start of a new block
-          match = line.match(/\A\[([A-Za-z0-9]+)\]\z/)
-          if match
-            # Store what we got before this title as a new block
-            blocks[title] = entries if title
-
-            # Start a new block
-            title = match[1]
-            entries = {}
-          elsif line != ""
-            # Collect entries until we get to the next title
-            key, value = line.split("=")
-            entries[key.to_sym] = value
-          end
-        end
-      end
-
-      blocks
-    end
-  end
-
-  def profiles_ini_path
-    File.join(Thunderbird.new.data_path, "profiles.ini")
-  end
-end

data/lib/thunderbird/subdirectory.rb

@@ -1,93 +0,0 @@
-require "thunderbird/subdirectory_placeholder"
-
-class Thunderbird::Subdirectory
-  # `path` is the UI path, it doesn't have the '.sbd' extensions
-  # that are present in the real, file system path
-  attr_reader :path
-  attr_reader :profile
-
-  def initialize(profile, path)
-    @profile = profile
-    @path = path
-  end
-
-  def set_up
-    raise "Cannot create a subdirectory without a path" if !sub_directory?
-
-    if sub_sub_directory?
-      parent_ok = parent.set_up
-      return false if !parent_ok
-    end
-
-    ok = check
-    return false if !ok
-
-    FileUtils.mkdir_p full_path
-    placeholder.touch
-
-    true
-  end
-
-  # subdirectory relative path is 'Foo.sbd/Bar.sbd/Baz.sbd'
-  def full_path
-    relative_path = File.join(subdirectories)
-    File.join(profile.local_folders_path, relative_path)
-  end
-
-  private
-
-  def sub_directory?
-    path_elements.any?
-  end
-
-  def sub_sub_directory?
-    path_elements.count > 1
-  end
-
-  def parent
-    return nil if !sub_sub_directory?
-
-    self.class.new(profile, File.join(path_elements[0..-2]))
-  end
-
-  # placeholder relative path is 'Foo.sbd/Bar.sbd/Baz'
-  def placeholder
-    @placeholder = begin
-      relative_path = File.join(subdirectories[0..-2], path_elements[-1])
-      path = File.join(profile.local_folders_path, relative_path)
-      Thunderbird::SubdirectoryPlaceholder.new(path)
-    end
-  end
-
-  def path_elements
-    path.split(File::SEPARATOR)
-  end
-
-  def exists?
-    File.exist?(full_path)
-  end
-
-  def directory?
-    File.directory?(full_path)
-  end
-
-  def subdirectories
-    path_elements.map { |p| "#{p}.sbd" }
-  end
-
-  def check
-    case
-    when exists? && !placeholder.exists?
-      Kernel.puts "Can't set up folder '#{full_path}': '#{full_path}' exists, but '#{placeholder.path}' is missing"
-      false
-    when placeholder.exists? && !placeholder.regular?
-      Kernel.puts "Can't set up folder '#{full_path}': '#{placeholder.path}' exists, but it is not a regular file"
-      false
-    when exists? && !directory?
-      Kernel.puts "Can't set up folder '#{full_path}': '#{full_path}' exists, but it is not a directory"
-      false
-    else
-      true
-    end
-  end
-end