imap-backup 5.0.0 → 6.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16911b1cea1983b844c3bbf7f74c03ff7436990fb5aa4d5b49fa18f03ae2e229
-  data.tar.gz: 265f63995e70ba3f68b98ec72d3a12b3e95100f7e6b320026b18bcc156888a92
+  metadata.gz: 4f9cac7966fac9858fdc380d8e9638c8d479035ee4053444a104b8c3eb5a304c
+  data.tar.gz: 5d9785b5e34d90535c8052d216d9b3dbf46ea743672f05129604729662f42f34
 SHA512:
-  metadata.gz: 4a541a7830ddab458f2366bdad4924f3e6cbc3ba858651839e6dcd1a530acbbeb01e2c59a44331af73a2d77f606f2d6b39eacd2b5a705d78ec20e93d2c2f2527
-  data.tar.gz: 7b60170e3d29e9882f3bcbf13b04daa162434944d59311a5a03a4b1c1022a7a9501b292c3eb929d6e07039c47cc433316a338e062ced4e8a6b7ea424e6c0678b
+  metadata.gz: a4595e4316dcdc20123a8ed2bec524d37c1de95da49281a30484349c317a996cf3c6946617a489880a0c41db41698b2ee862d9aee4f9c9b8fa34d67707ba924c
+  data.tar.gz: ccee268de9cb6225c72fe0cb11a2552fc793f664429d8dfa918e5b4697961c8e5e15e47ed653dac241f377385806443f17683614b8ff76075a8ff2f12e07f98c

data/README.md

@@ -1,6 +1,7 @@
-[![Build Status](https://circleci.com/gh/joeyates/imap-backup.svg?style=svg)][Continuous Integration]
-[![Source Analysis](https://codeclimate.com/github/joeyates/imap-backup/badges/gpa.svg)](https://codeclimate.com/github/joeyates/imap-backup)
-[![Test Coverage](https://codeclimate.com/github/joeyates/imap-backup/badges/coverage.svg)](https://codeclimate.com/github/joeyates/imap-backup/coverage)
+![Version](https://img.shields.io/gem/v/imap-backup?label=Version&logo=rubygems)
+[![Build Status](https://github.com/joeyates/imap-backup/actions/workflows/main.yml/badge.svg)][CI Status]
+![Coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/joeyates/b54fe758bfb405c04bef72dad293d707/raw/coverage.json)
+![License](https://img.shields.io/github/license/joeyates/imap-backup?color=brightgreen&label=License)
 
 # imap-backup
 
@@ -9,12 +10,12 @@
   * [Source Code]
   * [API documentation]
   * [Rubygem]
-  * [Continuous Integration]
+  * [CI Status]
 
 [Source Code]: https://github.com/joeyates/imap-backup "Source code at GitHub"
-[API documentation]: http://rubydoc.info/gems/imap-backup/frames "RDoc API Documentation at Rubydoc.info"
-[Rubygem]: http://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
-[Continuous Integration]: https://circleci.com/gh/joeyates/imap-backup "Build status by CirceCI"
+[API documentation]: https://rubydoc.info/gems/imap-backup/frames "RDoc API Documentation at Rubydoc.info"
+[Rubygem]: https://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
+[CI Status]: https://github.com/joeyates/imap-backup/actions/workflows/main.yml
 
 # Installation
 

data/bin/imap-backup

@@ -1,6 +1,10 @@
 #!/usr/bin/env ruby
 
 $LOAD_PATH.unshift(File.expand_path("../lib/", __dir__))
+
+require "cli_coverage"
+CliCoverage.conditionally_activate
+
 require "imap/backup/cli"
 require "imap/backup/logger"
 

data/docs/development.md

@@ -1,10 +1,10 @@
 # Testing
 
-## Integration Tests
+## Feature Specs
 
-Integration tests (feature specs) are run against a local IMAP server
-controlled by Docker Compose, which needs to be started
-before running the test suite.
+Specs under `specs/features` are integration specs run against a local IMAP server
+controlled by Docker Compose.
+Before running the test suite, it needs to be started:
 
 ```sh
 $ docker-compose up -d
@@ -26,6 +26,12 @@ or
 $ rspec --tag ~docker
 ```
 
+### Debugging
+
+The feature specs are run 'out of process' via the Aruba gem.
+In order to see debugging output from the process,
+use `last_command_started.output`.
+
 ## Access Docker imap server
 
 ```ruby

data/imap-backup.gemspec

@@ -27,16 +27,11 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "os"
   gem.add_runtime_dependency "rake"
   gem.add_runtime_dependency "thor", "~> 1.1"
+  gem.add_runtime_dependency "thunderbird", ">= 0.0.0"
 
   gem.add_development_dependency "aruba", ">= 0.0.0"
-  gem.add_development_dependency "codeclimate-test-reporter", "~> 0.4.8"
-  if RUBY_ENGINE == "jruby"
-    gem.add_development_dependency "pry-debugger-jruby"
-  else
-    gem.add_development_dependency "pry-byebug"
-  end
+  gem.add_development_dependency "pry-byebug"
   gem.add_development_dependency "rspec", ">= 3.0.0"
-  gem.add_development_dependency "rspec_junit_formatter"
   gem.add_development_dependency "rubocop-rspec"
   gem.add_development_dependency "simplecov"
 end

data/lib/cli_coverage.rb

@@ -0,0 +1,18 @@
+class CliCoverage
+  def self.conditionally_activate
+    if ENV["COVERAGE"]
+      require "simplecov"
+
+      # Collect coverage separately
+      SimpleCov.command_name "#{ENV['COVERAGE']} #{ARGV.join(' ')} coverage"
+
+      # Silence output
+      SimpleCov.formatter = SimpleCov::Formatter::SimpleFormatter
+      SimpleCov.print_error_status = false
+
+      # Ensure SimpleCov doesn't filter out all out code
+      project_root = File.expand_path("..", __dir__)
+      SimpleCov.root project_root
+    end
+  end
+end

data/lib/imap/backup/account/connection.rb

@@ -1,5 +1,6 @@
 require "imap/backup/client/apple_mail"
 require "imap/backup/client/default"
+require "imap/backup/serializer/directory"
 
 require "retry_on_error"
 
@@ -52,7 +53,7 @@ module Imap::Backup
     def status
       ensure_account_folder
       backup_folders.map do |folder|
-        s = Serializer::Mbox.new(account.local_path, folder.name)
+        s = Serializer.new(account.local_path, folder.name)
         {name: folder.name, local: s.uids, remote: folder.uids}
       end
     end
@@ -69,7 +70,7 @@ module Imap::Backup
         serializer.apply_uid_validity(folder.uid_validity)
         begin
           Downloader.new(
-            folder, serializer, block_size: config.download_block_size
+            folder, serializer, multi_fetch_size: account.multi_fetch_size
           ).run
         rescue Net::IMAP::ByeResponseError
           reconnect
@@ -86,7 +87,7 @@ module Imap::Backup
       base = Pathname.new(account.local_path)
       Pathname.glob(glob) do |path|
         name = path.relative_path_from(base).to_s[0..-6]
-        serializer = Serializer::Mbox.new(account.local_path, name)
+        serializer = Serializer.new(account.local_path, name)
         folder = Account::Folder.new(self, name)
         yield serializer, folder
       end
@@ -110,7 +111,6 @@ module Imap::Backup
     def reset
       @backup_folders = nil
       @client = nil
-      @config = nil
       @folder_names = nil
       @provider = nil
       @server = nil
@@ -144,7 +144,7 @@ module Imap::Backup
 
     def each_folder
       backup_folders.each do |folder|
-        serializer = Serializer::Mbox.new(account.local_path, folder.name)
+        serializer = Serializer.new(account.local_path, folder.name)
         yield folder, serializer
       end
     end
@@ -161,7 +161,7 @@ module Imap::Backup
           Imap::Backup::Logger.logger.debug(
             "Backup '#{old_name}' renamed and restored to '#{new_name}'"
           )
-          new_serializer = Serializer::Mbox.new(account.local_path, new_name)
+          new_serializer = Serializer.new(account.local_path, new_name)
           new_folder = Account::Folder.new(self, new_name)
           new_folder.create
           new_serializer.force_uid_validity(new_folder.uid_validity)
@@ -180,7 +180,7 @@ module Imap::Backup
       Utils.make_folder(
         File.dirname(account.local_path),
         File.basename(account.local_path),
-        Serializer::DIRECTORY_PERMISSIONS
+        Serializer::Directory::DIRECTORY_PERMISSIONS
       )
     end
 
@@ -195,9 +195,5 @@ module Imap::Backup
     def provider_options
       provider.options.merge(account.connection_options || {})
     end
-
-    def config
-      @config ||= Configuration.new
-    end
   end
 end

data/lib/imap/backup/account/folder.rb

@@ -68,22 +68,6 @@ module Imap::Backup
       []
     end
 
-    def fetch(uid)
-      examine
-      fetch_data_items =
-        retry_on_error(errors: UID_FETCH_RETRY_CLASSES) do
-          client.uid_fetch([uid.to_i], [BODY_ATTRIBUTE])
-        end
-      return nil if fetch_data_items.nil?
-
-      fetch_data_item = fetch_data_items[0]
-      attributes = fetch_data_item.attr
-
-      attributes[BODY_ATTRIBUTE]
-    rescue FolderNotFound
-      nil
-    end
-
     def fetch_multi(uids)
       examine
       fetch_data_items =

data/lib/imap/backup/account.rb

@@ -1,5 +1,7 @@
 module Imap::Backup
   class Account
+    DEFAULT_MULTI_FETCH_SIZE = 1
+
     attr_reader :username
     attr_reader :password
     attr_reader :local_path
@@ -7,7 +9,6 @@ module Imap::Backup
     attr_reader :server
     attr_reader :connection_options
     attr_reader :changes
-    attr_reader :marked_for_deletion
 
     def initialize(options)
       @username = options[:username]
@@ -16,6 +17,7 @@ module Imap::Backup
       @folders = options[:folders]
       @server = options[:server]
       @connection_options = options[:connection_options]
+      @multi_fetch_size = options[:multi_fetch_size]
       @changes = {}
       @marked_for_deletion = false
     end
@@ -25,18 +27,18 @@ module Imap::Backup
     end
 
     def valid?
-      username && password
+      username && password ? true : false
     end
 
     def modified?
       changes.any?
     end
 
-    def clear_changes!
+    def clear_changes
       @changes = {}
     end
 
-    def mark_for_deletion!
+    def mark_for_deletion
       @marked_for_deletion = true
     end
 
@@ -53,6 +55,7 @@ module Imap::Backup
       h[:folders] = @folders if @folders
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
+      h[:multi_fetch_size] = multi_fetch_size if @multi_fetch_size
       h
     end
 
@@ -82,20 +85,37 @@ module Imap::Backup
       update(:connection_options, parsed)
     end
 
+    def multi_fetch_size
+      int = @multi_fetch_size.to_i
+      if int.positive?
+        int
+      else
+        DEFAULT_MULTI_FETCH_SIZE
+      end
+    end
+
+    def multi_fetch_size=(value)
+      parsed = value.to_i
+      parsed = DEFAULT_MULTI_FETCH_SIZE if !parsed.positive?
+      update(:multi_fetch_size, parsed)
+    end
+
     private
 
     def update(field, value)
+      key = :"@#{field}"
       if changes[field]
         change = changes[field]
-        changes.delete(field) if change[:from] == value
+        if change[:from] == value
+          changes.delete(field)
+        else
+          change[:to] = value
+        end
+      else
+        current = instance_variable_get(key)
+        changes[field] = {from: current, to: value}
       end
-      set_field!(field, value)
-    end
 
-    def set_field!(field, value)
-      key = :"@#{field}"
-      current = instance_variable_get(key)
-      changes[field] = {from: current, to: value}
       instance_variable_set(key, value)
     end
   end

data/lib/imap/backup/cli/folders.rb

@@ -13,15 +13,15 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.account.username
+          Kernel.puts connection.account.username
           # TODO: Make folder_names private once this command
           # has been removed.
           folders = connection.folder_names
           if folders.nil?
-            warn "Unable to list account folders"
+            Kernel.warn "Unable to list account folders"
             return false
           end
-          folders.each { |f| puts "\t#{f}" }
+          folders.each { |f| Kernel.puts "\t#{f}" }
         end
       end
     end

data/lib/imap/backup/cli/migrate.rb

@@ -39,11 +39,11 @@ module Imap::Backup
         end
 
         if !destination_account
-          raise "Account #{destination_email} does not exist"
+          raise "Account '#{destination_email}' does not exist"
         end
 
         if !source_account
-          raise "Account #{source_email} does not exist"
+          raise "Account '#{source_email}' does not exist"
         end
       end
 
@@ -61,7 +61,7 @@ module Imap::Backup
         glob = File.join(source_local_path, "**", "*.imap")
         Pathname.glob(glob) do |path|
           name = source_folder_name(path)
-          serializer = Serializer::Mbox.new(source_local_path, name)
+          serializer = Serializer.new(source_local_path, name)
           folder = folder_for(name)
           yield serializer, folder
         end

data/lib/imap/backup/cli/restore.rb

@@ -3,16 +3,32 @@ module Imap::Backup
     include Thor::Actions
     include CLI::Helpers
 
+    attr_reader :email
     attr_reader :account_names
 
-    def initialize(options)
-      super([])
-      @account_names = (options[:accounts] || "").split(",")
+    def initialize(email = nil, options)
+      @email = email
+      @account_names =
+        if options.key?(:accounts)
+          options[:accounts].split(",")
+        end
     end
 
     no_commands do
       def run
-        each_connection(account_names) { |connection| connection.restore }
+        case
+        when email && !account_names
+          connection = connection(email)
+          connection.restore
+        when !email && !account_names
+          Logger.logger.info "Calling restore without an EMAIL parameter is deprecated"
+          each_connection([]) { |connection| connection.restore }
+        when email && account_names.any?
+          raise "Pass either an email or the --accounts option, not both"
+        when account_names.any?
+          Logger.logger.info "Calling restore with the --account option is deprected, please pass a single EMAIL argument"
+          each_connection(account_names) { |connection| connection.restore }
+        end
       end
     end
   end

data/lib/imap/backup/cli/utils.rb

@@ -14,7 +14,7 @@ module Imap::Backup
       connection.backup_folders.each do |folder|
         next if !folder.exist?
 
-        serializer = Serializer::Mbox.new(connection.account.local_path, folder.name)
+        serializer = Serializer.new(connection.account.local_path, folder.name)
         do_ignore_folder_history(folder, serializer)
       end
     end
@@ -75,7 +75,7 @@ module Imap::Backup
             Skipped #{uid}
           MESSAGE
 
-          serializer.save(uid, message)
+          serializer.append uid, message
         end
       end
 

data/lib/imap/backup/cli.rb

@@ -100,21 +100,20 @@ module Imap::Backup
       aliases: ["-s"]
     )
     def migrate(source_email, destination_email)
-      Migrate.new(source_email, destination_email, symbolized(options)).run
+      Migrate.new(source_email, destination_email, **symbolized(options)).run
     end
 
     desc "remote SUBCOMMAND [OPTIONS]", "View info about online accounts"
     subcommand "remote", Remote
 
-    desc "restore [OPTIONS]", "This command is deprecated, use `imap-backup restore ACCOUNT`"
+    desc "restore EMAIL", "Restores a single account"
     long_desc <<~DESC
-      By default, restores all local emails to their respective servers.
-      This command is deprecated.
-      Instead, use `imap-backup restore ACCOUNT` to restore a single account.
+      Restores all backed-up emails for the supplied account to
+      their original server.
     DESC
     accounts_option
-    def restore
-      Restore.new(symbolized(options)).run
+    def restore(email = nil)
+      Restore.new(email, symbolized(options)).run
     end
 
     desc "setup", "Configure imap-backup"

data/lib/imap/backup/configuration.rb

@@ -6,7 +6,6 @@ require "imap/backup/account"
 module Imap::Backup
   class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
-    DEFAULT_DOWNLOAD_BLOCK_SIZE = 1
     VERSION = "2.0"
 
     attr_reader :pathname
@@ -52,15 +51,6 @@ module Imap::Backup
       end
     end
 
-    def download_block_size
-      size = ENV["DOWNLOAD_BLOCK_SIZE"].to_i
-      if size > 0
-        size
-      else
-        DEFAULT_DOWNLOAD_BLOCK_SIZE
-      end
-    end
-
     def modified?
       ensure_loaded!
       return true if @saved_debug != @debug
@@ -103,7 +93,7 @@ module Imap::Backup
     end
 
     def remove_modified_flags
-      accounts.each { |a| a.clear_changes! }
+      accounts.each { |a| a.clear_changes }
     end
 
     def remove_deleted_accounts

data/lib/imap/backup/downloader.rb

@@ -1,33 +1,34 @@
 module Imap::Backup
+  class MultiFetchFailedError < StandardError; end
+
   class Downloader
     attr_reader :folder
     attr_reader :serializer
-    attr_reader :block_size
+    attr_reader :multi_fetch_size
 
-    def initialize(folder, serializer, block_size: 1)
+    def initialize(folder, serializer, multi_fetch_size: 1)
       @folder = folder
       @serializer = serializer
-      @block_size = block_size
+      @multi_fetch_size = multi_fetch_size
     end
 
     def run
       uids = folder.uids - serializer.uids
       count = uids.count
       debug "#{count} new messages"
-      uids.each_slice(block_size).with_index do |block, i|
+      uids.each_slice(multi_fetch_size).with_index do |block, i|
         uids_and_bodies = folder.fetch_multi(block)
         if uids_and_bodies.nil?
-          if block_size > 1
+          if multi_fetch_size > 1
             debug("Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
-            @block_size = 1
-            redo
+            raise MultiFetchFailedError
           else
             debug("Fetch failed for UID #{block[0]} - skipping")
             next
           end
         end
 
-        offset = i * block_size + 1
+        offset = i * multi_fetch_size + 1
         uids_and_bodies.each.with_index do |uid_and_body, j|
           uid = uid_and_body[:uid]
           body = uid_and_body[:body]
@@ -38,10 +39,13 @@ module Imap::Backup
             info("Fetch returned empty UID - skipping")
           else
             debug("uid: #{uid} (#{offset + j}/#{count}) - #{body.size} bytes")
-            serializer.save(uid, body)
+            serializer.append uid, body
           end
         end
       end
+    rescue MultiFetchFailedError
+      @multi_fetch_size = 1
+      retry
     end
 
     private

data/lib/imap/backup/serializer/directory.rb

@@ -0,0 +1,37 @@
+require "os"
+
+module Imap::Backup
+  class Serializer; end
+
+  class Serializer::Directory
+    DIRECTORY_PERMISSIONS = 0o700
+
+    attr_reader :relative
+    attr_reader :path
+
+    def initialize(path, relative)
+      @path = path
+      @relative = relative
+    end
+
+    def ensure_exists
+      if !File.directory?(full_path)
+        Utils.make_folder(
+          path, relative, DIRECTORY_PERMISSIONS
+        )
+      end
+
+      return if OS.windows?
+      return if Utils.mode(full_path) == DIRECTORY_PERMISSIONS
+
+      FileUtils.chmod DIRECTORY_PERMISSIONS, full_path
+    end
+
+    private
+
+    def full_path
+      containing_directory = File.join(path, relative)
+      File.expand_path(containing_directory)
+    end
+  end
+end