imap-backup 5.2.0 → 6.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 164962561883b82372b32826843681c6a802e88982f4906778758a073e1479d8
-  data.tar.gz: 8a71ad18952f675f61bc6342a3a9a60c2fc71797444142c2af5f98ef18ac13e9
+  metadata.gz: 4f9cac7966fac9858fdc380d8e9638c8d479035ee4053444a104b8c3eb5a304c
+  data.tar.gz: 5d9785b5e34d90535c8052d216d9b3dbf46ea743672f05129604729662f42f34
 SHA512:
-  metadata.gz: 600ff4b72954cecfb2037b9ef12b16029248a6c6e27b2b801661b5c746264a8aed7a99aa689cbeecc8e89f571b9f4c26350669e582bc087a518981bceb677141
-  data.tar.gz: b706026f0b8231f10dfbb0a0c39162ccdf6a8671c513b6cbf637352dcd8d99987a3f93d2b523db7f4a8e3ff7178877dc3d84c1ad506245a9773330abea2a792f
+  metadata.gz: a4595e4316dcdc20123a8ed2bec524d37c1de95da49281a30484349c317a996cf3c6946617a489880a0c41db41698b2ee862d9aee4f9c9b8fa34d67707ba924c
+  data.tar.gz: ccee268de9cb6225c72fe0cb11a2552fc793f664429d8dfa918e5b4697961c8e5e15e47ed653dac241f377385806443f17683614b8ff76075a8ff2f12e07f98c

data/README.md

@@ -1,3 +1,8 @@
+![Version](https://img.shields.io/gem/v/imap-backup?label=Version&logo=rubygems)
+[![Build Status](https://github.com/joeyates/imap-backup/actions/workflows/main.yml/badge.svg)][CI Status]
+![Coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/joeyates/b54fe758bfb405c04bef72dad293d707/raw/coverage.json)
+![License](https://img.shields.io/github/license/joeyates/imap-backup?color=brightgreen&label=License)
+
 # imap-backup
 
 *Backup GMail (or other IMAP) accounts to disk*
@@ -5,10 +10,12 @@
   * [Source Code]
   * [API documentation]
   * [Rubygem]
+  * [CI Status]
 
 [Source Code]: https://github.com/joeyates/imap-backup "Source code at GitHub"
-[API documentation]: http://rubydoc.info/gems/imap-backup/frames "RDoc API Documentation at Rubydoc.info"
-[Rubygem]: http://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
+[API documentation]: https://rubydoc.info/gems/imap-backup/frames "RDoc API Documentation at Rubydoc.info"
+[Rubygem]: https://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
+[CI Status]: https://github.com/joeyates/imap-backup/actions/workflows/main.yml
 
 # Installation
 

data/docs/development.md

@@ -1,10 +1,10 @@
 # Testing
 
-## Integration Tests
+## Feature Specs
 
-Integration tests (feature specs) are run against a local IMAP server
-controlled by Docker Compose, which needs to be started
-before running the test suite.
+Specs under `specs/features` are integration specs run against a local IMAP server
+controlled by Docker Compose.
+Before running the test suite, it needs to be started:
 
 ```sh
 $ docker-compose up -d
@@ -26,6 +26,12 @@ or
 $ rspec --tag ~docker
 ```
 
+### Debugging
+
+The feature specs are run 'out of process' via the Aruba gem.
+In order to see debugging output from the process,
+use `last_command_started.output`.
+
 ## Access Docker imap server
 
 ```ruby

data/lib/cli_coverage.rb

@@ -4,7 +4,7 @@ class CliCoverage
       require "simplecov"
 
       # Collect coverage separately
-      SimpleCov.command_name "#{ENV['COVERAGE']} coverage"
+      SimpleCov.command_name "#{ENV['COVERAGE']} #{ARGV.join(' ')} coverage"
 
       # Silence output
       SimpleCov.formatter = SimpleCov::Formatter::SimpleFormatter

data/lib/imap/backup/account/connection.rb

@@ -1,5 +1,6 @@
 require "imap/backup/client/apple_mail"
 require "imap/backup/client/default"
+require "imap/backup/serializer/directory"
 
 require "retry_on_error"
 
@@ -52,7 +53,7 @@ module Imap::Backup
     def status
       ensure_account_folder
       backup_folders.map do |folder|
-        s = Serializer::Mbox.new(account.local_path, folder.name)
+        s = Serializer.new(account.local_path, folder.name)
         {name: folder.name, local: s.uids, remote: folder.uids}
       end
     end
@@ -69,7 +70,7 @@ module Imap::Backup
         serializer.apply_uid_validity(folder.uid_validity)
         begin
           Downloader.new(
-            folder, serializer, block_size: config.download_block_size
+            folder, serializer, multi_fetch_size: account.multi_fetch_size
           ).run
         rescue Net::IMAP::ByeResponseError
           reconnect
@@ -86,7 +87,7 @@ module Imap::Backup
       base = Pathname.new(account.local_path)
       Pathname.glob(glob) do |path|
         name = path.relative_path_from(base).to_s[0..-6]
-        serializer = Serializer::Mbox.new(account.local_path, name)
+        serializer = Serializer.new(account.local_path, name)
         folder = Account::Folder.new(self, name)
         yield serializer, folder
       end
@@ -110,7 +111,6 @@ module Imap::Backup
     def reset
       @backup_folders = nil
       @client = nil
-      @config = nil
       @folder_names = nil
       @provider = nil
       @server = nil
@@ -144,7 +144,7 @@ module Imap::Backup
 
     def each_folder
       backup_folders.each do |folder|
-        serializer = Serializer::Mbox.new(account.local_path, folder.name)
+        serializer = Serializer.new(account.local_path, folder.name)
         yield folder, serializer
       end
     end
@@ -161,7 +161,7 @@ module Imap::Backup
           Imap::Backup::Logger.logger.debug(
             "Backup '#{old_name}' renamed and restored to '#{new_name}'"
           )
-          new_serializer = Serializer::Mbox.new(account.local_path, new_name)
+          new_serializer = Serializer.new(account.local_path, new_name)
           new_folder = Account::Folder.new(self, new_name)
           new_folder.create
           new_serializer.force_uid_validity(new_folder.uid_validity)
@@ -180,7 +180,7 @@ module Imap::Backup
       Utils.make_folder(
         File.dirname(account.local_path),
         File.basename(account.local_path),
-        Serializer::DIRECTORY_PERMISSIONS
+        Serializer::Directory::DIRECTORY_PERMISSIONS
       )
     end
 
@@ -195,9 +195,5 @@ module Imap::Backup
     def provider_options
       provider.options.merge(account.connection_options || {})
     end
-
-    def config
-      @config ||= Configuration.new
-    end
   end
 end

data/lib/imap/backup/account.rb

@@ -1,5 +1,7 @@
 module Imap::Backup
   class Account
+    DEFAULT_MULTI_FETCH_SIZE = 1
+
     attr_reader :username
     attr_reader :password
     attr_reader :local_path
@@ -7,7 +9,6 @@ module Imap::Backup
     attr_reader :server
     attr_reader :connection_options
     attr_reader :changes
-    attr_reader :marked_for_deletion
 
     def initialize(options)
       @username = options[:username]
@@ -16,6 +17,7 @@ module Imap::Backup
       @folders = options[:folders]
       @server = options[:server]
       @connection_options = options[:connection_options]
+      @multi_fetch_size = options[:multi_fetch_size]
       @changes = {}
       @marked_for_deletion = false
     end
@@ -25,18 +27,18 @@ module Imap::Backup
     end
 
     def valid?
-      username && password
+      username && password ? true : false
     end
 
     def modified?
       changes.any?
     end
 
-    def clear_changes!
+    def clear_changes
       @changes = {}
     end
 
-    def mark_for_deletion!
+    def mark_for_deletion
       @marked_for_deletion = true
     end
 
@@ -53,6 +55,7 @@ module Imap::Backup
       h[:folders] = @folders if @folders
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
+      h[:multi_fetch_size] = multi_fetch_size if @multi_fetch_size
       h
     end
 
@@ -82,20 +85,37 @@ module Imap::Backup
       update(:connection_options, parsed)
     end
 
+    def multi_fetch_size
+      int = @multi_fetch_size.to_i
+      if int.positive?
+        int
+      else
+        DEFAULT_MULTI_FETCH_SIZE
+      end
+    end
+
+    def multi_fetch_size=(value)
+      parsed = value.to_i
+      parsed = DEFAULT_MULTI_FETCH_SIZE if !parsed.positive?
+      update(:multi_fetch_size, parsed)
+    end
+
     private
 
     def update(field, value)
+      key = :"@#{field}"
       if changes[field]
         change = changes[field]
-        changes.delete(field) if change[:from] == value
+        if change[:from] == value
+          changes.delete(field)
+        else
+          change[:to] = value
+        end
+      else
+        current = instance_variable_get(key)
+        changes[field] = {from: current, to: value}
       end
-      set_field!(field, value)
-    end
 
-    def set_field!(field, value)
-      key = :"@#{field}"
-      current = instance_variable_get(key)
-      changes[field] = {from: current, to: value}
       instance_variable_set(key, value)
     end
   end

data/lib/imap/backup/cli/folders.rb

@@ -13,15 +13,15 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.account.username
+          Kernel.puts connection.account.username
           # TODO: Make folder_names private once this command
           # has been removed.
           folders = connection.folder_names
           if folders.nil?
-            warn "Unable to list account folders"
+            Kernel.warn "Unable to list account folders"
             return false
           end
-          folders.each { |f| puts "\t#{f}" }
+          folders.each { |f| Kernel.puts "\t#{f}" }
         end
       end
     end

data/lib/imap/backup/cli/migrate.rb

@@ -39,11 +39,11 @@ module Imap::Backup
         end
 
         if !destination_account
-          raise "Account #{destination_email} does not exist"
+          raise "Account '#{destination_email}' does not exist"
         end
 
         if !source_account
-          raise "Account #{source_email} does not exist"
+          raise "Account '#{source_email}' does not exist"
         end
       end
 
@@ -61,7 +61,7 @@ module Imap::Backup
         glob = File.join(source_local_path, "**", "*.imap")
         Pathname.glob(glob) do |path|
           name = source_folder_name(path)
-          serializer = Serializer::Mbox.new(source_local_path, name)
+          serializer = Serializer.new(source_local_path, name)
           folder = folder_for(name)
           yield serializer, folder
         end

data/lib/imap/backup/cli/utils.rb

@@ -14,7 +14,7 @@ module Imap::Backup
       connection.backup_folders.each do |folder|
         next if !folder.exist?
 
-        serializer = Serializer::Mbox.new(connection.account.local_path, folder.name)
+        serializer = Serializer.new(connection.account.local_path, folder.name)
         do_ignore_folder_history(folder, serializer)
       end
     end
@@ -75,7 +75,7 @@ module Imap::Backup
             Skipped #{uid}
           MESSAGE
 
-          serializer.save(uid, message)
+          serializer.append uid, message
         end
       end
 

data/lib/imap/backup/configuration.rb

@@ -6,7 +6,6 @@ require "imap/backup/account"
 module Imap::Backup
   class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
-    DEFAULT_DOWNLOAD_BLOCK_SIZE = 1
     VERSION = "2.0"
 
     attr_reader :pathname
@@ -52,15 +51,6 @@ module Imap::Backup
       end
     end
 
-    def download_block_size
-      size = ENV["DOWNLOAD_BLOCK_SIZE"].to_i
-      if size > 0
-        size
-      else
-        DEFAULT_DOWNLOAD_BLOCK_SIZE
-      end
-    end
-
     def modified?
       ensure_loaded!
       return true if @saved_debug != @debug
@@ -103,7 +93,7 @@ module Imap::Backup
     end
 
     def remove_modified_flags
-      accounts.each { |a| a.clear_changes! }
+      accounts.each { |a| a.clear_changes }
     end
 
     def remove_deleted_accounts

data/lib/imap/backup/downloader.rb

@@ -1,33 +1,34 @@
 module Imap::Backup
+  class MultiFetchFailedError < StandardError; end
+
   class Downloader
     attr_reader :folder
     attr_reader :serializer
-    attr_reader :block_size
+    attr_reader :multi_fetch_size
 
-    def initialize(folder, serializer, block_size: 1)
+    def initialize(folder, serializer, multi_fetch_size: 1)
       @folder = folder
       @serializer = serializer
-      @block_size = block_size
+      @multi_fetch_size = multi_fetch_size
     end
 
     def run
       uids = folder.uids - serializer.uids
       count = uids.count
       debug "#{count} new messages"
-      uids.each_slice(block_size).with_index do |block, i|
+      uids.each_slice(multi_fetch_size).with_index do |block, i|
         uids_and_bodies = folder.fetch_multi(block)
         if uids_and_bodies.nil?
-          if block_size > 1
+          if multi_fetch_size > 1
             debug("Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
-            @block_size = 1
-            redo
+            raise MultiFetchFailedError
           else
             debug("Fetch failed for UID #{block[0]} - skipping")
             next
           end
         end
 
-        offset = i * block_size + 1
+        offset = i * multi_fetch_size + 1
         uids_and_bodies.each.with_index do |uid_and_body, j|
           uid = uid_and_body[:uid]
           body = uid_and_body[:body]
@@ -38,10 +39,13 @@ module Imap::Backup
             info("Fetch returned empty UID - skipping")
           else
             debug("uid: #{uid} (#{offset + j}/#{count}) - #{body.size} bytes")
-            serializer.save(uid, body)
+            serializer.append uid, body
           end
         end
       end
+    rescue MultiFetchFailedError
+      @multi_fetch_size = 1
+      retry
     end
 
     private

data/lib/imap/backup/serializer/directory.rb

@@ -0,0 +1,37 @@
+require "os"
+
+module Imap::Backup
+  class Serializer; end
+
+  class Serializer::Directory
+    DIRECTORY_PERMISSIONS = 0o700
+
+    attr_reader :relative
+    attr_reader :path
+
+    def initialize(path, relative)
+      @path = path
+      @relative = relative
+    end
+
+    def ensure_exists
+      if !File.directory?(full_path)
+        Utils.make_folder(
+          path, relative, DIRECTORY_PERMISSIONS
+        )
+      end
+
+      return if OS.windows?
+      return if Utils.mode(full_path) == DIRECTORY_PERMISSIONS
+
+      FileUtils.chmod DIRECTORY_PERMISSIONS, full_path
+    end
+
+    private
+
+    def full_path
+      containing_directory = File.join(path, relative)
+      File.expand_path(containing_directory)
+    end
+  end
+end

data/lib/imap/backup/serializer/imap.rb

@@ -0,0 +1,120 @@
+require "json"
+
+module Imap::Backup
+  class Serializer::Imap
+    CURRENT_VERSION = 2
+
+    attr_reader :folder_path
+    attr_reader :loaded
+
+    def initialize(folder_path)
+      @folder_path = folder_path
+      @loaded = false
+      @uid_validity = nil
+      @uids = nil
+    end
+
+    def append(uid)
+      uids << uid
+      save
+    end
+
+    def exist?
+      File.exist?(pathname)
+    end
+
+    def include?(uid)
+      uids.include?(uid)
+    end
+
+    def index(uid)
+      uids.find_index(uid)
+    end
+
+    def rename(new_path)
+      if exist?
+        old_pathname = pathname
+        @folder_path = new_path
+        File.rename(old_pathname, pathname)
+      else
+        @folder_path = new_path
+      end
+    end
+
+    def uid_validity
+      ensure_loaded
+      @uid_validity
+    end
+
+    def uid_validity=(value)
+      ensure_loaded
+      @uid_validity = value
+      @uids ||= []
+      save
+    end
+
+    def uids
+      ensure_loaded
+      @uids || []
+    end
+
+    def update_uid(old, new)
+      index = uids.find_index(old.to_i)
+      return if index.nil?
+
+      uids[index] = new.to_i
+      save
+    end
+
+    private
+
+    def pathname
+      "#{folder_path}.imap"
+    end
+
+    def ensure_loaded
+      return if loaded
+
+      data = load
+      if data
+        @uids = data[:uids].map(&:to_i)
+        @uid_validity = data[:uid_validity]
+      else
+        @uids = []
+        @uid_validity = nil
+      end
+      @loaded = true
+    end
+
+    def load
+      return nil if !exist?
+
+      data = nil
+      begin
+        content = File.read(pathname)
+        data = JSON.parse(content, symbolize_names: true)
+      rescue JSON::ParserError
+        return nil
+      end
+
+      return nil if !data.key?(:uids)
+      return nil if !data[:uids].is_a?(Array)
+
+      data
+    end
+
+    def save
+      ensure_loaded
+
+      raise "Cannot save metadata without a uid_validity" if !uid_validity
+
+      data = {
+        version: CURRENT_VERSION,
+        uid_validity: @uid_validity,
+        uids: @uids
+      }
+      content = data.to_json
+      File.open(pathname, "w") { |f| f.write content }
+    end
+  end
+end