imap-backup 16.6.0 → 17.0.0.rc0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 325c46cce0eff42c96d4c44e686b32f0d52352cdfa038c17a132069b71bdf11e
-  data.tar.gz: 349426301b37fa38db3da6c2d71a3979892345828b88ffa2b52752a559f6a95b
+  metadata.gz: 4b4c01b65abcd4b7fe7f57d214ce3d0f141085058c3aac3245842bac7dc3bda3
+  data.tar.gz: 418df48403a0e2d28e63b4ab233d69de327e3e139b975ceac8056eb972d05eba
 SHA512:
-  metadata.gz: 6c4735412c68747d5365048e7748cc849bb1302a2da6e4af60354bf2a44c3c4dab5f14cf9ea7111cd7c6f416faea8ad429657a551755167662998e20ff7aae3a
-  data.tar.gz: 213bc6df849b8e7d72ded14920695e1a158d8657ac0d3bcd6632068f0c76258d2b5d0f37a5073867cf1f549249b1a2df651a0327ce803051311ed33fe5e757a0
+  metadata.gz: 6a31419c312cfcfe21bf8dcb36a9093827dd1c3f9a6d4bec599c8fd53a7e426801bf1f9c014c734a73334321e1ecb1973cdac18f5410a9330021c72473df4821
+  data.tar.gz: 81c4d68861e9b77ef064897834e9631b682d576e25ca4fb7625afb15b87dc11ba729bb27940edaefb8bc65a8a88f614ae11129171c8ced9a2d8ad03f122fcb5d

data/docs/TODO.md

@@ -33,3 +33,24 @@ Currently, only the Download Strategy Chooser screen has localized help function
   - Other setup screens (folder chooser, backup path, etc.)
 - Add translation keys for help text to locale files (`lib/imap/backup/locales/en.yml` and `lib/imap/backup/locales/it.yml`)
 - Follow the pattern: menu choice for "help" that displays localized help text and waits for key press
+
+# Fix mboxrd Quote Serialization Bug
+
+Status: [x]
+
+## Description
+
+The `add_extra_quote` method in `Email::Mboxrd::Message` incorrectly quotes all lines beginning with `From` (e.g. `From:` headers), rather than only lines beginning with `From ` (with a trailing space). The corresponding `clean_serialized` method has the same problem on load. This results in mbox files storing corrupted content. The fix involves correcting both regexes, bumping the metadata format version, and migrating existing files on load.
+
+## Technical Specifics
+
+- In [lib/imap/backup/serializer/imap.rb](lib/imap/backup/serializer/imap.rb):
+  - Bump `CURRENT_VERSION` from `3` to `3.1`
+  - Add `3` to `LOADABLE_VERSIONS`: `LOADABLE_VERSIONS = [3, 3.1].freeze`
+  - In `ensure_loaded`, use a `case data[:version]` branch:
+    - `when CURRENT_VERSION` — load messages normally
+    - `when 3` — load each message using the old (buggy) deserialization logic (`from_serialized_v3`), and set `@version = CURRENT_VERSION` so the next save writes version 3.1
+- In [lib/imap/backup/email/mboxrd/message.rb](lib/imap/backup/email/mboxrd/message.rb):
+  - Fix `add_extra_quote`: change regex from `/\n(>*From)/` to `/\n(>*From )/`
+  - Fix `clean_serialized`: change regex from `/^>(>*From)/` to `/^>(>*From )/`
+  - Add `clean_serialized_v3` (using the old regex `/^>(>*From)/`) and a corresponding `from_serialized_v3` class method for use during migration of version 3 files

data/lib/imap/backup/email/mboxrd/message.rb

@@ -14,7 +14,7 @@ module Imap::Backup
       #   and with one level of '>' quoting removed from other lines
       #   that start with 'From'
       def self.clean_serialized(serialized)
-        cleaned = serialized.gsub(/^>(>*From)/, "\\1")
+        cleaned = serialized.gsub(/^>(>*From )/, "\\1")
         # Serialized messages in this format *should* start with a line
         #   From xxx yy zz
         # rubocop:disable Style/IfUnlessModifier
@@ -32,6 +32,21 @@ module Imap::Backup
         new(clean_serialized(serialized))
       end
 
+      # Deserializes a message stored with the old (v3) quoting logic,
+      # which incorrectly quoted all 'From' lines, not just 'From ' lines.
+      def self.clean_serialized_v3(serialized)
+        cleaned = serialized.gsub(/^>(>*From)/, "\\1")
+        cleaned = cleaned.sub(/^From .*[\r\n]*/, "") if cleaned.start_with?("From ")
+        cleaned
+      end
+
+      # @param serialized [String] the on-disk version of a v3 message
+      #
+      # @return [Message] the original message
+      def self.from_serialized_v3(serialized)
+        new(clean_serialized_v3(serialized))
+      end
+
       # @return [String] the original message body
       attr_reader :supplied_body
 
@@ -110,7 +125,7 @@ module Imap::Backup
         # 'From ' can be taken as the beginning of messages.
         # http://www.digitalpreservation.gov/formats/fdd/fdd000385.shtml
         # Here we add an extra '>' before any "From" or ">From".
-        body.gsub(/\n(>*From)/, "\n>\\1")
+        body.gsub(/\n(>*From )/, "\n>\\1")
       end
 
       def asctime

data/lib/imap/backup/serializer/imap.rb

@@ -12,7 +12,9 @@ module Imap::Backup
   # Stores message metadata
   class Serializer::Imap
     # The version number to store in the metadata file
-    CURRENT_VERSION = 3
+    CURRENT_VERSION = 3.1
+
+    LOADABLE_VERSIONS = [3, 3.1].freeze
 
     # @return [Serializer::Files::Path] The path of the imap metadata file, without the '.imap'
     #   extension
@@ -41,7 +43,9 @@ module Imap::Backup
       tsx.begin({savepoint: {messages: messages.dup, uid_validity: uid_validity}}) do
         block.call
 
-        save_internal(version: version, uid_validity: uid_validity, messages: messages) if tsx.data
+        if tsx.data
+          save_internal(version: CURRENT_VERSION, uid_validity: uid_validity, messages: messages)
+        end
       rescue Exception => e
         Logger.logger.error "#{self.class} handling #{e.class}"
         rollback
@@ -72,7 +76,7 @@ module Imap::Backup
 
     def valid?
       return false if !exist?
-      return false if version != CURRENT_VERSION
+      return false if !loadable_version?(version)
       return false if !uid_validity
 
       true
@@ -208,7 +212,7 @@ module Imap::Backup
 
       ensure_loaded
 
-      save_internal(version: version, uid_validity: uid_validity, messages: messages)
+      save_internal(version: CURRENT_VERSION, uid_validity: uid_validity, messages: messages)
     end
 
     private
@@ -232,9 +236,19 @@ module Imap::Backup
 
       data = load
       if data
-        @messages = data[:messages].map { |m| Serializer::Message.new(mbox: mbox, **m) }
+        case data[:version]
+        when CURRENT_VERSION
+          @messages = data[:messages].map { |m| Serializer::Message.new(mbox: mbox, **m) }
+          @version = CURRENT_VERSION
+        when 3
+          @messages = load_v3_messages(data[:messages])
+          @uid_validity = data[:uid_validity]
+          @version = CURRENT_VERSION
+          @loaded = true
+          save_internal(version: CURRENT_VERSION, uid_validity: uid_validity, messages: messages)
+          return
+        end
         @uid_validity = data[:uid_validity]
-        @version = data[:version]
       else
         @messages = []
         @uid_validity = nil
@@ -255,6 +269,7 @@ module Imap::Backup
       end
 
       return nil if !data.key?(:version)
+      return nil if !loadable_version?(data[:version])
       return nil if !data.key?(:uid_validity)
       return nil if !data.key?(:messages)
       return nil if !data[:messages].is_a?(Array)
@@ -262,6 +277,29 @@ module Imap::Backup
       data
     end
 
+    def loadable_version?(version)
+      LOADABLE_VERSIONS.include?(version)
+    end
+
+    def load_v3_messages(message_data)
+      messages = []
+      offset = 0
+      new_content = +""
+      message_data.each do |m|
+        raw = mbox.read(m[:offset], m[:length])
+        original = Email::Mboxrd::Message.from_serialized_v3(raw)
+        serialized = original.to_serialized
+        messages << Serializer::Message.new(
+          mbox: mbox, uid: m[:uid], offset: offset,
+          length: serialized.bytesize, flags: m[:flags] || []
+        )
+        new_content << serialized
+        offset += serialized.bytesize
+      end
+      File.open(mbox.pathname, "wb") { |f| f.write(new_content) }
+      messages
+    end
+
     def mbox
       @mbox ||= Serializer::Mbox.new(files_path: files_path)
     end

data/lib/imap/backup/version.rb

@@ -2,13 +2,13 @@ module Imap; end
 
 module Imap::Backup
   # @private
-  MAJOR    = 16
+  MAJOR    = 17
   # @private
-  MINOR    = 6
+  MINOR    = 0
   # @private
   REVISION = 0
   # @private
-  PRE      = nil
+  PRE      = "rc0"
   # The application version
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 16.6.0
+  version: 17.0.0.rc0
 platform: ruby
 authors:
 - Joe Yates