email_reply_trimmer 0.1.12 → 0.2.0

data/lib/email_reply_trimmer.rb

@@ -1,12 +1,14 @@
+# frozen_string_literal: true
 require_relative "email_reply_trimmer/empty_line_matcher"
 require_relative "email_reply_trimmer/delimiter_matcher"
 require_relative "email_reply_trimmer/signature_matcher"
 require_relative "email_reply_trimmer/embedded_email_matcher"
 require_relative "email_reply_trimmer/email_header_matcher"
 require_relative "email_reply_trimmer/quote_matcher"
+require 'securerandom'
 
 class EmailReplyTrimmer
-  VERSION = "0.1.12"
+  VERSION = "0.2.0"
 
   DELIMITER    = "d"
   EMBEDDED     = "b"
@@ -23,7 +25,7 @@ class EmailReplyTrimmer
     return EMBEDDED     if EmbeddedEmailMatcher.match? line
     return EMAIL_HEADER if EmailHeaderMatcher.match? line
     return QUOTE        if QuoteMatcher.match? line
-    return TEXT
+    TEXT
   end
 
   def self.trim(text, split = false)
@@ -32,11 +34,14 @@ class EmailReplyTrimmer
     # do some cleanup
     preprocess!(text)
 
+    # stash the code blocks - replace them with hashes
+    text, blocks = hoist_code_blocks(text)
+
     # from now on, we'll work on a line-by-line basis
     lines = text.split("\n")
     lines_dup = lines.dup
 
-    # identify content of each lines
+    # create a string of characters, one per line, according to the line content
     pattern = lines.map { |l| identify_line_content(l) }.join
 
     # remove everything after the first delimiter
@@ -54,7 +59,7 @@ class EmailReplyTrimmer
     end
 
     # when the reply is at the end of the email
-    if pattern =~ /^(b[^t]+)*b[bqeh]+t[et]*$/
+    if is_reply_at_end?(pattern)
       index = pattern =~ /t[et]*$/
       pattern = ""
       lines = lines[index..-1]
@@ -78,8 +83,8 @@ class EmailReplyTrimmer
 
     # if there is some text before a huge quote ending the email,
     # then remove the quote
-    if pattern =~ /te*[qbe]+$/
-      index = pattern =~ /te*[qbe]+$/
+    if pattern =~ /t?e*[qbe]+$/
+      index = pattern =~ /t?e*[qbe]+$/
       pattern = pattern[0..index]
       lines = lines[0..index]
     end
@@ -123,6 +128,9 @@ class EmailReplyTrimmer
     # results
     trimmed = lines.join("\n").strip
 
+    # re-inject code blocks
+    blocks.each { |token, block| trimmed.gsub!(token, block) }
+
     if split
       [trimmed, compute_elided(lines_dup, lines)]
     else
@@ -144,66 +152,88 @@ class EmailReplyTrimmer
 
     if index = pattern =~ /(?:h[eqd]*?){3,}[tq]/
       embedded = lines[index..-1].join("\n").strip
+    elsif index = pattern =~ /b(?:[eqd]*){3,}[tq]/
+      # Exception for email clients (macOS / iOS) which embed fwd emails in quotes.
+      embedded = lines[index + 1..-1].map { |l| l.gsub(/^>\s*/, '') }.join("\n").strip
+    end
+
+    if index
       before = lines[0...(pattern[0...index] =~ /e*(b[eqd]*|b*[ed]*)$/)].join("\n").strip
-      return [embedded, before]
+      [embedded, before]
     end
   end
 
   private
 
-    def self.preprocess!(text)
-      # normalize line endings
-      text.gsub!("\r\n", "\n")
-
-      # remove PGP markers
-      text.gsub!(/\A-----BEGIN PGP SIGNED MESSAGE-----\n(?:Hash: \w+)?\s+/i, "")
-      text.gsub!(/^-----BEGIN PGP SIGNATURE-----$[\s\S]+^-----END PGP SIGNATURE-----/, "")
-
-      # remove unsubscribe links
-      text.gsub!(/^Unsubscribe: .+@.+(\n.+http:.+)?\s*\z/i, "")
-
-      # remove alias-style quotes marker
-      text.gsub!(/^.*>{5} "[^"\n]+" == .+ writes:/, "")
-
-      # change enclosed-style quotes format
-      text.gsub!(/^>>> ?(.+) ?>>>$\n([\s\S]+?)\n^<<< ?\1 ?<<<$/) { $2.gsub(/^/, "> ") }
-      text.gsub!(/^>{4,}[[:blank:]]*$\n([\s\S]+?)\n^<{4,}[[:blank:]]*$/) { $1.gsub(/^/, "> ") }
-
-      # fix all quotes formats
-      text.gsub!(/^((?:[[:blank:]]*[[:alpha:]]*[>|])+)/) { $1.gsub(/([[:alpha:]]+>|\|)/, ">") }
-
-      # fix embedded email markers that might span over multiple lines
-      (
-        EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES +
-        EmbeddedEmailMatcher::SOMEONE_WROTE_ON_DATE_REGEXES +
-        EmbeddedEmailMatcher::DATE_SOMEONE_WROTE_REGEXES +
-        [EmbeddedEmailMatcher::DATE_SOMEONE_EMAIL_REGEX]
-      ).each do |r|
-        text.gsub!(r) do |m|
-          m.count("\n") > 4 ? m : m.gsub(/\n+[[:space:]]*/, " ")
-        end
-      end
+  def self.hoist_code_blocks(text)
+    blocks = {}
+    pattern = /^```\w*$\n.*?^```$/m
 
-      # remove leading/trailing whitespaces
-      text.strip!
+    text.gsub!(pattern) do |block|
+      token = SecureRandom.hex
+      blocks[token] = block
+      token
     end
 
-    def self.compute_elided(text, lines)
-      elided = []
+    [text, blocks]
+  end
 
-      t = 0
-      l = 0
+  def self.preprocess!(text)
+    # normalize line endings
+    text.gsub!("\r\n", "\n")
 
-      while t < text.size
-        while l < lines.size && text[t] == lines[l]
-          t += 1
-          l += 1
-        end
-        elided << text[t]
-        t += 1
+    # remove PGP markers
+    text.gsub!(/\A-----BEGIN PGP SIGNED MESSAGE-----\n(?:Hash: \w+)?\s+/i, "")
+    text.gsub!(/^-----BEGIN PGP SIGNATURE-----$[\s\S]+^-----END PGP SIGNATURE-----/, "")
+
+    # remove unsubscribe links
+    text.gsub!(/^Unsubscribe: .+@.+(\n.+http:.+)?\s*\z/i, "")
+
+    # remove alias-style quotes marker
+    text.gsub!(/^.*>{5} "[^"\n]+" == .+ writes:/, "")
+
+    # change enclosed-style quotes format
+    text.gsub!(/^>>> ?(.+) ?>>>$\n([\s\S]+?)\n^<<< ?\1 ?<<<$/) { $2.gsub(/^/, "> ") }
+    text.gsub!(/^>{4,}[[:blank:]]*$\n([\s\S]+?)\n^<{4,}[[:blank:]]*$/) { $1.gsub(/^/, "> ") }
+
+    # fix all quotes formats
+    text.gsub!(/^((?:[[:blank:]]*[[:alpha:]]*[>|])+)/) { $1.gsub(/([[:alpha:]]+>|\|)/, ">") }
+
+    # fix embedded email markers that might span over multiple lines
+    (
+      EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES +
+      EmbeddedEmailMatcher::SOMEONE_WROTE_ON_DATE_REGEXES +
+      EmbeddedEmailMatcher::DATE_SOMEONE_WROTE_REGEXES +
+      [EmbeddedEmailMatcher::DATE_SOMEONE_EMAIL_REGEX]
+    ).each do |r|
+      text.gsub!(r) do |m|
+        m.count("\n") > 4 ? m : m.gsub(/\n+[[:space:]]*/, " ")
       end
+    end
+
+    # remove leading/trailing whitespaces
+    text.strip!
+  end
+
+  def self.compute_elided(text, lines)
+    elided = []
+
+    t = 0
+    l = 0
 
-      elided.join("\n").strip
+    while t < text.size
+      while l < lines.size && text[t] == lines[l]
+        t += 1
+        l += 1
+      end
+      elided << text[t]
+      t += 1
     end
 
+    elided.join("\n").strip
+  end
+
+  def self.is_reply_at_end?(pattern)
+    pattern =~ /^b[^t]+t[et]*$/
+  end
 end

data/test/before/forwarded_apple.txt

@@ -0,0 +1 @@
+This email has been forwarded from Apple Mail.

data/test/before/forwarded_gmail.txt

@@ -0,0 +1 @@
+This email has been forwarded from Gmail.

data/test/elided/forwarded_apple.txt

@@ -0,0 +1,15 @@
+> Begin forwarded message:
+>
+> From: Foo Discourse <foo@discourse.org>
+> Subject: This email is going to be forwarded
+> Date: 01 January 2019 at 12:00:00 EET
+> To: bar@discourse.org
+>
+> Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor
+> lacus in metus vehicula, non dictum mi consequat. Nunc eget quam sit
+> amet dolor scelerisque cursus. Etiam laoreet lectus a rutrum
+> sollicitudin. Nullam sit amet lorem id ante lobortis fringilla. Duis
+> dictum tempus sem et ultrices. Sed ac lectus ante. Vivamus interdum,
+> risus non tincidunt accumsan, purus mi accumsan dolor, vitae euismod
+> turpis orci ut est. Integer ut ante eu velit dignissim ullamcorper non
+> id nibh. Nulla auctor magna erat, et posuere sapien gravida convallis.

data/test/elided/forwarded_gmail.txt

@@ -0,0 +1,15 @@
+---------- Forwarded message ---------
+From: Foo Discourse <foo@discourse.org>
+Date: Tue, Jan 01, 2019 at 12:00 PM
+Subject: This email is going to be forwarded
+To: <bar@discourse.org>
+
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor
+lacus in metus vehicula, non dictum mi consequat. Nunc eget quam sit
+amet dolor scelerisque cursus. Etiam laoreet lectus a rutrum
+sollicitudin. Nullam sit amet lorem id ante lobortis fringilla. Duis
+dictum tempus sem et ultrices. Sed ac lectus ante. Vivamus interdum,
+risus non tincidunt accumsan, purus mi accumsan dolor, vitae euismod
+turpis orci ut est. Integer ut ante eu velit dignissim ullamcorper non
+id nibh. Nulla auctor magna erat, et posuere sapien gravida convallis.

data/test/elided/spam_1.txt

@@ -0,0 +1,75 @@
+> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>
+>> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>>
+>>> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>>>
+>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>
+>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>
+>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>
+>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>
+>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>
+>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>
+>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>
+>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>
+>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>
+>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>
+>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:

data/test/elided/spam_2.txt

@@ -0,0 +1,152 @@
+> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>
+>> Hello world!
+>>
+>> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>>
+>>> Hello world!
+>>>
+>>> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>>>
+>>>> Hello world!
+>>>>
+>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>
+>>>>> Hello world!
+>>>>>
+>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>
+>>>>>> Hello world!
+>>>>>>
+>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>
+>>>>>>> Hello world!
+>>>>>>>
+>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>
+>>>>>>>> Hello world!
+>>>>>>>>
+>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>
+>>>>>>>>> Hello world!
+>>>>>>>>>
+>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>
+>>>>>>>>>> Hello world!
+>>>>>>>>>>
+>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>
+>>>>>>>>>>> Hello world!
+>>>>>>>>>>>
+>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>
+>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>
+>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>
+>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>
+>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>
+>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>
+>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Hello world!
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

data/test/emails/block_code_spacers.txt

@@ -0,0 +1,13 @@
+This is a long email with embedded code.
+
+```
+# This should not be deleted
+#
+
+# Or trimmed
+# It is code
+####
+Code code code
+```
+
+End of the email.

data/test/emails/forwarded_apple.txt

@@ -0,0 +1,17 @@
+This email has been forwarded from Apple Mail.
+
+> Begin forwarded message:
+>
+> From: Foo Discourse <foo@discourse.org>
+> Subject: This email is going to be forwarded
+> Date: 01 January 2019 at 12:00:00 EET
+> To: bar@discourse.org
+>
+> Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor
+> lacus in metus vehicula, non dictum mi consequat. Nunc eget quam sit
+> amet dolor scelerisque cursus. Etiam laoreet lectus a rutrum
+> sollicitudin. Nullam sit amet lorem id ante lobortis fringilla. Duis
+> dictum tempus sem et ultrices. Sed ac lectus ante. Vivamus interdum,
+> risus non tincidunt accumsan, purus mi accumsan dolor, vitae euismod
+> turpis orci ut est. Integer ut ante eu velit dignissim ullamcorper non
+> id nibh. Nulla auctor magna erat, et posuere sapien gravida convallis.

data/test/emails/forwarded_gmail.txt

@@ -0,0 +1,17 @@
+This email has been forwarded from Gmail.
+
+---------- Forwarded message ---------
+From: Foo Discourse <foo@discourse.org>
+Date: Tue, Jan 01, 2019 at 12:00 PM
+Subject: This email is going to be forwarded
+To: <bar@discourse.org>
+
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor
+lacus in metus vehicula, non dictum mi consequat. Nunc eget quam sit
+amet dolor scelerisque cursus. Etiam laoreet lectus a rutrum
+sollicitudin. Nullam sit amet lorem id ante lobortis fringilla. Duis
+dictum tempus sem et ultrices. Sed ac lectus ante. Vivamus interdum,
+risus non tincidunt accumsan, purus mi accumsan dolor, vitae euismod
+turpis orci ut est. Integer ut ante eu velit dignissim ullamcorper non
+id nibh. Nulla auctor magna erat, et posuere sapien gravida convallis.

data/test/emails/spam_1.txt

@@ -0,0 +1,75 @@
+> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>
+>> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>>
+>>> On Thu, Apr 30, 2020 at 10:07 AM, "someone" <someone@example.com> wrote:
+>>>
+>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>
+>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>
+>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>
+>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>
+>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>
+>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>
+>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>
+>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>
+>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>
+>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>
+>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote:
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> On Thu, Apr 30, 2020 at 10:06 AM, "someone" <someone@example.com> wrote: