email_reply_trimmer 0.1.6 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c6eabb8ce3f3327f3abe63f3ee2fe147fb161d96
-  data.tar.gz: 7a07c2267ef47f4607a2ae2f05feac8c0a2584c1
+SHA256:
+  metadata.gz: c094ba472eabd06ed02276c4ec9e42cb32ccb873871e46e55c46cd4640425470
+  data.tar.gz: 968cee2c9b3de46ef86814dd84ee8449c400e9c5fa8d61363cbb90064204fadf
 SHA512:
-  metadata.gz: dba90e1fdc0b0a4f7032f9c2d1e9aabf575a390ee4cf39618037b9a820de8d7abad6338623b203211ef349dc3568b2e55440721dc89fa8ac144eda878cf0e463
-  data.tar.gz: 19be9d9b0496d31e81f7f7adb296d273aeea84130a8243d669857de751748a4e923584a14d4c8f0a90f338428e7dd219fa4212fddce38575b44ec58a598c5455
+  metadata.gz: 6c71cad161a2aaa27e813ff63f91b1e10359d73ffc95d0d01df0374dc8cc9a2ba506985cd4e33d12e63b3f41cd9e6f66bbe42293feea3f6e968c7d90e79a88de
+  data.tar.gz: e7e3661094d8b1183f572b7e847cac62a71215a341d12b627e0b2d5cfedfdad161f858b8aba3d97463d5ffda5919211da2b268dbd75e7edc48a95811d1e5b481

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in email_reply_trimmer.gemspec
+gemspec

data/Rakefile

@@ -6,7 +6,7 @@ def version
   @version ||= File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*['"](?'version'\d+\.\d+\.\d+)['"]/, "version"]
 end
 
-task :default => :test
+task default: :test
 
 require "rake/testtask"
 Rake::TestTask.new(:test)

data/email_reply_trimmer.gemspec

@@ -15,6 +15,10 @@ Gem::Specification.new do |s|
   s.license = "MIT"
 
   s.require_paths = ["lib"]
-  s.files = Dir["**/*"].reject { |path| File.directory?(path) }
+  s.files = Dir["**/*"].reject { |path| File.directory?(path) || path =~ /.*\.gem$/ }
   s.test_files = s.files.select { |path| path =~ /^test\/.+_test\.rb$/ }
+
+  s.add_development_dependency 'rake', '~> 12'
+  s.add_development_dependency 'minitest', '~> 5'
+  s.add_development_dependency 'rubocop', '~> 0.52.1'
 end

data/lib/email_reply_trimmer/delimiter_matcher.rb

@@ -1,6 +1,6 @@
 class DelimiterMatcher
 
-  DELIMITER_CHARACTERS ||= "-_,=+~#*ᐧ"
+  DELIMITER_CHARACTERS ||= "-_,=+~#*ᐧ—"
   DELIMITER_REGEX      ||= /^[[:blank:]]*[#{Regexp.escape(DELIMITER_CHARACTERS)}]+[[:blank:]]*$/
 
   def self.match?(line)

data/lib/email_reply_trimmer/email_header_matcher.rb

@@ -1,12 +1,12 @@
 class EmailHeaderMatcher
 
-  EMAIL_HEADERS_WITH_DATE_MARKERS = [
-    # Dutch
+  EMAIL_HEADERS_WITH_DATE_MARKERS ||= [
+    # Norwegian
     ["Sendt"],
     # English
-    ["Sent"],
+    ["Sent", "Date"],
     # French
-    ["Date"],
+    ["Date", "Le"],
     # German
     ["Gesendet"],
     # Portuguese
@@ -17,19 +17,25 @@ class EmailHeaderMatcher
     ["Fecha"],
     # Italian
     ["Data"],
+    # Dutch
+    ["Datum"],
+    # Swedish
+    ["Skickat"],
+    # Chinese
+    ["发送时间"],
   ]
 
-  EMAIL_HEADERS_WITH_DATE_REGEXES = EMAIL_HEADERS_WITH_DATE_MARKERS.map do |header|
-    /^[[:blank:]>\*]*(?:#{header.join("|")})[[:blank:]\*]*:.*\d+/
+  EMAIL_HEADERS_WITH_DATE_REGEXES ||= EMAIL_HEADERS_WITH_DATE_MARKERS.map do |header|
+    /^[[:blank:]*]*(?:#{header.join("|")})[[:blank:]*]*:.*\d+/
   end
 
-  EMAIL_HEADERS_WITH_TEXT_MARKERS = [
-    # Dutch
+  EMAIL_HEADERS_WITH_TEXT_MARKERS ||= [
+    # Norwegian
     ["Fra", "Til", "Emne"],
     # English
     ["From", "To", "Cc", "Reply-To", "Subject"],
     # French
-    ["De", "À", "Répondre à", "Objet"],
+    ["De", "Expéditeur", "À", "Destinataire", "Répondre à", "Objet"],
     # German
     ["Von", "An", "Betreff"],
     # Portuguese
@@ -37,14 +43,20 @@ class EmailHeaderMatcher
     # Spanish
     ["De", "Para", "Asunto"],
     # Italian
-    ["Da", "Risposta", "A", "Oggetto"]
+    ["Da", "Risposta", "A", "Oggetto"],
+    # Dutch
+    ["Van", "Beantwoorden - Aan", "Aan", "Onderwerp"],
+    # Swedish
+    ["Från", "Till", "Ämne"],
+    # Chinese
+    ["发件人", "收件人", "主题"],
   ]
 
-  EMAIL_HEADERS_WITH_TEXT_REGEXES = EMAIL_HEADERS_WITH_TEXT_MARKERS.map do |header|
-    /^[[:blank:]>\*]*(?:#{header.join("|")})[[:blank:]\*]*:.*[[:word:]]+/
+  EMAIL_HEADERS_WITH_TEXT_REGEXES ||= EMAIL_HEADERS_WITH_TEXT_MARKERS.map do |header|
+    /^[[:blank:]*]*(?:#{header.join("|")})[[:blank:]*]*:.*[[:word:]]+/i
   end
 
-  EMAIL_HEADER_REGEXES = [
+  EMAIL_HEADER_REGEXES ||= [
     EMAIL_HEADERS_WITH_DATE_REGEXES,
     EMAIL_HEADERS_WITH_TEXT_REGEXES,
   ].flatten

data/lib/email_reply_trimmer/embedded_email_matcher.rb

@@ -10,38 +10,43 @@ class EmbeddedEmailMatcher
   # Dnia 14 lip 2015 o godz. 00:25 Michael Downey <info@discourse.org> napisał(a):
   # Em seg, 27 de jul de 2015 17:13, Neil Lalonde <info@discourse.org> escreveu:
   # El jueves, 21 de noviembre de 2013, codinghorror escribió:
-  # Am 03.02.2016 3:35 nachm. schrieb Max Mustermann <mail@example.com>:
-  ON_DATE_SOMEONE_WROTE_MARKERS = [
+  # At 6/16/2016 08:32 PM, you wrote:
+  ON_DATE_SOMEONE_WROTE_REGEXES ||= [
+    # Chinese
+    /^[[:blank:]<>-]*在 (?:(?!\b(?>在|写道)\b).)+?写道[[:blank:].:>-]*$/im,
     # Dutch
-    ["Op","het volgende geschreven"],
+    /^[[:blank:]<>-]*Op (?:(?!\b(?>Op|het\svolgende\sgeschreven|schreef)\b).)+?(het\svolgende\sgeschreven|schreef[^:]+)[[:blank:].:>-]*$/im,
     # English
-    ["On", "wrote"],
+    /^[[:blank:]<>-]*In message (?:(?!\b(?>In message|writes)\b).)+?writes[[:blank:].:>-]*$/im,
+    /^[[:blank:]<>-]*(On|At) (?:(?!\b(?>On|wrote|writes|says|said)\b).)+?(wrote|writes|says|said)[[:blank:].:>-]*$/im,
     # French
-    ["Le", "a écrit "],
+    /^[[:blank:]<>-]*Le (?:(?!\b(?>Le|nous\sa\sdit|a\s+écrit)\b).)+?(nous\sa\sdit|a\s+écrit)[[:blank:].:>-]*$/im,
+    # German
+    /^[[:blank:]<>-]*Am (?:(?!\b(?>Am|schrieben\sSie)\b).)+?schrieben\sSie[[:blank:].:>-]*$/im,
+    /^[[:blank:]<>-]*Am (?:(?!\b(?>Am|geschrieben)\b).)+?(geschrieben|schrieb[^:]+)[[:blank:].:>-]*$/im,
     # Italian
-    ["Il", "ha scritto"],
+    /^[[:blank:]<>-]*Il (?:(?!\b(?>Il|ha\sscritto)\b).)+?ha\sscritto[[:blank:].:>-]*$/im,
     # Polish
-    ["Dnia", "napisał\\(a\\)"],
+    /^[[:blank:]<>-]*(Dnia|Dňa) (?:(?!\b(?>Dnia|Dňa|napisał)\b).)+?napisał(\(a\))?[[:blank:].:>-]*$/im,
     # Portuguese
-    ["Em", "escreveu"],
+    /^[[:blank:]<>-]*Em (?:(?!\b(?>Em|escreveu)\b).)+?escreveu[[:blank:].:>-]*$/im,
     # Spanish
-    ["El", "escribió"],
-    # German
-    ["Am", "schrieb"],
+    /^[[:blank:]<>-]*El (?:(?!\b(?>El|escribió)\b).)+?escribió[[:blank:].:>-]*$/im,
   ]
 
-  ON_DATE_SOMEONE_WROTE_REGEXES = ON_DATE_SOMEONE_WROTE_MARKERS.map do |on, wrote|
-    wrote.gsub!(/ +/, "[[:space:]]+") # the "wrote" part might span over multiple lines
-    /^([[:blank:]>\-]*#{on}\s(?:(?!#{on}\s|#{wrote}:?)[\s\S])*#{wrote}:?[[:blank:]\-]*)$/m
-  end
-
   # Op 10 dec. 2015 18:35 schreef "Arpit Jalan" <info@discourse.org>:
   # Am 18.09.2013 um 16:24 schrieb codinghorror <info@discourse.org>:
+  # Den 15. jun. 2016 kl. 20.42 skrev Jeff Atwood <info@discourse.org>:
+  # søn. 30. apr. 2017 kl. 00.26 skrev David Taylor <meta@discoursemail.com>:
   ON_DATE_WROTE_SOMEONE_MARKERS = [
     # Dutch
     ["Op", "schreef"],
     # German
     ["Am", "schrieb"],
+    # Norwegian
+    ["Den", "skrev"],
+    # Dutch
+    ["søn\.", "skrev"],
   ]
 
   ON_DATE_WROTE_SOMEONE_REGEXES = ON_DATE_WROTE_SOMEONE_MARKERS.map do |on, wrote|
@@ -49,15 +54,31 @@ class EmbeddedEmailMatcher
   end
 
   # суббота, 14 марта 2015 г. пользователь etewiah написал:
+  # 23 mar 2017 21:25 "Neil Lalonde" <meta@discoursemail.com> napisał(a):
+  # 30 серп. 2016 р. 20:45 "Arpit" no-reply@example.com пише:
   DATE_SOMEONE_WROTE_MARKERS = [
     # Russian
     ["пользователь", "написал"],
+    # Polish
+    ["", "napisał\\(a\\)"],
+    # Ukrainian
+    ["", "пише"],
   ]
 
   DATE_SOMEONE_WROTE_REGEXES = DATE_SOMEONE_WROTE_MARKERS.map do |user, wrote|
-    /.+#{user}.+#{wrote}:/
+    user.size == 0 ?
+      /\d{4}.{1,80}\n?.{0,80}?#{wrote}:/ :
+      /\d{4}.{1,80}#{user}.{0,80}\n?.{0,80}?#{wrote}:/
   end
 
+  # Max Mustermann <try_discourse@discoursemail.com> schrieb am Fr., 28. Apr. 2017 um 11:53 Uhr:
+  SOMEONE_WROTE_ON_DATE_REGEXES ||= [
+    # English
+    /^.+\bwrote\b[[:space:]]+\bon\b.+[^:]+:/,
+    # German
+    /^.+\bschrieb\b[[:space:]]+\bam\b.+[^:]+:/,
+  ]
+
   # 2016-03-03 17:21 GMT+01:00 Some One
   ISO_DATE_SOMEONE_REGEX = /^[[:blank:]>]*20\d\d-\d\d-\d\d \d\d:\d\d GMT\+\d\d:\d\d [\w[:blank:]]+$/
 
@@ -65,7 +86,7 @@ class EmbeddedEmailMatcher
   # 2013/10/2 camilohollanda <info@discourse.org>
   # вт, 5 янв. 2016 г. в 23:39, Erlend Sogge Heggen <info@discourse.org>:
   # ср, 1 апр. 2015, 18:29, Denis Didkovsky <info@discourse.org>:
-  DATE_SOMEONE_EMAIL_REGEX = /^[[:blank:]>]*.*\d{4}.+<[^@<>]+@[^@<>.]+\.[^@<>]+>:?$/
+  DATE_SOMEONE_EMAIL_REGEX = /\d{4}.{1,80}\s?<[^@<>]+@[^@<>.]+\.[^@<>]+>:?$/
 
   # codinghorror via Discourse Meta wrote:
   # codinghorror via Discourse Meta <info@discourse.org> schrieb:
@@ -77,11 +98,12 @@ class EmbeddedEmailMatcher
   ]
 
   SOMEONE_VIA_SOMETHING_WROTE_REGEXES = SOMEONE_VIA_SOMETHING_WROTE_MARKERS.map do |wrote|
-    /^[[:blank:]>]*.+ via .+ #{wrote}:?[[:blank:]]*$/
+    /^.+ via .+ #{wrote}:?[[:blank:]]*$/
   end
 
   # Some One <info@discourse.org> wrote:
-  SOMEONE_EMAIL_WROTE_REGEX = /^[[:blank:]>]*.+ <.+@.+\..+> wrote:?/
+  # Gavin Sinclair (gsinclair@soyabean.com.au) wrote:
+  SOMEONE_EMAIL_WROTE_REGEX = /^.+\b[\w.+-]+@[\w.-]+\.\w{2,}\b.+wrote:?$/
 
   # Posted by mpalmer on 01/21/2016
   POSTED_BY_SOMEONE_ON_DATE_REGEX = /^[[:blank:]>]*Posted by .+ on \d{2}\/\d{2}\/\d{4}$/i
@@ -92,17 +114,21 @@ class EmbeddedEmailMatcher
   # ----- Original Message -----
   # -----Original Message-----
   # *----- Original Message -----*
+  # ----- Reply message -----
+  # ------------------ 原始邮件 ------------------
   FORWARDED_EMAIL_REGEXES = [
     # English
     /^[[:blank:]>]*Begin forwarded message:/i,
-    /^[[:blank:]>]*Reply message/i,
-    /^[[:blank:]>\*]*-{2,}[[:blank:]]*(Forwarded|Original) Message[[:blank:]]*-{2,}/i,
+    /^[[:blank:]>*]*-{2,}[[:blank:]]*(Forwarded|Original|Reply) Message[[:blank:]]*-{2,}/i,
     # French
-    /^[[:blank:]>\*]*-{2,}[[:blank:]]*Message transféré[[:blank:]]*-{2,}/i,
+    /^[[:blank:]>]*Début du message transféré :/i,
+    /^[[:blank:]>*]*-{2,}[[:blank:]]*Message transféré[[:blank:]]*-{2,}/i,
     # German
-    /^[[:blank:]>\*]*-{2,}[[:blank:]]*Ursprüngliche Nachricht[[:blank:]]*-{2,}/i,
+    /^[[:blank:]>*]*-{2,}[[:blank:]]*Ursprüngliche Nachricht[[:blank:]]*-{2,}/i,
     # Spanish
-    /^[[:blank:]>\*]*-{2,}[[:blank:]]*Mensaje original[[:blank:]]*-{2,}/i,
+    /^[[:blank:]>*]*-{2,}[[:blank:]]*Mensaje original[[:blank:]]*-{2,}/i,
+    # Chinese
+    /^[[:blank:]>*]*-{2,}[[:blank:]]*原始邮件[[:blank:]]*-{2,}/i,
   ]
 
   EMBEDDED_REGEXES = [
@@ -110,6 +136,7 @@ class EmbeddedEmailMatcher
     ON_DATE_WROTE_SOMEONE_REGEXES,
     DATE_SOMEONE_WROTE_REGEXES,
     DATE_SOMEONE_EMAIL_REGEX,
+    SOMEONE_WROTE_ON_DATE_REGEXES,
     ISO_DATE_SOMEONE_REGEX,
     SOMEONE_VIA_SOMETHING_WROTE_REGEXES,
     SOMEONE_EMAIL_WROTE_REGEX,

data/lib/email_reply_trimmer/signature_matcher.rb

@@ -13,24 +13,38 @@ class SignatureMatcher
   # (sent from a phone)
   # (Sent from mobile device)
   # 從我的 iPhone 傳送
-  SIGNATURE_REGEXES = [
+  SIGNATURE_REGEXES ||= [
     # Chinese
-    /^[[:blank:]>]*從我的 iPhone 傳送/i,
+    /^[[:blank:]]*從我的 iPhone 傳送/i,
     # English
-    /^[[:blank:]>]*[[:word:]]+ from mobile/i,
-    /^[[:blank:]>]*[\(<]*sent (?:from|via|with|by) .+[\)>]*/i,
-    /^[[:blank:]>]*from my .{1,20}/i, # don't match too much
+    /^[[:blank:]]*[[:word:]]+ from mobile/i,
+    /^[[:blank:]]*[\(<]*Sent (from|via|with|by) .+[\)>]*/i,
+    /^[[:blank:]]*From my .{1,20}/i,
+    /^[[:blank:]]*Get Outlook for /i,
     # French
-    /^[[:blank:]>]*Envoyé depuis mon .+/i,
+    /^[[:blank:]]*Envoyé depuis (mon|Yahoo Mail)/i,
     # German
-    /^[[:blank:]>]*Von meinem .+ gesendet/i,
-    /^[[:blank:]>]*Diese Nachricht wurde von .+ gesendet/i,
+    /^[[:blank:]]*Von meinem .+ gesendet/i,
+    /^[[:blank:]]*Diese Nachricht wurde von .+ gesendet/i,
+    # Italian
+    /^[[:blank:]]*Inviato da /i,
+    # Norwegian
+    /^[[:blank:]]*Sendt fra min /i,
+    # Portuguese
+    /^[[:blank:]]*Enviado do meu /i,
     # Spanish
-    /^[[:blank:]>]*Enviado desde mi .+/i,
+    /^[[:blank:]]*Enviado desde mi /i,
+    # Dutch
+    /^[[:blank:]]*Verzonden met /i,
+    /^[[:blank:]]*Verstuurd vanaf mijn /i,
+    # Swedish
+    /^[[:blank:]]*från min /i,
   ]
 
   def self.match?(line)
-    SIGNATURE_REGEXES.any? { |r| line =~ r }
+    # remove any markdown links
+    stripped = line.gsub(/\[([^\]]+)\]\([^\)]+\)/) { $1 }
+    SIGNATURE_REGEXES.any? { |r| stripped =~ r }
   end
 
 end

data/lib/email_reply_trimmer.rb

@@ -6,7 +6,7 @@ require_relative "email_reply_trimmer/email_header_matcher"
 require_relative "email_reply_trimmer/quote_matcher"
 
 class EmailReplyTrimmer
-  VERSION = "0.1.6"
+  VERSION = "0.1.13"
 
   DELIMITER    = "d"
   EMBEDDED     = "b"
@@ -26,16 +26,11 @@ class EmailReplyTrimmer
     return TEXT
   end
 
-  def self.trim(text, split=false)
-    return if text.nil? || text =~ /\A[[:space:]]*\Z/m
+  def self.trim(text, split = false)
+    return if text.nil? || text =~ /\A[[:space:]]*\z/m
 
-    # normalize line endings
-    text.gsub!("\r\n", "\n")
-
-    # fix embedded email markers that might span over multiple lines
-    EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES.each do |r|
-      text.gsub!(r) { |m| m.gsub(/\n[[:space:]>\-]*/, " ") }
-    end
+    # do some cleanup
+    preprocess!(text)
 
     # from now on, we'll work on a line-by-line basis
     lines = text.split("\n")
@@ -59,8 +54,8 @@ class EmailReplyTrimmer
     end
 
     # when the reply is at the end of the email
-    if pattern =~ /^b+q+[eq]*t[te]*$/
-      index = pattern =~ /t/
+    if is_reply_at_end?(pattern)
+      index = pattern =~ /t[et]*$/
       pattern = ""
       lines = lines[index..-1]
     end
@@ -75,12 +70,20 @@ class EmailReplyTrimmer
 
     # if there is an embedded email marker, followed by a huge quote
     # then take everything up to that marker
-    if pattern =~ /te*b[eqbh]*[te]*$/
+    if pattern =~ /te*b[eqbh]*([te]*)$/ && $1.count("t") < 7
       index = pattern =~ /te*b[eqbh]*[te]*$/
       pattern = pattern[0..index]
       lines = lines[0..index]
     end
 
+    # if there is some text before a huge quote ending the email,
+    # then remove the quote
+    if pattern =~ /t?e*[qbe]+$/
+      index = pattern =~ /t?e*[qbe]+$/
+      pattern = pattern[0..index]
+      lines = lines[0..index]
+    end
+
     # if there still are some embedded email markers, just remove them
     while pattern =~ /b/
       index = pattern =~ /b/
@@ -95,8 +98,8 @@ class EmailReplyTrimmer
       size.times.each { |s| pattern[index + s] = EMAIL_HEADER }
     end
 
-    # if there are at least 3 consecutive email headers, take everything up to
-    # these headers
+    # if there are at least 3 consecutive email headers,
+    # take everything up to these headers
     if pattern =~ /t[eq]*h{3,}/
       index = pattern =~ /t[eq]*h{3,}/
       pattern = pattern[0..index]
@@ -128,15 +131,10 @@ class EmailReplyTrimmer
   end
 
   def self.extract_embedded_email(text)
-    return if text.nil? || text =~ /\A[[:space:]]*\Z/m
+    return if text.nil? || text =~ /\A[[:space:]]*\z/m
 
-    # normalize line endings
-    text.gsub!("\r\n", "\n")
-
-    # fix embedded email markers that might span over multiple lines
-    EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES.each do |r|
-      text.gsub!(r) { |m| m.gsub(/\n[[:space:]>\-]*/, " ") }
-    end
+    # do some cleanup
+    preprocess!(text)
 
     # from now on, we'll work on a line-by-line basis
     lines = text.split("\n")
@@ -146,6 +144,12 @@ class EmailReplyTrimmer
 
     if index = pattern =~ /(?:h[eqd]*?){3,}[tq]/
       embedded = lines[index..-1].join("\n").strip
+    elsif index = pattern =~ /b(?:[eqd]*){3,}[tq]/
+      # Exception for email clients (macOS / iOS) which embed fwd emails in quotes.
+      embedded = lines[index + 1..-1].map { |l| l.gsub(/^>\s*/, '') }.join("\n").strip
+    end
+
+    if index
       before = lines[0...(pattern[0...index] =~ /e*(b[eqd]*|b*[ed]*)$/)].join("\n").strip
       return [embedded, before]
     end
@@ -153,22 +157,62 @@ class EmailReplyTrimmer
 
   private
 
-    def self.compute_elided(text, lines)
-      elided = []
+  def self.preprocess!(text)
+    # normalize line endings
+    text.gsub!("\r\n", "\n")
+
+    # remove PGP markers
+    text.gsub!(/\A-----BEGIN PGP SIGNED MESSAGE-----\n(?:Hash: \w+)?\s+/i, "")
+    text.gsub!(/^-----BEGIN PGP SIGNATURE-----$[\s\S]+^-----END PGP SIGNATURE-----/, "")
 
-      t = 0
-      l = 0
+    # remove unsubscribe links
+    text.gsub!(/^Unsubscribe: .+@.+(\n.+http:.+)?\s*\z/i, "")
 
-      while t < text.size
-        while l < lines.size && text[t] == lines[l]
-          t += 1
-          l += 1
-        end
-        elided << text[t]
-        t += 1
+    # remove alias-style quotes marker
+    text.gsub!(/^.*>{5} "[^"\n]+" == .+ writes:/, "")
+
+    # change enclosed-style quotes format
+    text.gsub!(/^>>> ?(.+) ?>>>$\n([\s\S]+?)\n^<<< ?\1 ?<<<$/) { $2.gsub(/^/, "> ") }
+    text.gsub!(/^>{4,}[[:blank:]]*$\n([\s\S]+?)\n^<{4,}[[:blank:]]*$/) { $1.gsub(/^/, "> ") }
+
+    # fix all quotes formats
+    text.gsub!(/^((?:[[:blank:]]*[[:alpha:]]*[>|])+)/) { $1.gsub(/([[:alpha:]]+>|\|)/, ">") }
+
+    # fix embedded email markers that might span over multiple lines
+    (
+      EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES +
+      EmbeddedEmailMatcher::SOMEONE_WROTE_ON_DATE_REGEXES +
+      EmbeddedEmailMatcher::DATE_SOMEONE_WROTE_REGEXES +
+      [EmbeddedEmailMatcher::DATE_SOMEONE_EMAIL_REGEX]
+    ).each do |r|
+      text.gsub!(r) do |m|
+        m.count("\n") > 4 ? m : m.gsub(/\n+[[:space:]]*/, " ")
       end
+    end
 
-      elided.join("\n").strip
+    # remove leading/trailing whitespaces
+    text.strip!
+  end
+
+  def self.compute_elided(text, lines)
+    elided = []
+
+    t = 0
+    l = 0
+
+    while t < text.size
+      while l < lines.size && text[t] == lines[l]
+        t += 1
+        l += 1
+      end
+      elided << text[t]
+      t += 1
     end
 
+    elided.join("\n").strip
+  end
+
+  def self.is_reply_at_end?(pattern)
+    pattern =~ /^b[^t]+t[et]*$/
+  end
 end

data/test/before/forwarded_apple.txt

@@ -0,0 +1 @@
+This email has been forwarded from Apple Mail.

data/test/before/forwarded_gmail.txt

@@ -0,0 +1 @@
+This email has been forwarded from Gmail.

data/test/elided/email_headers_5.txt

@@ -0,0 +1,23 @@
+From: Erlend Sogge Heggen <meta@discoursemail.com>
+Reply-To: Erlend Sogge Heggen <meta+abcd@discoursemail.com>
+Date: Wednesday, 5 April 2017 at 17:01
+To: Jef <jef@bar.com>
+Subject: [Discourse Meta] [PM] Discourse for Communities of Practice, educational organisation
+
+
+erlend_sh<https://meta.discourse.org/u/erlend_sh> Erlend Sogge Heggen<https://meta.discourse.org/u/erlend_sh> Team
+April 5
+
+
+
+Hi Jef,
+
+Is your University a legally recognised educational institution? Otherwise I'm afraid you're not eligible for this discount.
+
+Sincerely,
+
+Erlend
+
+
+
+This email message and any attachments may contain confidential information and may be privileged. If you are not the intended recipient or otherwise not authorized to receive this message, you are prohibited to use, copy, disclose or take any action based on this email or any information contained herein. If you are not the intended recipient, please advise the sender immediately by replying to this email and permanently delete this message and any attachments from your system.

data/test/elided/embedded_ception.txt

@@ -2,7 +2,7 @@ On Mon, Feb 1, 2016 at 6:32 PM, Jeff Atwood <info@discourse.org> wrote:
 
 > This is Jeff's reply.
 >
-> On Mon, Feb 1, 2016 at 7:50 AM, Some One <foo@bar.com wrote:
+> On Mon, Feb 1, 2016 at 7:50 AM, Some One <foo@bar.com > > wrote:
 >
 >> Great!
 >>
@@ -14,7 +14,7 @@ On Mon, Feb 1, 2016 at 6:32 PM, Jeff Atwood <info@discourse.org> wrote:
 >>
 >>> WAT?
 >>>
->>> On Wed, Jan 27, 2016 at 10:48 PM, Some One < foo@bar.com> wrote:
+>>> On Wed, Jan 27, 2016 at 10:48 PM, Some One < >>> foo@bar.com> wrote:
 >>>
 >>>> Hi Team,
 >>>>
@@ -22,7 +22,7 @@ On Mon, Feb 1, 2016 at 6:32 PM, Jeff Atwood <info@discourse.org> wrote:
 >>>>
 >>>> Some One
 >>>>
->>>> On Wed, Jan 27, 2016 at 10:10 AM Discourse Team <team@discourse.org> wrote:
+>>>> On Wed, Jan 27, 2016 at 10:10 AM Discourse Team <team@discourse.org> >>>> wrote:
 >>>>
 >>>>> Hello :waves_hand:
 >>>>>

data/test/elided/embedded_email_12.txt

@@ -1,8 +1,8 @@
-On Thu, 31 Mar 2016 at 11:16 Some One < mailto:Some One <some.one@foo.bar> wrote:
+On Thu, 31 Mar 2016 at 11:16 Some One < mailto:Some One <some.one@foo.bar> > wrote:
 
 
 Two 2
 
-On Thu, 31 Mar 2016 at 10:05 Jeff Atwood < mailto:Jeff Atwood <team@discourse.org> wrote:
+On Thu, 31 Mar 2016 at 10:05 Jeff Atwood < mailto:Jeff Atwood <team@discourse.org> > wrote:
 
 Three 3

data/test/elided/embedded_email_13.txt

@@ -0,0 +1,9 @@
+At 6/16/2016 08:32 PM, you wrote:
+><https://meta.discourse.org/users/codinghorror>codinghorror
+><https://meta.discourse.org/users/codinghorror>Jeff Atwood co-founder
+>June 17
+>
+>Sorry I got a little mixed up with all the incoming replies. Are you
+>able to log in?
+>
+>Use your email address and "forgot password" if you need it reset.

data/test/elided/embedded_email_14.txt

@@ -0,0 +1,11 @@
+2016-10-24 15:36 GMT+02:00 Foo bar < info@foo.bar>:
+
+> Thank you so much Erlend, very thanks!
+>
+> 2016-10-24 15:03 GMT+02:00 Erlend Sogge Heggen <meta@discoursemail.com>:
+>
+>> erlend_sh <https://meta.discourse.org/users/erlend_sh> Erlend Sogge
+>> Heggen <https://meta.discourse.org/users/erlend_sh> team
+>> October 24
+>>
+>> I received your application and I've replied with setup instructions.

data/test/elided/embedded_email_15.txt

@@ -0,0 +1,4 @@
+2017-02-05 13:29 GMT+02:00 Very long author name < notifications@forum.some-discourse-host.local>:
+
+> [Original Messages is quoted here]
+> [...]

data/test/elided/embedded_email_16.txt

@@ -0,0 +1,4 @@
+--
+Eric Hodel - drbrain@segment7.net - http://segment7.net
+All messages signed with fingerprint:
+FEC2 57F1 D465 EB15 5D6E  7C11 332A 551C 796C 9F04

data/test/elided/embedded_email_17.txt

@@ -0,0 +1,2 @@
+On 15 May 2017 19:32, "Nomadic Sprite" <nomadic.sprite01@gmail.com> wrote:
+On 8 May 2017 6:51 pm, "Andy Jones" <Andy.Jones@jameshall.co.uk> wrote:

data/test/elided/embedded_email_18.txt

@@ -0,0 +1 @@
+On 8 May 2017 17:34, "Andy Jones" <Andy.Jones@jameshall.co.uk> wrote:

data/test/elided/embedded_email_chinese.txt

@@ -0,0 +1,4 @@
+> 在 2016年12月12日，下午8:45，Erlend Sogge Heggen <meta@discoursemail.com> 写道：
+> fu.zhang:
+> Some random question
+>

data/test/elided/embedded_email_german_4.txt

@@ -0,0 +1,15 @@
+Max Mustermann <try_discourse@discoursemail.com> schrieb am Fr., 28. Apr. 2017 um 11:53 Uhr:
+
+> max_2 <http://try.discourse.org/u/cfstras_2>
+> April 28
+
+> Hi there! you should be getting a mail.
+> ------------------------------
+
+> Visit Topic <http://try.discourse.org/t/this-is-my-internal-test/716/2>
+> or reply to this email to respond.
+
+> To unsubscribe from these emails, click here
+> <http://try.discourse.org/email/unsubscribe/badf00d>
+> .
+