email_reply_trimmer 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6da75911e9abd808e0f27b0fe0207fb5cd2e6a7d
-  data.tar.gz: 0ff5da3a54857c4822579fb5f4489915558eb335
+  metadata.gz: 61d47c5b31f114be7a83ff4ee618716f9a448726
+  data.tar.gz: 3654868aad0175023394df9c5a26fe5fd70f9ad5
 SHA512:
-  metadata.gz: 8002eea84ed92894029cd6a690405d9db0c40bd2249acaa19adcd51ba319d27c117e0e32a5a8ea0e231fadd1a85ba9283550c6e857a2e057182e9c86d3e5f823
-  data.tar.gz: 54165dc327a4425716f62c7bc0f21bacce2b4c29462bc5d801bd73938cc74b49de20e268dbd659cd443568284b810264f10599a24f569806466e6b6db2975239
+  metadata.gz: 62bb96a32309ea18a3d777fc52cfa900e46c6b62e65c0a010027b01938e39ab5cb0c4f2c68b10dd1308ca2c57bbd7e2df00cc18dfc15c1c4713cdff468387ad9
+  data.tar.gz: 69d4a9a6868904d4e8cebdff0031e9fcba389e227eb6be2068d54e480b6ef8f09de1f52d276adc5599c0c3deec72faa68d3f96abd11b06b5ca92e57d04e581b3

data/lib/email_reply_trimmer/delimiter_matcher.rb

@@ -1,7 +1,7 @@
 class DelimiterMatcher
 
-  DELIMITER_CHARACTERS ||= ['-', '_', '=', '+','~', '#', '*', 'ᐧ']
-  DELIMITER_REGEX      ||= /^[[:space:]]*[#{Regexp.escape(DELIMITER_CHARACTERS.join)}]+[[:space:]]*$/
+  DELIMITER_CHARACTERS ||= "-_,=+~#*ᐧ"
+  DELIMITER_REGEX      ||= /^[[:space:]]*[#{Regexp.escape(DELIMITER_CHARACTERS)}]+[[:space:]]*$/
 
   def self.match?(line)
     line =~ DELIMITER_REGEX

data/lib/email_reply_trimmer.rb

@@ -6,7 +6,7 @@ require_relative "email_reply_trimmer/email_header_matcher"
 require_relative "email_reply_trimmer/quote_matcher"
 
 class EmailReplyTrimmer
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 
   DELIMITER    = "d"
   EMBEDDED     = "b"
@@ -17,12 +17,12 @@ class EmailReplyTrimmer
   TEXT         = "t"
 
   def self.identify_line_content(line)
-    return EMPTY        if EmptyLineMatcher.match?(line)
-    return DELIMITER    if DelimiterMatcher.match?(line)
-    return SIGNATURE    if SignatureMatcher.match?(line)
-    return EMBEDDED     if EmbeddedEmailMatcher.match?(line)
-    return EMAIL_HEADER if EmailHeaderMatcher.match?(line)
-    return QUOTE        if QuoteMatcher.match?(line)
+    return EMPTY        if EmptyLineMatcher.match? line
+    return DELIMITER    if DelimiterMatcher.match? line
+    return SIGNATURE    if SignatureMatcher.match? line
+    return EMBEDDED     if EmbeddedEmailMatcher.match? line
+    return EMAIL_HEADER if EmailHeaderMatcher.match? line
+    return QUOTE        if QuoteMatcher.match? line
     return TEXT
   end
 
@@ -34,22 +34,26 @@ class EmailReplyTrimmer
 
     # fix embedded email markers that might span over multiple lines
     EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES.each do |r|
-      if text =~ r
-        text.gsub!(r) { |m| m.gsub(/\n[[:space:]>\-]*/, " ") }
-      end
+      text.gsub!(r) { |m| m.gsub(/\n[[:space:]>\-]*/, " ") }
     end
 
-    removed = []
-
     # from now on, we'll work on a line-by-line basis
     lines = text.split("\n")
+    lines_dup = lines.dup
 
     # identify content of each lines
     pattern = lines.map { |l| identify_line_content(l) }.join
 
-    # remove all signatures & delimiters
-    while pattern =~ /[ds]/
-      index = pattern =~ /[ds]/
+    # remove everything after the first delimiter
+    if pattern =~ /d/
+      index = pattern =~ /d/
+      pattern = pattern[0...index]
+      lines = lines[0...index]
+    end
+
+    # remove all mobile signatures
+    while pattern =~ /s/
+      index = pattern =~ /s/
       pattern.slice!(index)
       lines.slice!(index)
     end
@@ -58,7 +62,6 @@ class EmailReplyTrimmer
     # then take everything up to that marker
     if pattern =~ /te*b[^q]*$/
       index = pattern =~ /te*b[^q]*$/
-      removed = lines[(index + 1)..-1]
       pattern = pattern[0..index]
       lines = lines[0..index]
     end
@@ -67,7 +70,6 @@ class EmailReplyTrimmer
     # then take everything up to that marker
     if pattern =~ /te*b[eqbh]*[te]*$/
       index = pattern =~ /te*b[eqbh]*[te]*$/
-      removed = lines[(index + 1)..-1]
       pattern = pattern[0..index]
       lines = lines[0..index]
     end
@@ -75,8 +77,8 @@ class EmailReplyTrimmer
     # if there still are some embedded email markers, just remove them
     while pattern =~ /b/
       index = pattern =~ /b/
-      pattern[index] = "e"
-      lines[index] = ""
+      pattern.slice!(index)
+      lines.slice!(index)
     end
 
     # fix email headers when they span over multiple lines
@@ -90,7 +92,6 @@ class EmailReplyTrimmer
     # these headers
     if pattern =~ /t[eq]*h{3,}/
       index = pattern =~ /t[eq]*h{3,}/
-      removed = lines[(index + 1)..-1]
       pattern = pattern[0..index]
       lines = lines[0..index]
     end
@@ -111,13 +112,32 @@ class EmailReplyTrimmer
 
     # results
     trimmed = lines.join("\n").strip
-    elided = removed.join("\n").strip
 
     if split
-      [trimmed, elided]
+      [trimmed, compute_elided(lines_dup, lines)]
     else
       trimmed
     end
   end
 
+  private
+
+    def self.compute_elided(text, lines)
+      elided = []
+
+      t = 0
+      l = 0
+
+      while t < text.size
+        while l < lines.size && text[t] == lines[l]
+          t += 1
+          l += 1
+        end
+        elided << text[t]
+        t += 1
+      end
+
+      elided.join("\n").strip
+    end
+
 end

data/test/elided/delimiters.txt

@@ -0,0 +1,10 @@
+ᐧ
+--
+***
+####
+~~~~~
+======
+_______
+++++++++
+
+   -------

data/test/elided/email_headers_1.txt

@@ -1,3 +1,5 @@
+ ------------------------------
+
 *From:* Outlook user
 *Sent:* 2016-01-27
 *To:* info@discourse.org

data/test/elided/email_headers_2.txt

@@ -1,3 +1,4 @@
+________________________________________
 From: Discourse <info@discourse.org.
 Sent: Thursday, 28 January 2016 8:16 p.m.
 To: Someone

data/test/elided/email_headers_3.txt

@@ -8,6 +8,7 @@ Subject: VIS
         Here's an email with some very important stuff.
 
 
+________________________________
 Reply here<http://foo.bar> or hit reply from your inbox to help members by sharing your ideas.
 Mute this topic<http://42.wat> to stop getting updates, we'll send you the next one.
 

data/test/elided/embedded_ception.txt

@@ -31,6 +31,7 @@ On Mon, Feb 1, 2016 at 6:32 PM, Jeff Atwood <info@discourse.org> wrote:
 >
 
 
+--
 Some One
 Community Manager
 foo@bar.com

data/test/elided/embedded_email_10.txt

@@ -1,3 +1,8 @@
+Sent from Outlook Mobile<https://foo.bar>
+
+
+
+
 On Sun, Feb 7, 2016 at 12:12 AM -0800, "Arpit Jalan" <arpit.jalan@discourse.org<mailto:arpit.jalan@discourse.org>> wrote:
 
 Hi Some,
@@ -15,7 +20,9 @@ On Fri, 5 Feb 2016 at 10:42, Some One <foo@bar.com<mailto:foo@bar.com>> wrote:
 Arpit,
    Yes that sounds good.
 
+Sent from Outlook Mobile<https://foo.bar>
 
+_____________________________
 From: Arpit Jalan <arpit.jalan@discourse.org<mailto:arpit.jalan@discourse.org>>
 Sent: Thursday, February 4, 2016 10:05 AM
 Subject: Meta Discourse update

data/test/elided/embedded_email_7.txt

@@ -0,0 +1,10 @@
+On Tue, 2011-03-01 at 18:02 +0530, Some One wrote:
+
+>
+> This is another part of the embedded email.
+>
+>
+
+
+_______________________
+And here's my signature.

data/test/elided/embedded_email_german_1.txt

@@ -17,3 +17,6 @@ codinghorror via Discourse Meta <info@discourse.org> schrieb:
 >
 >To unsubscribe from these emails, visit your [user
 >preferences](http://meta.discourse.org/user_preferences).
+
+--
+Diese Nachricht wurde von meinem Android-Mobiltelefon mit K-9 Mail gesendet.

data/test/elided/embedded_email_italian.txt

@@ -23,5 +23,6 @@
 > To unsubscribe from these emails, change your [user
 > preferences](https://meta.discourse.org/my/preferences)
 
+--
 Stefano Costa @stekosteko
 Editor, Journal of Open Archaeology Data

data/test/elided/embedded_email_polish.txt

@@ -1,3 +1,6 @@
+--
+Łukasz Jan Niemier
+
 Dnia 14 lip 2015 o godz. 00:25 Michael Downey <info@discourse.org> napisał(a):
 
 >

data/test/elided/embedded_email_quote_text.txt

@@ -0,0 +1 @@
+On Mon, Aug 19, 2013 at 2:36 AM, SomeOne via Discourse Meta < info@discourse.org> wrote:

data/test/elided/embedded_email_spanish_2.txt

@@ -7,3 +7,4 @@ Asunto: [MP]Parser del email
 Visita el tema o responde a este email para publicar.
 Para no recibir m=C3=A1s notificaciones de este tema en particular, haz cli=
 c aqu=C3=AD. Para darte de baja de estos emails, cambia tus preferencias
+=

data/test/elided/forwarded_message.txt

@@ -0,0 +1,6 @@
+---------- Forwarded message ----------
+From: Some One <foo@bar.com>
+Date: Thu, Jan 28, 2016 at 4:00 PM
+Subject: Some subject that
+spans over 2 lines
+To: infod@discourse.org

data/test/elided/signatures.txt

@@ -0,0 +1,26 @@
+Envoyé depuis mon iPhone
+
+Von meinem Mobilgerät gesendet
+Diese Nachricht wurde von meinem Android-Mobiltelefon mit K-9 Mail gesendet.
+
+Someone from mobile
+From My Iphone 6
+Sent via mobile
+Sent with Airmail
+Sent from Windows Mail
+Sent from Mailbox
+Sent from Mailbox for iPad
+Sent from Yahoo Mail on Android
+Sent from my TI-85
+Sent from my iPhone
+Sent from my iPod
+Sent from my Alcatel Flash2
+Sent from my mobile device
+Sent from my cell, please excuse any typos.
+Sent from my Samsung Galaxy s5 Octacore device
+Sent from my HTC M8 Android phone.  Please excuse typoze
+Sent from my Windows 8 PC <http://windows.microsoft.com/consumer-preview>
+<<sent by galaxy>>
+(sent from a phone)
+(Sent from mobile device)
+從我的 iPhone 傳送

data/test/elided/usenet.txt

@@ -0,0 +1,7 @@
+--
+Mit lieben Grüßen
+
+John Doe
+http://blog.john.doe
+www.facebook.com/johndoe
+Mobil:   +12 345 6789 012

data/test/emails/usenet.txt

@@ -0,0 +1,9 @@
+Mal sehen was hier mit der Signatur passiert!
+
+--
+Mit lieben Grüßen
+
+John Doe
+http://blog.john.doe
+www.facebook.com/johndoe
+Mobil:   +12 345 6789 012

data/test/test_email_reply_trimmer.rb

@@ -1,7 +1,7 @@
 require "minitest/autorun"
 require "email_reply_trimmer"
 
-class TestEmailReplyTrimmer < Minitest::Unit::TestCase
+class TestEmailReplyTrimmer < Minitest::Test
 
   EMAILS  = Dir["test/emails/*.txt"].map  { |path| File.basename(path) }
   TRIMMED = Dir["test/trimmed/*.txt"].map { |path| File.basename(path) }

data/test/trimmed/embedded_email_7.txt

@@ -1,5 +1,4 @@
 This is a line before the embedded email.
-
 > Hello
 >
 > This is the embedded email.
@@ -9,11 +8,3 @@ This is some text
 after the
 
 embedded email.
-
->
-> This is another part of the embedded email.
->
->
-
-
-And here's my signature.

data/test/trimmed/embedded_email_polish.txt

@@ -1,3 +1 @@
 Oh, I've forgot to add. MIT
-
-Łukasz Jan Niemier

data/test/trimmed/usenet.txt

@@ -0,0 +1 @@
+Mal sehen was hier mit der Signatur passiert!

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: email_reply_trimmer
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Régis Hanol
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-31 00:00:00.000000000 Z
+date: 2016-10-12 00:00:00.000000000 Z
 dependencies: []
 description: EmailReplyTrimmer is a small library to trim replies from plain text
   email.
@@ -68,6 +68,7 @@ files:
 - test/elided/signatures.txt
 - test/elided/strip.txt
 - test/elided/text_only.txt
+- test/elided/usenet.txt
 - test/emails/delimiters.txt
 - test/emails/dual_embedded.txt
 - test/emails/email_headers_1.txt
@@ -107,6 +108,7 @@ files:
 - test/emails/signatures.txt
 - test/emails/strip.txt
 - test/emails/text_only.txt
+- test/emails/usenet.txt
 - test/test_email_reply_trimmer.rb
 - test/trimmed/delimiters.txt
 - test/trimmed/dual_embedded.txt
@@ -147,6 +149,7 @@ files:
 - test/trimmed/signatures.txt
 - test/trimmed/strip.txt
 - test/trimmed/text_only.txt
+- test/trimmed/usenet.txt
 homepage: https://github.com/discourse/email_reply_trimmer
 licenses:
 - MIT