runger_email_reply_trimmer 0.2.1

data/lib/runger_email_reply_trimmer.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require_relative 'runger_email_reply_trimmer/delimiter_matcher'
+
+require_relative 'runger_email_reply_trimmer/email_header_matcher'
+require_relative 'runger_email_reply_trimmer/embedded_email_matcher'
+require_relative 'runger_email_reply_trimmer/empty_line_matcher'
+require_relative 'runger_email_reply_trimmer/quote_matcher'
+require_relative 'runger_email_reply_trimmer/signature_matcher'
+
+module RungerEmailReplyTrimmer
+  module_function
+
+  DELIMITER    = 'd'
+  EMBEDDED     = 'b'
+  EMPTY        = 'e'
+  EMAIL_HEADER = 'h'
+  QUOTE        = 'q'
+  SIGNATURE    = 's'
+  TEXT         = 't'
+
+  def identify_line_content(line)
+    return EMPTY        if EmptyLineMatcher.match?(line)
+    return DELIMITER    if DelimiterMatcher.match?(line)
+    return SIGNATURE    if SignatureMatcher.match?(line)
+    return EMBEDDED     if EmbeddedEmailMatcher.match?(line)
+    return EMAIL_HEADER if EmailHeaderMatcher.match?(line)
+    return QUOTE        if QuoteMatcher.match?(line)
+
+    TEXT
+  end
+
+  def trim(text, split = false)
+    return if text.nil? || text =~ /\A[[:space:]]*\z/m
+
+    # do some cleanup
+    preprocess!(text)
+
+    # from now on, we'll work on a line-by-line basis
+    lines = text.split("\n")
+    lines_dup = lines.dup
+
+    # identify content of each lines
+    pattern = lines.map { |l| identify_line_content(l) }.join
+
+    # remove everything after the first delimiter
+    if pattern =~ /d/
+      index = pattern =~ /d/
+      pattern = pattern[0...index]
+      lines = lines[0...index]
+    end
+
+    # remove all mobile signatures
+    while pattern =~ /s/
+      index = pattern =~ /s/
+      pattern.slice!(index)
+      lines.slice!(index)
+    end
+
+    # when the reply is at the end of the email
+    if is_reply_at_end?(pattern)
+      index = pattern =~ /t[et]*$/
+      pattern = ''
+      lines = lines[index..]
+    end
+
+    # if there is an embedded email marker, not followed by a quote
+    # then take everything up to that marker
+    if pattern =~ /te*b[^q]*$/
+      index = pattern =~ /te*b[^q]*$/
+      pattern = pattern[0..index]
+      lines = lines[0..index]
+    end
+
+    # if there is an embedded email marker, followed by a huge quote
+    # then take everything up to that marker
+    if pattern =~ /te*b[eqbh]*([te]*)$/ && ::Regexp.last_match(1).count('t') < 7
+      index = pattern =~ /te*b[eqbh]*[te]*$/
+      pattern = pattern[0..index]
+      lines = lines[0..index]
+    end
+
+    # if there is some text before a huge quote ending the email,
+    # then remove the quote
+    if pattern =~ /t?e*[qbe]+$/
+      index = pattern =~ /t?e*[qbe]+$/
+      pattern = pattern[0..index]
+      lines = lines[0..index]
+    end
+
+    # if there still are some embedded email markers, just remove them
+    while pattern =~ /b/
+      index = pattern =~ /b/
+      pattern.slice!(index)
+      lines.slice!(index)
+    end
+
+    # fix email headers when they span over multiple lines
+    if pattern =~ /h+[hte]+h+e/
+      index = pattern =~ /h+[hte]+h+e/
+      size = pattern[/h+[hte]+h+e/].size
+      size.times.each { |s| pattern[index + s] = EMAIL_HEADER }
+    end
+
+    # if there are at least 3 consecutive email headers,
+    # take everything up to these headers
+    if pattern =~ /t[eq]*h{3,}/
+      index = pattern =~ /t[eq]*h{3,}/
+      pattern = pattern[0..index]
+      lines = lines[0..index]
+    end
+
+    # if there still are some email headers, just remove them
+    while pattern =~ /h/
+      index = pattern =~ /h/
+      pattern.slice!(index)
+      lines.slice!(index)
+    end
+
+    # remove trailing quotes when there's at least one line of text
+    if pattern =~ /t/ && pattern =~ /[eq]+$/
+      index = pattern =~ /[eq]+$/
+      pattern = pattern[0...index]
+      lines = lines[0...index]
+    end
+
+    # results
+    trimmed = lines.join("\n").strip
+
+    if split
+      [trimmed, compute_elided(lines_dup, lines)]
+    else
+      trimmed
+    end
+  end
+
+  def extract_embedded_email(text)
+    return if text.nil? || text =~ /\A[[:space:]]*\z/m
+
+    # do some cleanup
+    preprocess!(text)
+
+    # from now on, we'll work on a line-by-line basis
+    lines = text.split("\n")
+
+    # identify content of each lines
+    pattern = lines.map { |l| identify_line_content(l) }.join
+
+    if (index = pattern =~ /(?:h[eqd]*?){3,}[tq]/)
+      embedded = lines[index..].join("\n").strip
+    elsif (index = pattern =~ /b(?:[eqd]*){3,}[tq]/)
+      # Exception for email clients (macOS / iOS) which embed fwd emails in quotes.
+      embedded = lines[index + 1..].map { |l| l.gsub(/^>\s*/, '') }.join("\n").strip
+    end
+
+    if index
+      before = lines[0...(pattern[0...index] =~ /e*(b[eqd]*|b*[ed]*)$/)].join("\n").strip
+      [embedded, before]
+    end
+  end
+
+  def preprocess!(text)
+    # normalize line endings
+    text.gsub!("\r\n", "\n")
+
+    # remove PGP markers
+    text.gsub!(/\A-----BEGIN PGP SIGNED MESSAGE-----\n(?:Hash: \w+)?\s+/i, '')
+    text.gsub!(/^-----BEGIN PGP SIGNATURE-----$[\s\S]+^-----END PGP SIGNATURE-----/, '')
+
+    # remove unsubscribe links
+    text.gsub!(/^Unsubscribe: .+@.+(\n.+http:.+)?\s*\z/i, '')
+
+    # remove alias-style quotes marker
+    text.gsub!(/^.*>{5} "[^"\n]+" == .+ writes:/, '')
+
+    # change enclosed-style quotes format
+    text.gsub!(/^>>> ?(.+) ?>>>$\n([\s\S]+?)\n^<<< ?\1 ?<<<$/) {
+      ::Regexp.last_match(2).gsub(/^/, '> ')
+    }
+    text.gsub!(/^>{4,}[[:blank:]]*$\n([\s\S]+?)\n^<{4,}[[:blank:]]*$/) {
+      ::Regexp.last_match(1).gsub(/^/, '> ')
+    }
+
+    # fix all quotes formats
+    text.gsub!(/^((?:[[:blank:]]*[[:alpha:]]*[>|])+)/) {
+      ::Regexp.last_match(1).gsub(/([[:alpha:]]+>|\|)/, '>')
+    }
+
+    # fix embedded email markers that might span over multiple lines
+    (
+      EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES +
+      EmbeddedEmailMatcher::SOMEONE_WROTE_ON_DATE_REGEXES +
+      EmbeddedEmailMatcher::DATE_SOMEONE_WROTE_REGEXES +
+      [EmbeddedEmailMatcher::DATE_SOMEONE_EMAIL_REGEX]
+    ).each do |r|
+      text.gsub!(r) do |m|
+        m.count("\n") > 4 ? m : m.gsub(/\n+[[:space:]]*/, ' ')
+      end
+    end
+
+    # remove leading/trailing whitespaces
+    text.strip!
+  end
+
+  def compute_elided(text, lines)
+    elided = []
+
+    t = 0
+    l = 0
+
+    while t < text.size
+      while l < lines.size && text[t] == lines[l]
+        t += 1
+        l += 1
+      end
+      elided << text[t]
+      t += 1
+    end
+
+    elided.join("\n").strip
+  end
+
+  def is_reply_at_end?(pattern)
+    pattern =~ /^b[^t]+t[et]*$/
+  end
+end

data/runger_email_reply_trimmer.gemspec

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'lib/runger_email_reply_trimmer/version'
+
+Gem::Specification.new do |s|
+  s.name = 'runger_email_reply_trimmer'
+  s.version = RungerEmailReplyTrimmer::VERSION
+
+  s.summary = 'Library to trim replies from plain text email.'
+  s.description = 'RungerEmailReplyTrimmer is a library to trim replies from plain text email.'
+
+  s.authors = ['David Runger', 'Régis Hanol']
+  s.email = ['davidjrunger@gmail.com', 'regis+rubygems@hanol.fr']
+
+  s.homepage = 'https://github.com/davidrunger/runger_email_reply_trimmer'
+  s.license = 'MIT'
+
+  s.require_paths = ['lib']
+  s.files = Dir['**/*'].reject { |path| File.directory?(path) || path =~ /.*\.gem$/ }
+  s.metadata['rubygems_mfa_required'] = 'true'
+end

data/test/before/email_headers_1.txt

@@ -0,0 +1 @@
+This is a reply from Outlook!

data/test/before/email_headers_2.txt

@@ -0,0 +1 @@
+This is a reply from Outlook!

data/test/before/email_headers_3.txt

@@ -0,0 +1 @@
+This is the actual reply.

data/test/before/email_headers_4.txt

@@ -0,0 +1 @@
+test

data/test/before/embedded_email_10.txt

@@ -0,0 +1,25 @@
+Thank you.
+
+Sent from Outlook Mobile<https://foo.bar>
+
+
+
+
+On Sun, Feb 7, 2016 at 12:12 AM -0800, "Arpit Jalan" <arpit.jalan@discourse.org<mailto:arpit.jalan@discourse.org>> wrote:
+
+Hi Some,
+
+https://meta.discourse.org is now running on latest Discourse version!
+
+Regards,
+Arpit
+
+On Fri, Feb 5, 2016 at 10:43 AM Arpit Jalan <arpit.jalan@discourse.org<mailto:arpit.jalan@discourse.org>> wrote:
+Okay, sure!
+
+Arpit
+On Fri, 5 Feb 2016 at 10:42, Some One <foo@bar.com<mailto:foo@bar.com>> wrote:
+Arpit,
+   Yes that sounds good.
+
+Sent from Outlook Mobile<https://foo.bar>

data/test/before/embedded_email_german_3.txt

@@ -0,0 +1 @@
+Gruß Discourse

data/test/before/embedded_email_spanish_2.txt

@@ -0,0 +1 @@
+Igual que siempre (inclusive ahora), sin nada raro :/

data/test/before/forwarded_apple.txt

@@ -0,0 +1 @@
+This email has been forwarded from Apple Mail.

data/test/before/forwarded_gmail.txt

@@ -0,0 +1 @@
+This email has been forwarded from Gmail.

data/test/elided/delimiters.txt

@@ -0,0 +1,10 @@
+ᐧ
+--
+***
+####
+~~~~~
+======
+_______
+++++++++
+
+   -------

data/test/elided/dual_embedded.txt

@@ -0,0 +1,10 @@
+On Mon, Feb 1, 2016 at 4:58 AM -0800, "Some One" <foo@bar.com> wrote:
+
+
+This is the 2nd email.
+
+
+On Sun, Jan 31, 2016 at 9:58 PM -0800, "Another One" <info@discourse.org> wrote:
+
+
+This is the 1st email.

data/test/elided/email_headers_1.txt

@@ -0,0 +1,14 @@
+ ------------------------------
+
+*From:* Outlook user
+*Sent:* 2016-01-27
+*To:* info@discourse.org
+*Subject:* Are you using Outlook?
+
+
+
+Just some random text wich is part
+
+of the embedded email...
+
+Outlook user

data/test/elided/email_headers_2.txt

@@ -0,0 +1,9 @@
+________________________________________
+From: Discourse <info@discourse.org.
+Sent: Thursday, 28 January 2016 8:16 p.m.
+To: Someone
+Subject: (no subject)
+
+Lorem ipsum.
+
+Ipsor dolor sum met.

data/test/elided/email_headers_3.txt

@@ -0,0 +1,16 @@
+From: Some One <discuss@foo.bar<mailto:discuss@foo.bar>>
+Reply-To: "For.bar" <reply+275e18486b01289e3250bebe85ef6496@members.foo.bar<mailto:reply+275e18486b01289e3250bebe85ef6496@members.foo.bar>>
+Date: Monday, February 8, 2016 11:44 AM
+To: Discourse <discourse@discourse.org<mailto:discourse@discourse.org>>
+Subject: VIS
+
+
+        Here's an email with some very important stuff.
+
+
+________________________________
+Reply here<http://foo.bar> or hit reply from your inbox to help members by sharing your ideas.
+Mute this topic<http://42.wat> to stop getting updates, we'll send you the next one.
+
+
+DO NOT FORWARD THIS EMAIL!

data/test/elided/email_headers_4.txt

@@ -0,0 +1,15 @@
+Da: Sally54721
+Risposta: Testy McTesterson / Test
+Data: giovedì 8 ottobre 2015 15:26
+A: Testy McTesterson
+Oggetto: Test | Issue (#3)
+
+[@example](http://example.com/u/example)
+
+—
+Reply to this email directly or [view it on GitLab](http://git.example.com/example/Test/issues/3).  {"@context":"[http://schema.org","@type":"EmailMessage","action":{"@type":"ViewAction","name":"View](http://schema.org%22,%22@type%22:%22EmailMessage%22,%22action%22:%7B%22@type%22:%22ViewAction%22,%22name%22:%22View) Issue","url":"[http://git.example.com/example/Test/issues/3"}](http://git.example.com/example/Test/issues/3%22%7D)} You're receiving this notification because you are a member of the Testy McTesterson / Test project team.
+
+--
+Questo messaggio e' stato analizzato con Libra ESVA ed e' risultato non infetto.
+[Clicca qui per segnalarlo come spam.](http://esva.example.com/cgi-bin/learn-msg.cgi?id=1234567890.ABCDEF)
+[Clicca qui per metterlo in blacklist](http://esva.example.com/cgi-bin/learn-msg.cgi?blacklist=1&id=1234567890.ABCDEF)

data/test/elided/email_headers_5.txt

@@ -0,0 +1,23 @@
+From: Erlend Sogge Heggen <meta@discoursemail.com>
+Reply-To: Erlend Sogge Heggen <meta+abcd@discoursemail.com>
+Date: Wednesday, 5 April 2017 at 17:01
+To: Jef <jef@bar.com>
+Subject: [Discourse Meta] [PM] Discourse for Communities of Practice, educational organisation
+
+
+erlend_sh<https://meta.discourse.org/u/erlend_sh> Erlend Sogge Heggen<https://meta.discourse.org/u/erlend_sh> Team
+April 5
+
+
+
+Hi Jef,
+
+Is your University a legally recognised educational institution? Otherwise I'm afraid you're not eligible for this discount.
+
+Sincerely,
+
+Erlend
+
+
+
+This email message and any attachments may contain confidential information and may be privileged. If you are not the intended recipient or otherwise not authorized to receive this message, you are prohibited to use, copy, disclose or take any action based on this email or any information contained herein. If you are not the intended recipient, please advise the sender immediately by replying to this email and permanently delete this message and any attachments from your system.

data/test/elided/embedded_ception.txt

@@ -0,0 +1,38 @@
+On Mon, Feb 1, 2016 at 6:32 PM, Jeff Atwood <info@discourse.org> wrote:
+
+> This is Jeff's reply.
+>
+> On Mon, Feb 1, 2016 at 7:50 AM, Some One <foo@bar.com > > wrote:
+>
+>> Great!
+>>
+>> Many thanks.
+>>
+>> ~s
+>>
+>> On Mon, Feb 1, 2016 at 5:05 AM Discourse Team <team@discourse.org> wrote:
+>>
+>>> WAT?
+>>>
+>>> On Wed, Jan 27, 2016 at 10:48 PM, Some One < >>> foo@bar.com> wrote:
+>>>
+>>>> Hi Team,
+>>>>
+>>>> How is it doing?
+>>>>
+>>>> Some One
+>>>>
+>>>> On Wed, Jan 27, 2016 at 10:10 AM Discourse Team <team@discourse.org> >>>> wrote:
+>>>>
+>>>>> Hello :waves_hand:
+>>>>>
+>>>>
+>>>
+>
+
+
+--
+Some One
+Community Manager
+foo@bar.com
+(123) 456-7890

data/test/elided/embedded_email_1.txt

@@ -0,0 +1,6 @@
+On Wed, Sep 25, 2013, at 03:57 PM, richard_clark wrote:
+
+> This is the embedded email
+
+This is after the embedded email and will not show up because 99% of the times
+this is the signature...

data/test/elided/embedded_email_10.txt

@@ -0,0 +1,40 @@
+Sent from Outlook Mobile<https://foo.bar>
+
+
+
+
+On Sun, Feb 7, 2016 at 12:12 AM -0800, "Arpit Jalan" <arpit.jalan@discourse.org<mailto:arpit.jalan@discourse.org>> wrote:
+
+Hi Some,
+
+https://meta.discourse.org is now running on latest Discourse version!
+
+Regards,
+Arpit
+
+On Fri, Feb 5, 2016 at 10:43 AM Arpit Jalan <arpit.jalan@discourse.org<mailto:arpit.jalan@discourse.org>> wrote:
+Okay, sure!
+
+Arpit
+On Fri, 5 Feb 2016 at 10:42, Some One <foo@bar.com<mailto:foo@bar.com>> wrote:
+Arpit,
+   Yes that sounds good.
+
+Sent from Outlook Mobile<https://foo.bar>
+
+_____________________________
+From: Arpit Jalan <arpit.jalan@discourse.org<mailto:arpit.jalan@discourse.org>>
+Sent: Thursday, February 4, 2016 10:05 AM
+Subject: Meta Discourse update
+To: Some One <foo@bar.com<mailto:foo@bar.com>>, Discourse Team <team@discourse.org<mailto:team@discourse.org>>
+
+
+
+Hi Some One,
+
+Time to update meta to the latest Discourse version!
+
+Do you want me to take care of it?
+
+Regards,
+Arpit

data/test/elided/embedded_email_11.txt

@@ -0,0 +1,3 @@
+2016-03-03 17:21 GMT+01:00 Some One
+
+After

data/test/elided/embedded_email_12.txt

@@ -0,0 +1,8 @@
+On Thu, 31 Mar 2016 at 11:16 Some One < mailto:Some One <some.one@foo.bar> > wrote:
+
+
+Two 2
+
+On Thu, 31 Mar 2016 at 10:05 Jeff Atwood < mailto:Jeff Atwood <team@discourse.org> > wrote:
+
+Three 3

data/test/elided/embedded_email_13.txt

@@ -0,0 +1,9 @@
+At 6/16/2016 08:32 PM, you wrote:
+><https://meta.discourse.org/users/codinghorror>codinghorror
+><https://meta.discourse.org/users/codinghorror>Jeff Atwood co-founder
+>June 17
+>
+>Sorry I got a little mixed up with all the incoming replies. Are you
+>able to log in?
+>
+>Use your email address and "forgot password" if you need it reset.

data/test/elided/embedded_email_14.txt

@@ -0,0 +1,11 @@
+2016-10-24 15:36 GMT+02:00 Foo bar < info@foo.bar>:
+
+> Thank you so much Erlend, very thanks!
+>
+> 2016-10-24 15:03 GMT+02:00 Erlend Sogge Heggen <meta@discoursemail.com>:
+>
+>> erlend_sh <https://meta.discourse.org/users/erlend_sh> Erlend Sogge
+>> Heggen <https://meta.discourse.org/users/erlend_sh> team
+>> October 24
+>>
+>> I received your application and I've replied with setup instructions.

data/test/elided/embedded_email_15.txt

@@ -0,0 +1,4 @@
+2017-02-05 13:29 GMT+02:00 Very long author name < notifications@forum.some-discourse-host.local>:
+
+> [Original Messages is quoted here]
+> [...]

data/test/elided/embedded_email_16.txt

@@ -0,0 +1,4 @@
+--
+Eric Hodel - drbrain@segment7.net - http://segment7.net
+All messages signed with fingerprint:
+FEC2 57F1 D465 EB15 5D6E  7C11 332A 551C 796C 9F04

data/test/elided/embedded_email_17.txt

@@ -0,0 +1,2 @@
+On 15 May 2017 19:32, "Nomadic Sprite" <nomadic.sprite01@gmail.com> wrote:
+On 8 May 2017 6:51 pm, "Andy Jones" <Andy.Jones@jameshall.co.uk> wrote:

data/test/elided/embedded_email_18.txt

@@ -0,0 +1 @@
+On 8 May 2017 17:34, "Andy Jones" <Andy.Jones@jameshall.co.uk> wrote:

data/test/elided/embedded_email_2.txt

@@ -0,0 +1,9 @@
+---- On Tue, 22 Dec 2015 14:17:36 +0530 Sam Saffron<info@discourse.org> wrote ----
+
+
+         sam Sam Saffron co-founder
+ December 22
+   You are not using the right endpoint, go to user profile / badges, have a look at dev tools to see what it calls
+ There is a discrete endpoint to get all the badges a user has
+
+   To respond, reply to this email or visit the topic.

data/test/elided/embedded_email_3.txt

@@ -0,0 +1,14 @@
+On Thu, Jun 20, 2013 at 4:18 PM, matt2 via Discourse Meta < info@discourse.org> wrote:
+
+> matt2 posted in 'Discourse on Ubuntu: Video Walkthrough' on Discourse Meta:
+> ------------------------------
+>
+> Has anyone tried on AWS? a public AMI would be awesome.
+> ------------------------------
+>
+> Please visit this link to respond:
+> http://meta.discourse.org/t/discourse-on-ubuntu-video-walkthrough/7478/4
+>
+> To unsubscribe from these emails, visit your user preferences<http://meta.discourse.org/user_preferences>
+> .
+>

data/test/elided/embedded_email_4.txt

@@ -0,0 +1,15 @@
+2013/7/20 sam via Discourse Meta <info@discourse.org>
+
+> sam replied to your post in 'Unable to create group' on Discourse Meta:
+> ------------------------------
+>
+> Is this an error handling thing, if you name the group "test" and only
+> place yourself in it, does it persist?
+> ------------------------------
+>
+> To respond, reply to this email or visit
+> http://meta.discourse.org/t/unable-to-create-group/8198/6 in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences<http://meta.discourse.org/user_preferences>
+> .
+>

data/test/elided/embedded_email_5.txt

@@ -0,0 +1,3 @@
+Some One <info@discourse.org> wrote:
+
+Something that will be considered as the embedded email.

data/test/elided/embedded_email_6.txt

@@ -0,0 +1,3 @@
+On 28 Jan 2016, at 19:04, Some One wrote:
+
+> Did you write something interesting?

data/test/elided/embedded_email_7.txt

@@ -0,0 +1,10 @@
+On Tue, 2011-03-01 at 18:02 +0530, Some One wrote:
+
+>
+> This is another part of the embedded email.
+>
+>
+
+
+_______________________
+And here's my signature.

data/test/elided/embedded_email_8.txt

@@ -0,0 +1,3 @@
+-------- Original Message --------
+
+THE END.

data/test/elided/embedded_email_9.txt

@@ -0,0 +1,3 @@
+*-------- Original Message --------*
+
+THE END.

data/test/elided/embedded_email_chinese.txt

@@ -0,0 +1,4 @@
+> 在 2016年12月12日，下午8:45，Erlend Sogge Heggen <meta@discoursemail.com> 写道：
+> fu.zhang:
+> Some random question
+>

data/test/elided/embedded_email_dutch_1.txt

@@ -0,0 +1,10 @@
+> Op 2 feb. 2015 om 05:28 heeft VannillaSky <info@discourse.org> het volgende geschreven:
+>
+>
+> VannillaSky
+> February 2
+> Solved. Going forward...
+>
+> To respond, reply to this email or visit https://meta.discourse.org/t/import-posts-from-facebook-group-into-discourse/6089/33 in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences.