email_reply_trimmer 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7e62f9968467e01fa3a5889f8be9cf71de984d48
+  data.tar.gz: 0a04448f2c487e2b8086e511d82106f5392cdd8a
+SHA512:
+  metadata.gz: 76db5365d1c3ae729a862ed55f4f704c529afaf8e25b925865ddb60967ff0c3ca36e69ce95c414b7c01609b1cd4ae42b09299f14cebc521fe84d78ca336972dc
+  data.tar.gz: 637c4cbf1df229ef073786d527a0d501013144c3e618b499309e4701b994b9a5fd917a032e2ef744e69a5800be83b734cd056aaef418ac544ab060625d0b5059

data/LICENSE

@@ -0,0 +1,9 @@
+The MIT License (MIT)
+
+Copyright (c) Discourse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,21 @@
+# Discourse Email Reply Trimmer
+
+EmailReplyTrimmer is a small library to trim replies from plain text email.
+
+## Usage
+
+To trim replies:
+
+`trimmed_body = EmailReplyTrimmer.trim(email_body)`
+
+## Installation
+
+Get it from [GitHub](https://github.com/discourse/email_reply_trimmer).
+
+Run `rake` to run the tests.
+
+## Inspirations
+
+ - [GitHub's Email Reply Parser](https://github.com/github/email_reply_parser)
+ - [MailGun's Talon](https://github.com/mailgun/talon)
+ - [Vitor R. Carvalho's Learning to Extract Signature and Reply Lines from Email](http://www.cs.cmu.edu/~vitor/papers/sigFilePaper_finalversion.pdf)

data/Rakefile

@@ -0,0 +1,12 @@
+def name
+  @name ||= Dir["*.gemspec"].first.split(".").first
+end
+
+def version
+  @version ||= File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*['"](?'version'\d+\.\d+\.\d+)['"]/, "version"]
+end
+
+task :default => :test
+
+require "rake/testtask"
+Rake::TestTask.new(:test)

data/email_reply_trimmer.gemspec

@@ -0,0 +1,20 @@
+require_relative "lib/email_reply_trimmer"
+
+Gem::Specification.new do |s|
+  s.name = "email_reply_trimmer"
+  s.version = EmailReplyTrimmer::VERSION
+  s.date = Time.now.strftime('%Y-%m-%d')
+
+  s.summary = "Library to trim replies from plain text email."
+  s.description = "EmailReplyTrimmer is a small library to trim replies from plain text email."
+
+  s.authors = ["Régis Hanol"]
+  s.email = ["regis+rubygems@hanol.fr"]
+
+  s.homepage = "https://github.com/discourse/email_reply_trimmer"
+  s.license = "MIT"
+
+  s.require_paths = ["lib"]
+  s.files = Dir["**/*"].reject { |path| File.directory?(path) }
+  s.test_files = s.files.select { |path| path =~ /^test\/.+_test\.rb$/ }
+end

data/lib/email_reply_trimmer/delimiter_matcher.rb

@@ -0,0 +1,10 @@
+class DelimiterMatcher
+
+  DELIMITER_CHARACTERS ||= ['-', '_', '=', '+','~', '#', '*', 'ᐧ']
+  DELIMITER_REGEX      ||= /^[[:space:]]*[#{Regexp.escape(DELIMITER_CHARACTERS.join)}]+[[:space:]]*$/
+
+  def self.match?(line)
+    line =~ DELIMITER_REGEX
+  end
+
+end

data/lib/email_reply_trimmer/email_header_matcher.rb

@@ -0,0 +1,46 @@
+class EmailHeaderMatcher
+
+  EMAIL_HEADERS_WITH_DATE_MARKERS = [
+    # Dutch
+    ["Sendt"],
+    # English
+    ["Sent"],
+    # French
+    ["Date"],
+    # Portuguese
+    ["Enviada em"],
+    # Spanish (Mexican)
+    ["Fecha"],
+  ]
+
+  EMAIL_HEADERS_WITH_DATE_REGEXES = EMAIL_HEADERS_WITH_DATE_MARKERS.map do |header|
+    /^[[:space:]\*]*(?:#{header.join("|")})[[:space:]\*]*:.*\d+/
+  end
+
+  EMAIL_HEADERS_WITH_TEXT_MARKERS = [
+    # Dutch
+    ["Fra", "Til", "Emne"],
+    # English
+    ["From", "To", "Cc", "Reply-To", "Subject"],
+    # French
+    ["De", "À", "Répondre à", "Objet"],
+    # Portuguese
+    ["De", "Para", "Assunto"],
+    # Spanish (Mexican)
+    ["De", "Para", "Asunto"],
+  ]
+
+  EMAIL_HEADERS_WITH_TEXT_REGEXES = EMAIL_HEADERS_WITH_TEXT_MARKERS.map do |header|
+    /^[[:space:]\*]*(?:#{header.join("|")})[[:space:]\*]*:.*[[:word:]]+/
+  end
+
+  EMAIL_HEADER_REGEXES = [
+    EMAIL_HEADERS_WITH_DATE_REGEXES,
+    EMAIL_HEADERS_WITH_TEXT_REGEXES,
+  ].flatten
+
+  def self.match?(line)
+    EMAIL_HEADER_REGEXES.any? { |r| line =~ r }
+  end
+
+end

data/lib/email_reply_trimmer/embedded_email_matcher.rb

@@ -0,0 +1,108 @@
+class EmbeddedEmailMatcher
+
+  # On Wed, Sep 25, 2013, at 03:57 PM, jorge_castro wrote:
+  # On Thursday, June 27, 2013, knwang via Discourse Meta wrote:
+  # On Wed, 2015-12-02 at 13:58 +0000, Tom Newsom wrote:
+  # On 10/12/15 12:30, Jeff Atwood wrote:
+  # ---- On Tue, 22 Dec 2015 14:17:36 +0530 Sam Saffron<info@discourse.org> wrote ----
+  # Op 24 aug. 2013 om 16:48 heeft ven88 via Discourse Meta <info@discourse.org> het volgende geschreven:
+  # Le 4 janv. 2016 19:03, "Neil Lalonde" <info@discourse.org> a écrit :
+  # Dnia 14 lip 2015 o godz. 00:25 Michael Downey <info@discourse.org> napisał(a):
+  # Em seg, 27 de jul de 2015 17:13, Neil Lalonde <info@discourse.org> escreveu:
+  # El jueves, 21 de noviembre de 2013, codinghorror escribió:
+  ON_DATE_SOMEONE_WROTE_MARKERS = [
+    # Dutch
+    ["Op","het volgende geschreven"],
+    # English
+    ["On", "wrote"],
+    # French
+    ["Le", "a écrit "],
+    # Polish
+    ["Dnia", "napisał\\(a\\)"],
+    # Portuguese
+    ["Em", "escreveu"],
+    # Spanish
+    ["El", "escribió"],
+  ]
+
+  ON_DATE_SOMEONE_WROTE_REGEXES = ON_DATE_SOMEONE_WROTE_MARKERS.map do |on, wrote|
+    wrote.gsub!(" ", "[[:space:]]") # the "wrote" part might span over multiple lines
+    /^([ >\t\-]*#{on}\s(?:(?!#{on}\s|#{wrote}:?)[\s\S])*#{wrote}:?[ \t\-]*)$/m
+  end
+
+  # Op 10 dec. 2015 18:35 schreef "Arpit Jalan" <info@discourse.org>:
+  # Am 18.09.2013 um 16:24 schrieb codinghorror <info@discourse.org>:
+  ON_DATE_WROTE_SOMEONE_MARKERS = [
+    # Dutch
+    ["Op", "schreef"],
+    # German
+    ["Am", "schrieb"],
+  ]
+
+  ON_DATE_WROTE_SOMEONE_REGEXES = ON_DATE_WROTE_SOMEONE_MARKERS.map do |on, wrote|
+    /^[[:space:]]*#{on}\s.+\s#{wrote}\s[^:]+:/
+  end
+
+  # суббота, 14 марта 2015 г. пользователь etewiah написал:
+  DATE_SOMEONE_WROTE_MARKERS = [
+    # Russian
+    ["пользователь", "написал"],
+  ]
+
+  DATE_SOMEONE_WROTE_REGEXES = DATE_SOMEONE_WROTE_MARKERS.map do |user, wrote|
+    /[[:space:]]*.+#{user}.+#{wrote}:/
+  end
+
+  # 2015-10-18 0:17 GMT+03:00 Matt Palmer <info@discourse.org>:
+  # 2013/10/2 camilohollanda <info@discourse.org>
+  # вт, 5 янв. 2016 г. в 23:39, Erlend Sogge Heggen <info@discourse.org>:
+  # ср, 1 апр. 2015, 18:29, Denis Didkovsky <info@discourse.org>:
+  DATE_SOMEONE_EMAIL_REGEX = /^[[:space:]]*.*\d+.+<.+@.+\..+>:?/
+
+  # codinghorror via Discourse Meta wrote:
+  # codinghorror via Discourse Meta <info@discourse.org> schrieb:
+  SOMEONE_VIA_SOMETHING_WROTE_MARKERS = [
+    # English
+    "wrote",
+    # German
+    "schrieb",
+  ]
+
+  SOMEONE_VIA_SOMETHING_WROTE_REGEXES = SOMEONE_VIA_SOMETHING_WROTE_MARKERS.map do |wrote|
+    /^[[:space:]]*.+ via .+ #{wrote}:?[[:space:]]*$/
+  end
+
+  # Some One <info@discourse.org> wrote:
+  SOMEONE_EMAIL_WROTE_REGEX = /^[[:space:]]*.+ <.+@.+\..+> wrote:?/
+
+  # Posted by mpalmer on 01/21/2016
+  POSTED_BY_SOMEONE_ON_DATE_REGEX = /^[[:space:]]*Posted by .+ on \d{2}\/\d{2}\/\d{4}$/i
+
+  # Begin forwarded message:
+  # Reply Message
+  # ----- Forwarded Message -----
+  # ----- Original Message -----
+  # -----Original Message-----
+  FORWARDED_EMAIL_REGEXES = [
+    # English
+    /^[[:space:]]*Begin forwarded message:/i,
+    /^[[:space:]]*Reply message/i,
+    /^[[:space:]]*-{2,}[[:space:]]*(Forwarded|Original) Message[[:space:]]*-{2,}/i
+  ]
+
+  EMBEDDED_REGEXES = [
+    ON_DATE_SOMEONE_WROTE_REGEXES,
+    ON_DATE_WROTE_SOMEONE_REGEXES,
+    DATE_SOMEONE_WROTE_REGEXES,
+    DATE_SOMEONE_EMAIL_REGEX,
+    SOMEONE_VIA_SOMETHING_WROTE_REGEXES,
+    SOMEONE_EMAIL_WROTE_REGEX,
+    POSTED_BY_SOMEONE_ON_DATE_REGEX,
+    FORWARDED_EMAIL_REGEXES,
+  ].flatten
+
+  def self.match?(line)
+    EMBEDDED_REGEXES.any? { |r| line =~ r }
+  end
+
+end

data/lib/email_reply_trimmer/empty_line_matcher.rb

@@ -0,0 +1,7 @@
+class EmptyLineMatcher
+
+  def self.match?(line)
+    line =~ /^[[:space:]]*$/
+  end
+
+end

data/lib/email_reply_trimmer/quote_matcher.rb

@@ -0,0 +1,7 @@
+class QuoteMatcher
+
+  def self.match?(line)
+    line =~ /^[[:space:]]*>/
+  end
+
+end

data/lib/email_reply_trimmer/signature_matcher.rb

@@ -0,0 +1,33 @@
+class SignatureMatcher
+
+  # Envoyé depuis mon iPhone
+  # Von meinem Mobilgerät gesendet
+  # Diese Nachricht wurde von meinem Android-Mobiltelefon mit K-9 Mail gesendet.
+  # Nik from mobile
+  # From My Iphone 6
+  # Sent via mobile
+  # Sent with Airmail
+  # Sent from Windows Mail
+  # Sent from my TI-85
+  # <<sent by galaxy>>
+  # (sent from a phone)
+  # (Sent from mobile device)
+  SIGNATURE_REGEXES = [
+    # English
+    /^[[:space:]]*[[:word:]]+ from mobile/i,
+    /^[[:space:]]*sent (?:from|via|with|by) .+/i,
+    /^[[:space:]]*\(sent (?:from|via|with|by) .+\)/i,
+    /^[[:space:]]*<<sent (?:from|via|with|by) .+>>/i,
+    /^[[:space:]]*from my .{1,20}/i, # don't match too much
+    # French
+    /^[[:space:]]*Envoyé depuis mon .+/i,
+    # German
+    /^[[:space:]]*Von meinem Mobilgerät gesendet/i,
+    /^[[:space:]]*Diese Nachricht wurde von .+ gesendet/i,
+  ]
+
+  def self.match?(line)
+    SIGNATURE_REGEXES.any? { |r| line =~ r }
+  end
+
+end

data/lib/email_reply_trimmer.rb

@@ -0,0 +1,100 @@
+require_relative "email_reply_trimmer/empty_line_matcher"
+require_relative "email_reply_trimmer/delimiter_matcher"
+require_relative "email_reply_trimmer/signature_matcher"
+require_relative "email_reply_trimmer/embedded_email_matcher"
+require_relative "email_reply_trimmer/email_header_matcher"
+require_relative "email_reply_trimmer/quote_matcher"
+
+class EmailReplyTrimmer
+  VERSION = "0.0.1"
+
+  DELIMITER    = "d"
+  EMBEDDED     = "b"
+  EMPTY        = "e"
+  EMAIL_HEADER = "h"
+  QUOTE        = "q"
+  SIGNATURE    = "s"
+  TEXT         = "t"
+
+  def self.identify_line_content(line)
+    return EMPTY        if EmptyLineMatcher.match?(line)
+    return DELIMITER    if DelimiterMatcher.match?(line)
+    return SIGNATURE    if SignatureMatcher.match?(line)
+    return EMBEDDED     if EmbeddedEmailMatcher.match?(line)
+    return EMAIL_HEADER if EmailHeaderMatcher.match?(line)
+    return QUOTE        if QuoteMatcher.match?(line)
+    return TEXT
+  end
+
+  def self.trim(text)
+    # normalize line endings
+    text = text.gsub!(/\r*\n/, "\n")
+
+    # fix embedded email markers that might span over multiple lines
+    EmbeddedEmailMatcher::ON_DATE_SOMEONE_WROTE_REGEXES.each do |r|
+      next unless text =~ r
+      text.gsub!($1, $1.gsub(/\n[[:space:]>\-]*/, " "))
+    end
+
+    # from now on, we'll work on a line-by-line basis
+    lines = text.split("\n")
+
+    # identify content of each lines
+    pattern = lines.map { |l| identify_line_content(l) }.join
+
+    # remove all signatures & delimiters
+    while pattern =~ /[ds]/
+      index = pattern =~ /[ds]/
+      pattern.slice!(index)
+      lines.slice!(index)
+    end
+
+    # if there is an embedded email marker, followed only by a quote and eventually some text
+    # then take everything up to that marker
+    if pattern =~ /t[eq]*b[eq]*[et]*$/
+      index = pattern =~ /t[eq]*b[eq]*[et]*$/
+      pattern = pattern[0..index]
+      lines = lines[0..index]
+    end
+
+    # if there still are some embedded email markers, just empty them
+    while pattern =~ /b/
+      index = pattern =~ /b/
+      pattern[index] = "e"
+      lines[index] = ""
+    end
+
+    # fix email headers when they span over multiple lines
+    if pattern =~ /hh[ht]+e/
+      index = pattern =~ /hh[ht]+e/
+      size = pattern[/hh[ht]+/].size
+      size.times.each { |s| pattern[index + s] = EMAIL_HEADER }
+    end
+
+    # if there are at least 3 consecutive email headers, take everything up to
+    # these headers
+    if pattern =~ /t[eq]*h{3,}/
+      index = pattern =~ /t[eq]*h{3,}/
+      pattern = pattern[0..index]
+      lines = lines[0..index]
+    end
+
+    # if there still are some email headers, just remove them
+    while pattern =~ /h/
+      index = pattern =~ /h/
+      pattern.slice!(index)
+      lines.slice!(index)
+    end
+
+    # remove trailing quotes when there's at least one line of text
+    if pattern =~ /t/ && pattern =~ /[eq]+$/
+      index = pattern =~ /[eq]+$/
+      pattern = pattern[0...index]
+      lines = lines[0...index]
+    end
+
+    # result
+    lines.join("\n").strip
+  end
+
+end

data/test/emails/delimiters.txt

@@ -0,0 +1,14 @@
+This is not a ---------- delimiter.
+
+But there should be no delimiter after this line
+
+ᐧ
+--
+***
+####
+~~~~~
+======
+_______
+++++++++
+
+   -------

data/test/emails/email_headers_1.txt

@@ -0,0 +1,17 @@
+This is a reply from Outlook!
+
+
+ ------------------------------
+
+*From:* Outlook user
+*Sent:* 2016-01-27
+*To:* info@discourse.org
+*Subject:* Are you using Outlook?
+
+
+
+Just some random text wich is part
+
+of the embedded email...
+
+Outlook user

data/test/emails/email_headers_2.txt

@@ -0,0 +1,10 @@
+This is a reply from Outlook!
+________________________________________
+From: Discourse <info@discourse.org.
+Sent: Thursday, 28 January 2016 8:16 p.m.
+To: Someone
+Subject: (no subject)
+
+Lorem ipsum.
+
+Ipsor dolor sum met.

data/test/emails/embedded_email_1.txt

@@ -0,0 +1,8 @@
+This is before the embedded email.
+
+On Wed, Sep 25, 2013, at 03:57 PM, richard_clark wrote:
+
+> This is the embedded email
+
+This is after the embedded email and will not show up because 99% of the times
+this is the signature...

data/test/emails/embedded_email_2.txt

@@ -0,0 +1,16 @@
+I have checked the available documentation/links to find out the end points. If there are some other endpoints, can you please tell the endpoints
+
+Best Regards
+
+
+
+
+---- On Tue, 22 Dec 2015 14:17:36 +0530 Sam Saffron<info@discourse.org> wrote ----
+
+
+         sam Sam Saffron co-founder
+ December 22
+   You are not using the right endpoint, go to user profile / badges, have a look at dev tools to see what it calls
+ There is a discrete endpoint to get all the badges a user has
+
+   To respond, reply to this email or visit the topic.

data/test/emails/embedded_email_3.txt

@@ -0,0 +1,24 @@
+Sure, [Bitnami](http://bitnami.com/stack/discourse) supports EC2 and there
+are several topics here about it.
+
+Our [official install guide](
+https://github.com/discourse/discourse/blob/master/docs/INSTALL-ubuntu.md)
+also works great on an Amazon EC2 provisioned virtual machine with very
+little change.
+
+
+On Thu, Jun 20, 2013 at 4:18 PM, matt2 via Discourse Meta <
+info@discourse.org> wrote:
+
+> matt2 posted in 'Discourse on Ubuntu: Video Walkthrough' on Discourse Meta:
+> ------------------------------
+>
+> Has anyone tried on AWS? a public AMI would be awesome.
+> ------------------------------
+>
+> Please visit this link to respond:
+> http://meta.discourse.org/t/discourse-on-ubuntu-video-walkthrough/7478/4
+>
+> To unsubscribe from these emails, visit your user preferences<http://meta.discourse.org/user_preferences>
+> .
+>

data/test/emails/embedded_email_4.txt

@@ -0,0 +1,19 @@
+If I do exactly that, then the test group won't be saved after a refresh of
+the page.
+
+
+2013/7/20 sam via Discourse Meta <info@discourse.org>
+
+> sam replied to your post in 'Unable to create group' on Discourse Meta:
+> ------------------------------
+>
+> Is this an error handling thing, if you name the group "test" and only
+> place yourself in it, does it persist?
+> ------------------------------
+>
+> To respond, reply to this email or visit
+> http://meta.discourse.org/t/unable-to-create-group/8198/6 in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences<http://meta.discourse.org/user_preferences>
+> .
+>

data/test/emails/embedded_email_5.txt

@@ -0,0 +1,5 @@
+I am testing the SOMEONE_EMAIL_WROTE_REGEX regular expression.
+
+Some One <info@discourse.org> wrote:
+
+Something that will be considered as the embedded email.

data/test/emails/embedded_email_6.txt

@@ -0,0 +1,11 @@
+Ensuring that ON_DATE_SOMEONE_WROTE_REGEXES is not greedy.
+
+On this day
+
+I wrote something
+
+which was very interesting.
+
+On 28 Jan 2016, at 19:04, Some One wrote:
+
+> Did you write something interesting?

data/test/emails/embedded_email_7.txt

@@ -0,0 +1,20 @@
+This is a line before the embedded email.
+On Tue, 2011-03-01 at 18:02 +0530, Some One wrote:
+> Hello
+>
+> This is the embedded email.
+
+This is some text
+
+after the
+
+embedded email.
+
+>
+> This is another part of the embedded email.
+>
+>
+
+
+_______________________
+And here's my signature.

data/test/emails/embedded_email_8.txt

@@ -0,0 +1,5 @@
+Foo
+
+-------- Original Message --------
+
+THE END.

data/test/emails/embedded_email_dutch_1.txt

@@ -0,0 +1,13 @@
+Nice of you to pick it up! Could you make a PR? Then other people can use it too. Would be great.
+
+
+> Op 2 feb. 2015 om 05:28 heeft VannillaSky <info@discourse.org> het volgende geschreven:
+>
+>
+> VannillaSky
+> February 2
+> Solved. Going forward...
+>
+> To respond, reply to this email or visit https://meta.discourse.org/t/import-posts-from-facebook-group-into-discourse/6089/33 in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences.

data/test/emails/embedded_email_dutch_2.txt

@@ -0,0 +1,62 @@
+The adblocker idea from @elberet might be it. Regretfully I only have time
+to test this in 2 weeks. I will let you know then...
+Op 16 aug. 2014 05:13 schreef "codinghorror" <info@discourse.org> het
+volgende geschreven:
+
+>     codinghorror <https://meta.discourse.org/users/codinghorror>
+> August 15
+>
+> Hmm, if that is true we need to fix that somehow.
+>
+> To respond, reply to this email or visit
+> https://meta.discourse.org/t/configuring-google-oauth2-login-for-discourse/15858/38
+> in your browser.
+>  ------------------------------
+> Previous Replies    elberet <https://meta.discourse.org/users/elberet>
+> August 15
+>
+> Are you using an ad blocker? Adblock Plus, for instance, removes the login
+> buttons.
+>      bwvanmanen <https://meta.discourse.org/users/bwvanmanen>
+> August 15
+>
+> The 'Sign in with Google' button does appear when I try to log in from
+> another pc. However the button does not appear on my pc using another
+> browser.
+>      bwvanmanen <https://meta.discourse.org/users/bwvanmanen>
+> August 15
+>
+> I followed the steps from the opening post, but I don't get a google login
+> option when users try to log in. There is just the normal user/password,
+> the word OR, and nothing next to that. Any clues?
+>
+>       neil <https://meta.discourse.org/users/neil>
+> August 1
+>
+> Glad you got it fixed. It's good to have the resolution in this topic too.
+> I added a link to your topic in the first post.
+>      jgehrcke <https://meta.discourse.org/users/jgehrcke>
+> July 31
+>
+> Sorry, that was kind of a cross-posting. Indeed, I got that fixed as you
+> say, as reported here:
+>   jgehrcke said:
+>
+> I understand the solution you are proposing. The proper fix, without
+> customizing the disource.conf in the container, however, would be to inform
+> the web application about the protocol used via an HTTP header, as you are
+> also suggesting. The question is: does Discourse respect/support
+> X-Forwarded-Protocol? Then I can easily configure the nginx reverse proxy
+> with proxy_set_header X-Forwarded-Protocol $scheme; Edit: I can confirm
+> that properly setting the X-Forwarded-Proto header in the nginx …
+>
+>  Thanks and sorry for not reporting back here!
+>   ------------------------------
+>
+> To respond, reply to this email or visit
+> https://meta.discourse.org/t/configuring-google-oauth2-login-for-discourse/15858/38
+> in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences
+> <https://meta.discourse.org/my/preferences>.
+>

data/test/emails/embedded_email_french.txt

@@ -0,0 +1,12 @@
+C'est super !
+Le 4 janv. 2016 19:03, "Neil Lalonde"
+<info@discourse.org>
+a écrit :
+
+> <https://meta.discourse.org/users/neil> team
+> January 4
+>
+> Discourse c'est top !
+>
+> Neil
+>

data/test/emails/embedded_email_german.txt

@@ -0,0 +1,26 @@
+alright, no big deal - I assume in this case the code needs some refactoring to better support I18N anyway.
+
+
+
+codinghorror via Discourse Meta <info@discourse.org> schrieb:
+>codinghorror posted in 'Visually separate "Like it too" / "Undo like"'
+>on Discourse Meta:
+>
+>---
+>So turns out this is sort of a complex change for something so small.
+>Our code at that location assumes all phrases end in periods. Changing
+>that conditionally to sometimes an em-dash, sometimes not, is kind of..
+>a pain in the butt.
+>
+>Will keep an eye on it, but low priority given the work required.
+>
+>---
+>To respond, reply to this email or visit
+>http://meta.discourse.org/t/visually-separate-like-it-too-undo-like/8464/3
+>in your browser.
+>
+>To unsubscribe from these emails, visit your [user
+>preferences](http://meta.discourse.org/user_preferences).
+
+--
+Diese Nachricht wurde von meinem Android-Mobiltelefon mit K-9 Mail gesendet.

data/test/emails/embedded_email_polish.txt

@@ -0,0 +1,34 @@
+Oh, I've forgot to add. MIT
+
+--
+Łukasz Jan Niemier
+
+Dnia 14 lip 2015 o godz. 00:25 Michael Downey <info@discourse.org> napisał(a):
+
+>
+>   downey Michael Downey Senior Tester
+> July 13
+> Sounds interesting. What is the license?
+>
+> To respond, reply to this email or visit https://meta.discourse.org/t/static-pages-plugin/31027/2 in your browser.
+>
+> Previous Replies
+>
+>   hauleth
+> July 13
+> WIP
+>
+> Location: https://github.com/hauleth/low_voltage
+>
+> Authors: @hauleth
+>
+> Simple plugin to add "static" pages to Discourse instance. For now usage is kind of complicated at it needs to fetch static pages from app/views/pages. Ideas for future implements:
+>
+> Create DB model to hold pages
+> Adding admin page that will allow editing that pages
+> Custom styles for static pages
+> Any help appreciated.
+>
+> To respond, reply to this email or visit https://meta.discourse.org/t/static-pages-plugin/31027/2 in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences.

data/test/emails/embedded_email_portuguese.txt

@@ -0,0 +1,18 @@
+Olá,
+Qualquer dúvida estamos ai.
+
+Em 2 de fevereiro de 2015 10:58, discourse <
+info@discourse.org> escreveu:
+
+>    discourse <https://meta.discourse.org/users/discourse>
+> February 2
+>
+> Bom dia. Obrigado pela atenção
+>   ------------------------------
+>
+> To respond, reply to this email or visit
+> https://meta.discourse.org/ in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences
+> <https://meta.discourse.org/my/preferences>.
+>

data/test/emails/embedded_email_quote_text.txt

@@ -0,0 +1,10 @@
+On Mon, Aug 19, 2013 at 2:36 AM, SomeOne via Discourse Meta <
+info@discourse.org> wrote:
+
+> This seems like a problem that would be better solved at the web server
+> level, rather than by the application. If nothing else, adding an instance
+> of Nginx as an SSL/TLS reverse-proxy would very quickly take care of this.
+>
+
+when I said looks possible, I pointed to rack because that's still the web
+server level

data/test/emails/embedded_email_russian.txt

@@ -0,0 +1,27 @@
+Why don't you override particular strings with JS?
+
+понедельник, 6 октября 2014 г. пользователь lidel написал:
+
+>     lidel <https://meta.discourse.org/users/lidel>
+> October 6
+>  winterbox:
+>
+> Do you have any plan to make this plugin?
+>
+>  discourse-locale-override
+> <https://github.com/lidel/discourse-locale-override> is just a hack, and
+> as @sam <https://meta.discourse.org/users/sam> pointed out, it only works
+> if you update your instance via commandline. I created it to speed up my
+> work on pl_PL translation only.
+>
+> I see there is a need for a proper plugin (this and other threads), but
+> until such plugin appears this is a semi-working workaround.
+>   ------------------------------
+>
+> To respond, reply to this email or visit
+> https://meta.discourse.org/t/use-custom-translation-file/13786/41 in your
+> browser.
+>
+> To unsubscribe from these emails, visit your user preferences
+> <https://meta.discourse.org/my/preferences>.
+>

data/test/emails/embedded_email_spanish.txt

@@ -0,0 +1,41 @@
+I've configured Twitter Authentication, so this error happends on the
+application, after twitter posts to my discourse install via the callback.
+
+In the front error I can get a correct message saying "Something waa wrong,
+check you usename and password" and in the backend I see that error in the
+logs.
+
+
+
+El jueves, 21 de noviembre de 2013, codinghorror escribió:
+
+>    codinghorror <http://meta.discourse.org/users/codinghorror>
+> November 21
+>
+> How are you creating this user? Describe it step by step.
+>
+> To respond, reply to this email or visit
+> http://meta.discourse.org/t/error-creating-new-users-pg-numericvalueoutofrange/11046/2in your browser.
+>  ------------------------------
+> Previous Replies    PabloC <http://meta.discourse.org/users/pabloc>
+> November 21
+>
+> Hi guys,
+>
+> While I create a new user, I'm finding this error in the logs :
+>
+> Processing by UsersController#create as */*
+>   Parameters: {"name"=>"Ps4 Support Forum", "email"=>"pablocorral+ps4@gmail.com <javascript:_e({}, 'cvml', 'pablocorral%2Bps4@gmail.com');>", "username"=>"PS4SupportForum", "password_confirmation"=>"[FILTERED]", "challenge"=>"11a4ebe83d9b2a9e0d45bc50b457ee17"}
+> PG::NumericValueOutOfRange: ERROR:  value "2201945804" is out of range for type integer
+>
+> Any idea?
+>
+> Tks!
+>   ------------------------------
+>
+> To respond, reply to this email or visit
+> http://meta.discourse.org/t/error-creating-new-users-pg-numericvalueoutofrange/11046/2in your browser.
+>
+> To unsubscribe from these emails, visit your user preferences<http://meta.discourse.org/user_preferences>
+> .
+>

data/test/emails/forwarded_message.txt

@@ -0,0 +1,9 @@
+---------- Forwarded message ----------
+From: Some One <foo@bar.com>
+Date: Thu, Jan 28, 2016 at 4:00 PM
+Subject: Some subject that
+spans over 2 lines
+To: infod@discourse.org
+
+
+This is a forwarded email. And just that.

data/test/emails/normalize_line_endings.txt

@@ -0,0 +1,4 @@
+Email with
+windows
+line
+endings

data/test/emails/quote_and_text.txt

@@ -0,0 +1,3 @@
+> This is a quote.
+
+And this is some text.

data/test/emails/quote_only.txt

@@ -0,0 +1 @@
+> Email with only quote.

data/test/emails/retains_spaces_and_formatting.txt

@@ -0,0 +1,14 @@
+Formatting and
+
+spaces
+
+- A
+- B
+- C
+
+should be retained
+
+
+    - Item #1
+    - Item #2
+

data/test/emails/signatures.txt

@@ -0,0 +1,27 @@
+This email was sent from way too much different devices ;)
+
+Envoyé depuis mon iPhone
+
+Von meinem Mobilgerät gesendet
+Diese Nachricht wurde von meinem Android-Mobiltelefon mit K-9 Mail gesendet.
+
+Someone from mobile
+From My Iphone 6
+Sent via mobile
+Sent with Airmail
+Sent from Windows Mail
+Sent from Mailbox
+Sent from Mailbox for iPad
+Sent from Yahoo Mail on Android
+Sent from my TI-85
+Sent from my iPhone
+Sent from my iPod
+Sent from my Alcatel Flash2
+Sent from my mobile device
+Sent from my cell, please excuse any typos.
+Sent from my Samsung Galaxy s5 Octacore device
+Sent from my HTC M8 Android phone.  Please excuse typoze
+Sent from my Windows 8 PC <http://windows.microsoft.com/consumer-preview>
+<<sent by galaxy>>
+(sent from a phone)
+(Sent from mobile device)

data/test/emails/strip.txt

@@ -0,0 +1,10 @@
+
+
+
+
+
+Strip leading and trailing empty lines.
+
+
+
+

data/test/emails/text_only.txt

@@ -0,0 +1 @@
+Email with only text.

data/test/replies/delimiters.txt

@@ -0,0 +1,3 @@
+This is not a ---------- delimiter.
+
+But there should be no delimiter after this line

data/test/replies/email_headers_1.txt

@@ -0,0 +1 @@
+This is a reply from Outlook!

data/test/replies/email_headers_2.txt

@@ -0,0 +1 @@
+This is a reply from Outlook!

data/test/replies/embedded_email_1.txt

@@ -0,0 +1 @@
+This is before the embedded email.

data/test/replies/embedded_email_2.txt

@@ -0,0 +1,3 @@
+I have checked the available documentation/links to find out the end points. If there are some other endpoints, can you please tell the endpoints
+
+Best Regards

data/test/replies/embedded_email_3.txt

@@ -0,0 +1,7 @@
+Sure, [Bitnami](http://bitnami.com/stack/discourse) supports EC2 and there
+are several topics here about it.
+
+Our [official install guide](
+https://github.com/discourse/discourse/blob/master/docs/INSTALL-ubuntu.md)
+also works great on an Amazon EC2 provisioned virtual machine with very
+little change.

data/test/replies/embedded_email_4.txt

@@ -0,0 +1,2 @@
+If I do exactly that, then the test group won't be saved after a refresh of
+the page.

data/test/replies/embedded_email_5.txt

@@ -0,0 +1 @@
+I am testing the SOMEONE_EMAIL_WROTE_REGEX regular expression.

data/test/replies/embedded_email_6.txt

@@ -0,0 +1,7 @@
+Ensuring that ON_DATE_SOMEONE_WROTE_REGEXES is not greedy.
+
+On this day
+
+I wrote something
+
+which was very interesting.

data/test/replies/embedded_email_7.txt

@@ -0,0 +1,19 @@
+This is a line before the embedded email.
+
+> Hello
+>
+> This is the embedded email.
+
+This is some text
+
+after the
+
+embedded email.
+
+>
+> This is another part of the embedded email.
+>
+>
+
+
+And here's my signature.

data/test/replies/embedded_email_8.txt

@@ -0,0 +1 @@
+Foo

data/test/replies/embedded_email_dutch_1.txt

@@ -0,0 +1 @@
+Nice of you to pick it up! Could you make a PR? Then other people can use it too. Would be great.

data/test/replies/embedded_email_dutch_2.txt

@@ -0,0 +1,2 @@
+The adblocker idea from @elberet might be it. Regretfully I only have time
+to test this in 2 weeks. I will let you know then...

data/test/replies/embedded_email_french.txt

@@ -0,0 +1 @@
+C'est super !

data/test/replies/embedded_email_german.txt

@@ -0,0 +1 @@
+alright, no big deal - I assume in this case the code needs some refactoring to better support I18N anyway.

data/test/replies/embedded_email_polish.txt

@@ -0,0 +1,3 @@
+Oh, I've forgot to add. MIT
+
+Łukasz Jan Niemier

data/test/replies/embedded_email_portuguese.txt

@@ -0,0 +1,2 @@
+Olá,
+Qualquer dúvida estamos ai.

data/test/replies/embedded_email_quote_text.txt

@@ -0,0 +1,7 @@
+> This seems like a problem that would be better solved at the web server
+> level, rather than by the application. If nothing else, adding an instance
+> of Nginx as an SSL/TLS reverse-proxy would very quickly take care of this.
+>
+
+when I said looks possible, I pointed to rack because that's still the web
+server level

data/test/replies/embedded_email_russian.txt

@@ -0,0 +1 @@
+Why don't you override particular strings with JS?

data/test/replies/embedded_email_spanish.txt

@@ -0,0 +1,6 @@
+I've configured Twitter Authentication, so this error happends on the
+application, after twitter posts to my discourse install via the callback.
+
+In the front error I can get a correct message saying "Something waa wrong,
+check you usename and password" and in the backend I see that error in the
+logs.

data/test/replies/forwarded_message.txt

@@ -0,0 +1 @@
+This is a forwarded email. And just that.

data/test/replies/normalize_line_endings.txt

@@ -0,0 +1,4 @@
+Email with
+windows
+line
+endings

data/test/replies/quote_and_text.txt

@@ -0,0 +1,3 @@
+> This is a quote.
+
+And this is some text.

data/test/replies/quote_only.txt

@@ -0,0 +1 @@
+> Email with only quote.

data/test/replies/retains_spaces_and_formatting.txt

@@ -0,0 +1,14 @@
+Formatting and
+
+spaces
+
+- A
+- B
+- C
+
+should be retained
+
+
+    - Item #1
+    - Item #2
+

data/test/replies/signatures.txt

@@ -0,0 +1 @@
+This email was sent from way too much different devices ;)

data/test/replies/strip.txt

@@ -0,0 +1 @@
+Strip leading and trailing empty lines.

data/test/replies/text_only.txt

@@ -0,0 +1 @@
+Email with only text.

data/test/test_email_reply_trimmer.rb

@@ -0,0 +1,33 @@
+require "minitest/autorun"
+require "email_reply_trimmer"
+
+class TestEmailReplyTrimmer < Minitest::Test
+
+  EMAILS  = Dir["test/emails/*.txt"].map  { |path| File.basename(path) }
+  REPLIES = Dir["test/replies/*.txt"].map { |path| File.basename(path) }
+
+  def test_all_emails_have_a_matching_reply
+    assert_equal EMAILS, REPLIES, "Files in /emails and /replies folders should match 1-to-1"
+  end
+
+  def test_normalize_line_endings_email_has_windows_line_endings
+    assert_match /\r\n/, File.read("test/emails/normalize_line_endings.txt")
+  end
+
+  EMAILS.each do |filename|
+    name = File.basename(filename, ".txt")
+    define_method("test_#{name}") do
+      assert_equal trim(filename), reply(filename), "EMAIL: #{filename}"
+    end
+  end
+
+  def trim(filename)
+    body = File.read("test/emails/#{filename}")
+    EmailReplyTrimmer.trim(body)
+  end
+
+  def reply(filename)
+    File.read("test/replies/#{filename}").strip
+  end
+
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: email_reply_trimmer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Régis Hanol
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-29 00:00:00.000000000 Z
+dependencies: []
+description: EmailReplyTrimmer is a small library to trim replies from plain text
+  email.
+email:
+- regis+rubygems@hanol.fr
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- email_reply_trimmer.gemspec
+- lib/email_reply_trimmer.rb
+- lib/email_reply_trimmer/delimiter_matcher.rb
+- lib/email_reply_trimmer/email_header_matcher.rb
+- lib/email_reply_trimmer/embedded_email_matcher.rb
+- lib/email_reply_trimmer/empty_line_matcher.rb
+- lib/email_reply_trimmer/quote_matcher.rb
+- lib/email_reply_trimmer/signature_matcher.rb
+- test/emails/delimiters.txt
+- test/emails/email_headers_1.txt
+- test/emails/email_headers_2.txt
+- test/emails/embedded_email_1.txt
+- test/emails/embedded_email_2.txt
+- test/emails/embedded_email_3.txt
+- test/emails/embedded_email_4.txt
+- test/emails/embedded_email_5.txt
+- test/emails/embedded_email_6.txt
+- test/emails/embedded_email_7.txt
+- test/emails/embedded_email_8.txt
+- test/emails/embedded_email_dutch_1.txt
+- test/emails/embedded_email_dutch_2.txt
+- test/emails/embedded_email_french.txt
+- test/emails/embedded_email_german.txt
+- test/emails/embedded_email_polish.txt
+- test/emails/embedded_email_portuguese.txt
+- test/emails/embedded_email_quote_text.txt
+- test/emails/embedded_email_russian.txt
+- test/emails/embedded_email_spanish.txt
+- test/emails/forwarded_message.txt
+- test/emails/normalize_line_endings.txt
+- test/emails/quote_and_text.txt
+- test/emails/quote_only.txt
+- test/emails/retains_spaces_and_formatting.txt
+- test/emails/signatures.txt
+- test/emails/strip.txt
+- test/emails/text_only.txt
+- test/replies/delimiters.txt
+- test/replies/email_headers_1.txt
+- test/replies/email_headers_2.txt
+- test/replies/embedded_email_1.txt
+- test/replies/embedded_email_2.txt
+- test/replies/embedded_email_3.txt
+- test/replies/embedded_email_4.txt
+- test/replies/embedded_email_5.txt
+- test/replies/embedded_email_6.txt
+- test/replies/embedded_email_7.txt
+- test/replies/embedded_email_8.txt
+- test/replies/embedded_email_dutch_1.txt
+- test/replies/embedded_email_dutch_2.txt
+- test/replies/embedded_email_french.txt
+- test/replies/embedded_email_german.txt
+- test/replies/embedded_email_polish.txt
+- test/replies/embedded_email_portuguese.txt
+- test/replies/embedded_email_quote_text.txt
+- test/replies/embedded_email_russian.txt
+- test/replies/embedded_email_spanish.txt
+- test/replies/forwarded_message.txt
+- test/replies/normalize_line_endings.txt
+- test/replies/quote_and_text.txt
+- test/replies/quote_only.txt
+- test/replies/retains_spaces_and_formatting.txt
+- test/replies/signatures.txt
+- test/replies/strip.txt
+- test/replies/text_only.txt
+- test/test_email_reply_trimmer.rb
+homepage: https://github.com/discourse/email_reply_trimmer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Library to trim replies from plain text email.
+test_files: []
+has_rdoc: