discourse_email_parser 0.6.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 628e0332e75b83a5206356998c27c4195f6f09b1
+  data.tar.gz: b985069be7090e9bd87f2944a80fcc54e5ba2ae8
+SHA512:
+  metadata.gz: 472e57c5da6d459041e1a8a5f46f7c5ea2f153fb7dc742d049ad1b163713a2288d1e8d491f1657c05122e82ca826ee21481fa40c18a99fc5b71a8411a54ff061
+  data.tar.gz: 62aa7ce4d6827917f0000661ef2482da7000eec122d6c742f682dfab8e9ade254c20725592ae3e0c2d2278d8dc22ecd0c0b0b9eb943eac68ac37466ae0b21fb8

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) GitHub
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,15 @@
+# Discourse Email Parser
+
+DiscourseEmailParser is a small library to parse plain text email content.
+
+##Usage
+
+To parse reply body:
+
+`parsed_body = DiscourseEmailParser.parse_reply(email_body)`
+
+## Installation
+
+Get it from [GitHub][github].  Run `rake` to run the tests.
+
+[github]: https://github.com/discourse/discourse_email_parser

data/Rakefile

@@ -0,0 +1,130 @@
+require 'rubygems'
+require 'rake'
+require 'date'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+  Date.today.to_s
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+#############################################################################
+#
+# Standard tasks
+#
+#############################################################################
+
+task :default => :test
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -r ./lib/#{name}.rb"
+end
+
+#############################################################################
+#
+# Custom tasks (add your own tasks here)
+#
+#############################################################################
+
+
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+desc "Create tag v#{version} and build and push #{gem_file} to Rubygems"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git commit --allow-empty -a -m 'Release #{version}'"
+  sh "git tag v#{version}"
+  sh "git push origin master"
+  sh "git push origin v#{version}"
+  sh "gem push pkg/#{name}-#{version}.gem"
+end
+
+desc "Build #{gem_file} into the pkg directory"
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec => :validate do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+  replace_header(head, :date)
+  #comment this out if your rubyforge_project has a different name
+  replace_header(head, :rubyforge_project)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    map { |file| "    #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end
+
+desc "Validate #{gemspec_file}"
+task :validate do
+  libfiles = Dir['lib/*'] - ["lib/#{name}.rb", "lib/#{name}"]
+  unless libfiles.empty?
+    puts "Directory `lib` should only contain a `#{name}.rb` file and `#{name}` dir."
+    exit!
+  end
+  unless Dir['VERSION*'].empty?
+    puts "A `VERSION` file at root level violates Gem best practices."
+    exit!
+  end
+end

data/discourse_email_parser.gemspec

@@ -0,0 +1,51 @@
+$LOAD_PATH.unshift '.'
+require 'lib/discourse_email_parser'
+
+Gem::Specification.new do |s|
+  s.name              = 'discourse_email_parser'
+  s.version           = DiscourseEmailParser::VERSION
+  s.date              = Time.now.strftime('%Y-%m-%d')
+
+  s.summary     = "Small library to parse plain text email content."
+  s.description = "DiscourseEmailParser is a small library to parse plain text email content."
+
+  s.authors  = ["Rick Olson", "Arpit Jalan"]
+  s.email    = 'arpit.jalan@discourse.org'
+  s.homepage = 'http://github.com/discourse/discourse_email_parser'
+  s.license  = 'MIT'
+
+  s.require_paths = %w[lib]
+
+  s.files = %w[
+    LICENSE
+    README.md
+    Rakefile
+    discourse_email_parser.gemspec
+    lib/discourse_email_parser.rb
+    script/release
+    script/test
+    test/discourse_email_parser_test.rb
+    test/emails/correct_sig.txt
+    test/emails/email_1_1.txt
+    test/emails/email_1_2.txt
+    test/emails/email_1_3.txt
+    test/emails/email_1_4.txt
+    test/emails/email_1_5.txt
+    test/emails/email_1_6.txt
+    test/emails/email_1_7.txt
+    test/emails/email_1_8.txt
+    test/emails/email_2_1.txt
+    test/emails/email_2_2.txt
+    test/emails/email_BlackBerry.txt
+    test/emails/email_bullets.txt
+    test/emails/email_iPhone.txt
+    test/emails/email_multi_word_sent_from_my_mobile_device.txt
+    test/emails/email_one_is_not_on.txt
+    test/emails/email_sent_from_my_not_signature.txt
+    test/emails/email_sig_delimiter_in_middle_of_line.txt
+    test/emails/greedy_on.txt
+    test/emails/pathological.txt
+  ]
+
+  s.test_files = s.files.select { |path| path =~ /^test\/.*_test\.rb/ }
+end

data/lib/discourse_email_parser.rb

@@ -0,0 +1,285 @@
+require 'strscan'
+
+# https://github.com/github/email_reply_parser/blob/master/lib/email_reply_parser.rb
+#
+# DiscourseEmailParser is a small library to parse plain text email content.  The
+# goal is to identify which fragments are quoted, part of a signature, or
+# original body content.  We want to support both top and bottom posters, so
+# no simple "REPLY ABOVE HERE" content is used.
+#
+# Beyond RFC 5322 (which is handled by the [Ruby mail gem][mail]), there aren't
+# any real standards for how emails are created.  This attempts to parse out
+# common conventions for things like replies:
+#
+#     this is some text
+#
+#     On <date>, <author> wrote:
+#     > blah blah
+#     > blah blah
+#
+# ... and signatures:
+#
+#     this is some text
+#
+#     --
+#     Bob
+#     http://homepage.com/~bob
+#
+# Each of these are parsed into Fragment objects.
+#
+# DiscourseEmailParser also attempts to figure out which of these blocks should
+# be hidden from users.
+#
+# [mail]: https://github.com/mikel/mail
+class DiscourseEmailParser
+  VERSION = "0.6.0"
+
+  # Public: Splits an email body into a list of Fragments.
+  #
+  # text - A String email body.
+  #
+  # Returns an Email instance.
+  def self.read(text)
+    Email.new.read(text)
+  end
+
+  # Public: Get the text of the visible portions of the given email body.
+  #
+  # text - A String email body.
+  #
+  # Returns a String.
+  def self.parse_reply(text)
+    self.read(text).visible_text
+  end
+
+  ### Emails
+
+  # An Email instance represents a parsed body String.
+  class Email
+    # Emails have an Array of Fragments.
+    attr_reader :fragments
+
+    def initialize
+      @fragments = []
+    end
+
+    # Public: Gets the combined text of the visible fragments of the email body.
+    #
+    # Returns a String.
+    def visible_text
+      fragments.select{|f| !f.hidden?}.map{|f| f.to_s}.join("\n").rstrip
+    end
+
+    # Splits the given text into a list of Fragments.  This is roughly done by
+    # reversing the text and parsing from the bottom to the top.  This way we
+    # can check for 'On <date>, <author> wrote:' lines above quoted blocks.
+    #
+    # text - A String email body.
+    #
+    # Returns this same Email instance.
+    def read(text)
+      # in 1.9 we want to operate on the raw bytes
+      text = text.dup.force_encoding('binary') if text.respond_to?(:force_encoding)
+
+      # Normalize line endings.
+      text.gsub!("\r\n", "\n")
+
+      # Check for multi-line reply headers. Some clients break up
+      # the "On DATE, NAME <EMAIL> wrote:" line into multiple lines.
+      if text =~ /^(?!On.*On\s.+?wrote:)(On\s(.+?)wrote:)$/nm
+        # Remove all new lines from the reply header.
+        text.gsub! $1, $1.gsub("\n", " ")
+      end
+
+      # Check for "---- Original Message ----"
+      # and strip email content after that part
+      if text =~ /^([\s_-]+Original (?i)message?[\s_-]+$.*)/nm
+        text.gsub!($1, "")
+      end
+
+      # Some users may reply directly above a line of underscores.
+      # In order to ensure that these fragments are split correctly,
+      # make sure that all lines of underscores are preceded by
+      # at least two newline characters.
+      text.gsub!(/([^\n])(?=\n_{7}_+)$/m, "\\1\n")
+
+      # The text is reversed initially due to the way we check for hidden
+      # fragments.
+      text = text.reverse
+
+      # This determines if any 'visible' Fragment has been found.  Once any
+      # visible Fragment is found, stop looking for hidden ones.
+      @found_visible = false
+
+      # This instance variable points to the current Fragment.  If the matched
+      # line fits, it should be added to this Fragment.  Otherwise, finish it
+      # and start a new Fragment.
+      @fragment = nil
+
+      # Use the StringScanner to pull out each line of the email content.
+      @scanner = StringScanner.new(text)
+      while line = @scanner.scan_until(/\n/n)
+        scan_line(line)
+      end
+
+      # Be sure to parse the last line of the email.
+      if (last_line = @scanner.rest.to_s).size > 0
+        scan_line(last_line)
+      end
+
+      # Finish up the final fragment.  Finishing a fragment will detect any
+      # attributes (hidden, signature, reply), and join each line into a
+      # string.
+      finish_fragment
+
+      @scanner = @fragment = nil
+
+      # Now that parsing is done, reverse the order.
+      @fragments.reverse!
+      self
+    end
+
+  private
+    EMPTY = "".freeze
+    SIGNATURE = '(?m)(--\s*$|__\s*$)|(^(\w+\s*){1,3} ym morf tneS$)'
+
+    begin
+      require 're2'
+      SIG_REGEX = RE2::Regexp.new(SIGNATURE)
+    rescue LoadError
+      SIG_REGEX = Regexp.new(SIGNATURE)
+    end
+
+    ### Line-by-Line Parsing
+
+    # Scans the given line of text and figures out which fragment it belongs
+    # to.
+    #
+    # line - A String line of text from the email.
+    #
+    # Returns nothing.
+    def scan_line(line)
+      line.chomp!("\n")
+      line.lstrip! unless SIG_REGEX.match(line)
+
+      # We're looking for leading `>`'s to see if this line is part of a
+      # quoted Fragment.
+      is_quoted = !!(line =~ /(>+)$/n)
+
+      # Mark the current Fragment as a signature if the current line is empty
+      # and the Fragment starts with a common signature indicator.
+      if @fragment && line == EMPTY
+        if SIG_REGEX.match @fragment.lines.last
+          @fragment.signature = true
+          finish_fragment
+        end
+      end
+
+      # If the line matches the current fragment, add it.  Note that a common
+      # reply header also counts as part of the quoted Fragment, even though
+      # it doesn't start with `>`.
+      if @fragment &&
+          ((@fragment.quoted? == is_quoted) ||
+           (@fragment.quoted? && (quote_header?(line) || line == EMPTY)))
+        @fragment.lines << line
+
+      # Otherwise, finish the fragment and start a new one.
+      else
+        finish_fragment
+        @fragment = Fragment.new(is_quoted, line)
+      end
+    end
+
+    # Detects if a given line is a header above a quoted area.  It is only
+    # checked for lines preceding quoted regions.
+    #
+    # line - A String line of text from the email.
+    #
+    # Returns true if the line is a valid header, or false.
+    def quote_header?(line)
+      line =~ /^:etorw.*nO$/n
+    end
+
+    # Builds the fragment string and reverses it, after all lines have been
+    # added.  It also checks to see if this Fragment is hidden.  The hidden
+    # Fragment check reads from the bottom to the top.
+    #
+    # Any quoted Fragments or signature Fragments are marked hidden if they
+    # are below any visible Fragments.  Visible Fragments are expected to
+    # contain original content by the author.  If they are below a quoted
+    # Fragment, then the Fragment should be visible to give context to the
+    # reply.
+    #
+    #     some original text (visible)
+    #
+    #     > do you have any two's? (quoted, visible)
+    #
+    #     Go fish! (visible)
+    #
+    #     > --
+    #     > Player 1 (quoted, hidden)
+    #
+    #     --
+    #     Player 2 (signature, hidden)
+    #
+    def finish_fragment
+      if @fragment
+        @fragment.finish
+        if !@found_visible
+          if @fragment.quoted? || @fragment.signature? ||
+              @fragment.to_s.strip == EMPTY
+            @fragment.hidden = true
+          else
+            @found_visible = true
+          end
+        end
+        @fragments << @fragment
+      end
+      @fragment = nil
+    end
+  end
+
+  ### Fragments
+
+  # Represents a group of paragraphs in the email sharing common attributes.
+  # Paragraphs should get their own fragment if they are a quoted area or a
+  # signature.
+  class Fragment < Struct.new(:quoted, :signature, :hidden)
+    # This is an Array of String lines of content.  Since the content is
+    # reversed, this array is backwards, and contains reversed strings.
+    attr_reader :lines,
+
+    # This is reserved for the joined String that is build when this Fragment
+    # is finished.
+      :content
+
+    def initialize(quoted, first_line)
+      self.signature = self.hidden = false
+      self.quoted = quoted
+      @lines      = [first_line]
+      @content    = nil
+      @lines.compact!
+    end
+
+    alias quoted?    quoted
+    alias signature? signature
+    alias hidden?    hidden
+
+    # Builds the string content by joining the lines and reversing them.
+    #
+    # Returns nothing.
+    def finish
+      @content = @lines.join("\n")
+      @lines = nil
+      @content.reverse!
+    end
+
+    def to_s
+      @content
+    end
+
+    def inspect
+      to_s.inspect
+    end
+  end
+end

data/script/release

@@ -0,0 +1,2 @@
+#!/bin/sh
+rake release

data/script/test

@@ -0,0 +1,2 @@
+#!/bin/sh
+rake test

data/test/discourse_email_parser_test.rb

@@ -0,0 +1,202 @@
+require 'rubygems'
+require 'test/unit'
+require 'pathname'
+require 'pp'
+
+dir = Pathname.new File.expand_path(File.dirname(__FILE__))
+require dir + '..' + 'lib' + 'discourse_email_parser'
+
+EMAIL_FIXTURE_PATH = dir + 'emails'
+
+class DiscourseEmailParserTest < Test::Unit::TestCase
+  def test_reads_simple_body
+    reply = email(:email_1_1)
+    assert_equal 3, reply.fragments.size
+
+    assert reply.fragments.none? { |f| f.quoted? }
+    assert_equal [false, true, true],
+      reply.fragments.map { |f| f.signature? }
+    assert_equal [false, true, true],
+      reply.fragments.map { |f| f.hidden? }
+
+    assert_equal "Hi folks
+
+What is the best way to clear a Riak bucket of all key, values after
+running a test?
+I am currently using the Java HTTP API.\n", reply.fragments[0].to_s
+
+    assert_equal "--Abhishek Kona\n\n", reply.fragments[1].to_s
+  end
+
+  def test_reads_top_post
+    reply = email(:email_1_3)
+    assert_equal 5, reply.fragments.size
+
+    assert_equal [false, false, true, false, false],
+      reply.fragments.map { |f| f.quoted? }
+    assert_equal [false, true, true, true, true],
+      reply.fragments.map { |f| f.hidden? }
+    assert_equal [false, true, false, false, true],
+      reply.fragments.map { |f| f.signature? }
+
+    assert_match /^Oh thanks.\n\nHaving/, reply.fragments[0].to_s
+    assert_match /^--A/, reply.fragments[1].to_s
+    assert_match /^On [^\:]+\:/, reply.fragments[2].to_s
+    assert_match /^_/, reply.fragments[4].to_s
+  end
+
+  def test_reads_bottom_post
+    reply = email(:email_1_2)
+    assert_equal 6, reply.fragments.size
+
+    assert_equal [false, true, false, true, false, false],
+      reply.fragments.map { |f| f.quoted? }
+    assert_equal [false, false, false, false, false, true],
+      reply.fragments.map { |f| f.signature? }
+    assert_equal [false, false, false, true, true, true],
+      reply.fragments.map { |f| f.hidden? }
+
+    assert_equal "Hi,", reply.fragments[0].to_s
+    assert_match /^On [^\:]+\:/, reply.fragments[1].to_s
+    assert_match /^You can list/, reply.fragments[2].to_s
+    assert_match /^> /, reply.fragments[3].to_s
+    assert_match /^_/, reply.fragments[5].to_s
+  end
+
+  def test_reads_inline_replies
+    reply = email(:email_1_8)
+    assert_equal 7, reply.fragments.size
+
+    assert_equal [true, false, true, false, true, false, false],
+      reply.fragments.map { |f| f.quoted? }
+    assert_equal [false, false, false, false, false, false, true],
+      reply.fragments.map { |f| f.signature? }
+    assert_equal [false, false, false, false, true, true, true],
+      reply.fragments.map { |f| f.hidden? }
+
+    assert_match /^On [^\:]+\:/, reply.fragments[0].to_s
+    assert_match /^I will reply/, reply.fragments[1].to_s
+    assert_match "okay?", reply.fragments[2].to_s
+    assert_match /^and under this./, reply.fragments[3].to_s
+    assert_match /inline/, reply.fragments[4].to_s
+    assert_equal "\n", reply.fragments[5].to_s
+    assert_equal "--\nHey there, this is my signature\n", reply.fragments[6].to_s
+  end
+
+  def test_recognizes_date_string_above_quote
+    reply = email :email_1_4
+
+    assert_match /^Awesome/, reply.fragments[0].to_s
+    assert_match /^On/,      reply.fragments[1].to_s
+    assert_match /Loader/,   reply.fragments[1].to_s
+  end
+
+  def test_a_complex_body_with_only_one_fragment
+    reply = email :email_1_5
+
+    assert_equal 1, reply.fragments.size
+  end
+
+  def test_reads_email_with_correct_signature
+    reply = email :correct_sig
+
+    assert_equal 2, reply.fragments.size
+    assert_equal [false, false], reply.fragments.map { |f| f.quoted? }
+    assert_equal [false, true], reply.fragments.map { |f| f.signature? }
+    assert_equal [false, true], reply.fragments.map { |f| f.hidden? }
+    assert_match /^-- \nrick/, reply.fragments[1].to_s
+  end
+
+  def test_deals_with_multiline_reply_headers
+    reply = email :email_1_6
+
+    assert_match /^I get/,   reply.fragments[0].to_s
+    assert_match /^On/,      reply.fragments[1].to_s
+    assert_match /Was this/, reply.fragments[1].to_s
+  end
+
+  def test_deals_with_windows_line_endings
+    reply = email :email_1_7
+
+    assert_match /:\+1:/,     reply.fragments[0].to_s
+    assert_match /^On/,       reply.fragments[1].to_s
+    assert_match /Steps 0-2/, reply.fragments[1].to_s
+  end
+
+  def test_does_not_modify_input_string
+    original = "The Quick Brown Fox Jumps Over The Lazy Dog"
+    DiscourseEmailParser.read original
+    assert_equal "The Quick Brown Fox Jumps Over The Lazy Dog", original
+  end
+
+  def test_returns_only_the_visible_fragments_as_a_string
+    reply = email(:email_2_1)
+    assert_equal reply.fragments.select{|r| !r.hidden?}.map{|r| r.to_s}.join("\n").rstrip, reply.visible_text
+  end
+
+  def test_parse_out_just_top_for_outlook_reply
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_2_1.txt").to_s
+    assert_equal "Outlook with a reply", DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_parse_out_just_top_for_outlook_with_reply_directly_above_line
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_2_2.txt").to_s
+    assert_equal "Outlook with a reply directly above line", DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_parse_out_sent_from_iPhone
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_iPhone.txt").to_s
+    assert_equal "Here is another email", DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_parse_out_sent_from_BlackBerry
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_BlackBerry.txt").to_s
+    assert_equal "Here is another email", DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_parse_out_send_from_multiword_mobile_device
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_multi_word_sent_from_my_mobile_device.txt").to_s
+    assert_equal "Here is another email", DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_do_not_parse_out_send_from_in_regular_sentence
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_sent_from_my_not_signature.txt").to_s
+    assert_equal "Here is another email\n\nSent from my desk, is much easier then my mobile phone.", DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_retains_bullets
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_bullets.txt").to_s
+    assert_equal "test 2 this should list second\n\nand have spaces\n\nand retain this formatting\n\n\n   - how about bullets\n   - and another",
+      DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_parse_reply
+    body = IO.read EMAIL_FIXTURE_PATH.join("email_1_2.txt").to_s
+    assert_equal DiscourseEmailParser.read(body).visible_text, DiscourseEmailParser.parse_reply(body)
+  end
+
+  def test_one_is_not_on
+    reply = email("email_one_is_not_on")
+    assert_match /One outstanding question/, reply.fragments[0].to_s
+    assert_match /^On Oct 1, 2012/, reply.fragments[1].to_s
+  end
+
+  def test_mulitple_on
+    reply = email("greedy_on")
+    assert_match /^On your remote host/, reply.fragments[0].to_s
+    assert_match /^On 9 Jan 2014/, reply.fragments[1].to_s
+    assert_equal [false, true, false], reply.fragments.map { |f| f.quoted? }
+    assert_equal [false, false, false], reply.fragments.map { |f| f.signature? }
+    assert_equal [false, true, true], reply.fragments.map { |f| f.hidden? }
+  end
+
+  def test_doesnt_remove_signature_delimiter_in_mid_line
+    reply = email(:email_sig_delimiter_in_middle_of_line)
+    assert_equal 1, reply.fragments.size
+  end
+
+  def email(name)
+    body = IO.read EMAIL_FIXTURE_PATH.join("#{name}.txt").to_s
+    DiscourseEmailParser.read body
+  end
+end

data/test/emails/correct_sig.txt

@@ -0,0 +1,4 @@
+this is an email with a correct -- signature.
+
+-- 
+rick

data/test/emails/email_1_1.txt

@@ -0,0 +1,13 @@
+Hi folks
+
+What is the best way to clear a Riak bucket of all key, values after
+running a test?
+I am currently using the Java HTTP API.
+
+--Abhishek Kona
+
+
+_______________________________________________
+riak-users mailing list
+riak-users@lists.basho.com
+http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com

data/test/emails/email_1_2.txt

@@ -0,0 +1,51 @@
+Hi,
+On Tue, 2011-03-01 at 18:02 +0530, Abhishek Kona wrote:
+> Hi folks
+>
+> What is the best way to clear a Riak bucket of all key, values after
+> running a test?
+> I am currently using the Java HTTP API.
+
+You can list the keys for the bucket and call delete for each. Or if you
+put the keys (and kept track of them in your test) you can delete them
+one at a time (without incurring the cost of calling list first.)
+
+Something like:
+
+        String bucket = "my_bucket";
+        BucketResponse bucketResponse = riakClient.listBucket(bucket);
+        RiakBucketInfo bucketInfo = bucketResponse.getBucketInfo();
+
+        for(String key : bucketInfo.getKeys()) {
+            riakClient.delete(bucket, key);
+        }
+
+
+would do it.
+
+See also
+
+http://wiki.basho.com/REST-API.html#Bucket-operations
+
+which says
+
+"At the moment there is no straightforward way to delete an entire
+Bucket. There is, however, an open ticket for the feature. To delete all
+the keys in a bucket, you’ll need to delete them all individually."
+
+>
+> -- Abhishek Kona
+>
+>
+> _______________________________________________
+> riak-users mailing list
+> riak-users@lists.basho.com
+> http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
+
+
+
+
+_______________________________________________
+riak-users mailing list
+riak-users@lists.basho.com
+http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com

data/test/emails/email_1_3.txt

@@ -0,0 +1,55 @@
+Oh thanks.
+
+Having the function would be great.
+
+--Abhishek Kona
+
+On 01/03/11 7:07 PM, Russell Brown wrote:
+> Hi,
+> On Tue, 2011-03-01 at 18:02 +0530, Abhishek Kona wrote:
+>> Hi folks
+>>
+>> What is the best way to clear a Riak bucket of all key, values after
+>> running a test?
+>> I am currently using the Java HTTP API.
+> You can list the keys for the bucket and call delete for each. Or if you
+> put the keys (and kept track of them in your test) you can delete them
+> one at a time (without incurring the cost of calling list first.)
+>
+> Something like:
+>
+>          String bucket = "my_bucket";
+>          BucketResponse bucketResponse = riakClient.listBucket(bucket);
+>          RiakBucketInfo bucketInfo = bucketResponse.getBucketInfo();
+>
+>          for(String key : bucketInfo.getKeys()) {
+>              riakClient.delete(bucket, key);
+>          }
+>
+>
+> would do it.
+>
+> See also
+>
+> http://wiki.basho.com/REST-API.html#Bucket-operations
+>
+> which says
+>
+> "At the moment there is no straightforward way to delete an entire
+> Bucket. There is, however, an open ticket for the feature. To delete all
+> the keys in a bucket, you’ll need to delete them all individually."
+>
+>> --Abhishek Kona
+>>
+>>
+>> _______________________________________________
+>> riak-users mailing list
+>> riak-users@lists.basho.com
+>> http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
+>
+
+
+_______________________________________________
+riak-users mailing list
+riak-users@lists.basho.com
+http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com

data/test/emails/email_1_4.txt

@@ -0,0 +1,8 @@
+Awesome! I haven't had another problem with it.
+
+On Aug 22, 2011, at 7:37 PM, defunkt<reply@reply.github.com> wrote:
+
+
+
+
+> Loader seems to be working well.

data/test/emails/email_1_5.txt

@@ -0,0 +1,15 @@
+One: Here's what I've got.
+
+- This would be the first bullet point that wraps to the second line
+to the next
+- This is the second bullet point and it doesn't wrap
+- This is the third bullet point and I'm having trouble coming up with enough
+to say
+- This is the fourth bullet point
+
+Two:
+- Here is another bullet point
+- And another one
+
+This is a paragraph that talks about a bunch of stuff. It goes on and on
+for a while.

data/test/emails/email_1_6.txt

@@ -0,0 +1,15 @@
+I get proper rendering as well.
+
+Sent from a magnificent torch of pixels
+
+On Dec 16, 2011, at 12:47 PM, Corey Donohoe
+<reply@reply.github.com>
+wrote:
+
+> Was this caching related or fixed already?  I get proper rendering here.
+>
+> ![](https://img.skitch.com/20111216-m9munqjsy112yqap5cjee5wr6c.jpg)
+>
+> ---
+> Reply to this email directly or view it on GitHub:
+> https://github.com/github/github/issues/2278#issuecomment-3182418

data/test/emails/email_1_7.txt

@@ -0,0 +1,12 @@
+:+1:
+
+On Tue, Sep 25, 2012 at 8:59 AM, Chris Wanstrath
+<notifications@github.com>wrote:
+
+> Steps 0-2 are in prod. Gonna let them sit for a bit then start cleaning up
+> the old code with 3 & 4.
+>
+> 
+> Reply to this email directly or view it on GitHub.
+>
+>

data/test/emails/email_1_8.txt

@@ -0,0 +1,37 @@
+On Tue, Apr 29, 2014 at 4:22 PM, Example Dev <sugar@example.com>wrote:
+
+> okay.  Well, here's some stuff I can write.
+>
+> And if I write a 2 second line you and maybe reply under this?
+>
+> Or if you didn't really feel like it, you could reply under this line.Or
+> if you didn't really feel like it, you could reply under this line. Or if
+> you didn't really feel like it, you could reply under this line. Or if you
+> didn't really feel like it, you could reply under this line.
+>
+
+I will reply under this one
+
+>
+> okay?
+>
+
+and under this.
+
+>
+> -- Tim
+>
+> On Tue, April 29, 2014 at 4:21 PM, Tim Haines <tmhaines@example.com> wrote:
+> > hi there
+> >
+> > After you reply to this I'm going to send you some inline responses.
+> >
+> > --
+> > Hey there, this is my signature
+>
+>
+>
+
+
+--
+Hey there, this is my signature

data/test/emails/email_2_1.txt

@@ -0,0 +1,25 @@
+Outlook with a reply
+
+
+ ------------------------------
+
+*From:* Google Apps Sync Team [mailto:mail-noreply@google.com]
+*Sent:* Thursday, February 09, 2012 1:36 PM
+*To:* jow@xxxx.com
+*Subject:* Google Apps Sync was updated!
+
+
+
+Dear Google Apps Sync user,
+
+Google Apps Sync for Microsoft Outlook® was recently updated. Your computer
+now has the latest version (version 2.5). This release includes bug fixes
+to improve product reliability. For more information about these and other
+changes, please see the help article here:
+
+http://www.google.com/support/a/bin/answer.py?answer=153463
+
+Sincerely,
+
+The Google Apps Sync Team.
+

data/test/emails/email_2_2.txt

@@ -0,0 +1,10 @@
+Outlook with a reply directly above line
+________________________________________
+From: CRM Comments [crm-comment@example.com]
+Sent: Friday, 23 March 2012 5:08 p.m.
+To: John S. Greene
+Subject: [contact:106] John Greene
+
+A new comment has been added to the Contact named 'John Greene':
+
+I am replying to a comment.

data/test/emails/email_BlackBerry.txt

@@ -0,0 +1,3 @@
+Here is another email
+
+Sent from my BlackBerry

data/test/emails/email_bullets.txt

@@ -0,0 +1,22 @@
+test 2 this should list second
+
+and have spaces
+
+and retain this formatting
+
+
+   - how about bullets
+   - and another
+
+
+On Fri, Feb 24, 2012 at 10:19 AM, <examples@email.goalengine.com> wrote:
+
+> Give us an example of how you applied what they learned to achieve
+> something in your organization
+
+
+
+
+-- 
+
+*Joe Smith | Director, Product Management*

data/test/emails/email_iPhone.txt

@@ -0,0 +1,3 @@
+Here is another email
+
+Sent from my iPhone

data/test/emails/email_multi_word_sent_from_my_mobile_device.txt

@@ -0,0 +1,3 @@
+Here is another email
+
+Sent from my Verizon Wireless BlackBerry

data/test/emails/email_one_is_not_on.txt

@@ -0,0 +1,10 @@
+Thank, this is really helpful.
+
+One outstanding question I had:
+
+Locally (on development), when I run...
+
+On Oct 1, 2012, at 11:55 PM, Dave Tapley wrote:
+
+> The good news is that I've found a much better query for lastLocation.
+>

data/test/emails/email_sent_from_my_not_signature.txt

@@ -0,0 +1,3 @@
+Here is another email
+
+Sent from my desk, is much easier then my mobile phone.

data/test/emails/email_sig_delimiter_in_middle_of_line.txt

@@ -0,0 +1,7 @@
+Hi there!
+
+Stuff happened.
+
+And here is a fix -- this is not a signature.
+
+kthxbai

data/test/emails/greedy_on.txt

@@ -0,0 +1,16 @@
+On your remote host you can run:
+
+     telnet 127.0.0.1 52698
+
+This should connect to TextMate (on your Mac, via the tunnel). If that
+fails, the tunnel is not working.
+
+On 9 Jan 2014, at 2:47, George Plymale wrote:
+
+> I am having an odd issue wherein suddenly port forwarding stopped
+> working in a particular scenario for me.  By default I have ssh set to
+> use the following config (my ~/.ssh/config file):
+> […]
+> ---
+> Reply to this email directly or view it on GitHub:
+> https://github.com/textmate/rmate/issues/29

data/test/emails/pathological.txt

@@ -0,0 +1,20 @@
+I think you're onto something. I will try to fix the problem as soon as I
+get back to a computer.
+On Dec 8, 2013 2:10 PM, "John Sullivan" <notifications@github.com> wrote:
+
+> I think your code is shortening the reference sequence you return to be
+> the same size as the query sequence, and we end up losing data. Here's some
+> debugging output from me putzing around...
+>
+> name: gi|253409428|ref|GQ227366.1| Influenza A virus (A/pika/Qinghai/BI/2007(H5N1)) segment 1 polymerase PB2 (PB2) gene, complete cds
+> score: 39.0
+>
+> organism.sequence: ATGGAGAGAATAAAGGAATTAAGAGATCTAATGTCACAGTCCCGCACTCGCGAGATACTAACAAAGACCACTGTGGACCATATGGCCATAATCAAGAAATACACATCAGGAAGACAAGAGAAGAACCCTGCTCTCAGAATGAAATGGATGATGGCAATGAAATATCCAATCACAGCGGACAAGAGAATAATAGAGATGATTCCTGAAAGGAATGAACAAGGACAGACACTCTGGAGCAAGACAAATGATGCTGGATCGGACAGGGTGATGGTGTCTCCCCTAGCTGTAACTTGGTGGAATAGGAATGGGCCGACGACAAGTACAGTTCATTATCCAAAGGTTTACAAAACATACTTTGAGAAGGTTGAAAGGTTAAAACATGGAACCTTCGGTCCCGTTCATTTCCGAAACCAAGTTAAAATACGCCGCCGAGTTGATACAAATCCTGGCCATGCAGATCTCAGTGCTAAAGAAGCACAAGATGTCATCATGGAGGTCGTTTTCCCAAATGAAGTGGGAGCTAGAATATTGACTTCAGAGTCACAGTTGACAATAACGAAAGAGAAAAAAGAAGAGCTCCAAGATTGTAAGATTGCTCCCTTAATGGTTGCATACATGTTGGAAAGGGAACTGGTCCGCAAAACCAGATTCCTACCAGTAGCAGGCGGAACAAGCAGTGTGTACATTGAGGTATTGCATTTGACTCAAGGAACCTGCTGGGCACAGATGTACACTCCAGGCGGAGAAGTAAGAAATGACGATGTTGACCAGAGTTTGATCATTGCTGCCAGAAACATTGTTAGGAGAGCAACGGTATCAGCGGATCCACTGGCATCACTGCTGGAGATGTGTCACAGCACACAAATTGGTGGGATAAGGATGGTGGACATCCTTAGGCAAACTCCAACTGAGGAACAAGCTGTGGATATATGCAAAGCAGCAATGGGTCTGAGGATTAGTTCATCCTTTAGCTTTGGAG
+>  GCTTCACTTTCAAAAGAACAAGTGGATCATCCGCCACGAAGGAAGAGGAAGTGCTTACAGGCAACCTCCAAACATTGAAAATAAGAGTACATGAGGGGTATGAGGAGTTCACAATGGTTGGGCAGAGGGCAACAGCTATCCTGAGGAAAGCAACTAGAAGGCTGATTCAGTTGATAGTAAGTGGAAGAAACGAACAATCAATCGCTGAGGCAATCATTGTAGCAATGGTGTTCTCACAGGAGGATCGCATGATAAAAGCAGTCCGAGGCGATCTGAATTTCGTAAACAGAGCAAACCAAAGATTAAACCCCATGCATCAACTCCTGAGACATTTTCAAAAGGACGCAAAAGTGCTATTTCAGAATTGGGGAACTGAGCCAATTGATAATGTCATGGGGATGATCGGAATATTACCTGACATGACTCCCAGCACAGAAACGTCACTGAGAGGAGTGAGAGTTAGTAAAATGGGAGTAGATGAGTATTCCAGCACTGAGAGAGTAGTTGTAAGCATTGACCGCTTCTTAAGGGTTCGAGACCAGCGGGGGAACGTACTCTTATCTCCCGAAGAGGTCAGCGAAACCCAGGGAACAGAGAAGTTGACAATAACATATTCATCATCAATGATGTGGGAAATCAACGGTCCTGAGTCAGTGCTTGTTAACACTTACCAATGGATCATTAGAAACTGGGAGACCGTGAAAATTCAGTGGTCTCAGGACCCCACGATGTTGTACAATAAGATGGAGTTTGAACCGTTCCAATCCTTGGTACCTAAAGCTGCCAGAGGTCAATACAGTGGATTTGTGAGAACATTATTCCAACAAATGCGTGACGTACTGGGGACATTTGATACTGTCCAGATAATAAAGCTGCTACCATTTGCAGCAGCCCCACCGAAGCAGAGCAGAATGCAGTTTTCTTCTCTAACTGTGAATGTGAGAGGCTCAGGAATGAGAATACTCATAAGGGGCAATTCCCCTGTGTTCAACTACAA
+>  TAAGGCAACCCAAAGACTTACCGTTCTTGGAAAGGACGCAGGTGCATTAACAGAGGATCCAGATGAGGGGACAGCCGGAGTGGAATCTGCAGTACTGAGGGGGTTCCTAATTCTAGGCAAGGAGGACAAAAGATATGGACCAGCATTGAGCATCAATGAACTGAGCAATCTTGCAAAAGGGGAGAAAGCTAATGTGCTGATAGGGCAAGGAGACGTGGTGTTGGTAATGAAACGGAAACGGGACTCTAGCATACTTACTGACAGCCAGACAGCGACCAAAAGAATTCGGATGGCCATCAATTAGTGTCGAATTGTTTAAAAACGACCTTGTTTCTACT
+> reference_alignment: ________________________________________________
+>
+> query: AGCGAAAGCAGGTCAAATATATTCAATATGGAGAGAATAAAAGAATTAAG
+>
+> query_alignment: GCGAAAGCAGGTCAAATATATTCAATATGGAGAGAATAAAAGAATTAAG
+>

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: discourse_email_parser
+version: !ruby/object:Gem::Version
+  version: 0.6.0
+platform: ruby
+authors:
+- Rick Olson
+- Arpit Jalan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-11 00:00:00.000000000 Z
+dependencies: []
+description: DiscourseEmailParser is a small library to parse plain text email content.
+email: arpit.jalan@discourse.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- discourse_email_parser.gemspec
+- lib/discourse_email_parser.rb
+- script/release
+- script/test
+- test/discourse_email_parser_test.rb
+- test/emails/correct_sig.txt
+- test/emails/email_1_1.txt
+- test/emails/email_1_2.txt
+- test/emails/email_1_3.txt
+- test/emails/email_1_4.txt
+- test/emails/email_1_5.txt
+- test/emails/email_1_6.txt
+- test/emails/email_1_7.txt
+- test/emails/email_1_8.txt
+- test/emails/email_2_1.txt
+- test/emails/email_2_2.txt
+- test/emails/email_BlackBerry.txt
+- test/emails/email_bullets.txt
+- test/emails/email_iPhone.txt
+- test/emails/email_multi_word_sent_from_my_mobile_device.txt
+- test/emails/email_one_is_not_on.txt
+- test/emails/email_sent_from_my_not_signature.txt
+- test/emails/email_sig_delimiter_in_middle_of_line.txt
+- test/emails/greedy_on.txt
+- test/emails/pathological.txt
+homepage: http://github.com/discourse/discourse_email_parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Small library to parse plain text email content.
+test_files:
+- test/discourse_email_parser_test.rb
+has_rdoc: