mail_extract 0.1.0

data/.gitignore

@@ -0,0 +1,41 @@
+!.gitignore
+*.gem
+*.rbc
+*.sw[a-p]
+*.tmproj
+*.tmproject
+*.un~
+*~
+.DS_Store
+.Spotlight-V100
+.Trashes
+._*
+.bundle
+.config
+.directory
+.elc
+.redcar
+.yardoc
+/.emacs.desktop
+/.emacs.desktop.lock
+Desktop.ini
+Gemfile.lock
+Icon?
+InstalledFiles
+Session.vim
+Thumbs.db
+\#*\#
+_yardoc
+auto-save-list
+coverage
+doc/
+lib/bundler/man
+pkg
+pkg/*
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+tmtags
+tramp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format=nested
+--backtrace

data/README.md

@@ -0,0 +1,54 @@
+# MailExtract
+
+MailExtract is a small ruby library to parse plain-text email contents.
+
+It removes all quoted text and signatures leaving only original text. 
+
+## Installation
+
+    gem install mail_extract
+
+## Usage
+
+### General usage
+
+    require 'mail_extract'
+
+    body = MailExtract::Parser.new('MESSAGE').body
+
+    # or via shortcut
+    body = MailExtract.new('MESSAGE').body
+
+### Using with Mail gem
+
+    require 'mail'
+    require 'mail_extract'
+
+    mail = Mail.read_from_string(YOUR_MESSAGE_BODY)
+    
+    # find only plain-text parts
+    if mail.multipart?
+      part = mail.parts.select { |p| p.content_type =~ /text\/plain/ }.first rescue nil
+      unless part.nil?
+        message = part.body.decoded
+      end
+    else
+      message = part.body.decoded
+    end
+
+    clean_message = MailExtract.new(message).body
+
+## Known issues
+
+- Invalid signature patterns (that does not follow --, ___)
+- Invalid quote patterns (that does not start with >)
+
+## License
+
+Copyright © 2011 Dan Sosedoff.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+
+require "bundler"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/*_spec.rb'
+end
+
+task :default => :spec
+task :test => :spec

data/lib/mail_extract.rb

@@ -0,0 +1,12 @@
+require 'mail_extract/line'
+require 'mail_extract/parser'
+
+module MailExtract
+  class << self
+    # Shortcut to MailExtract::Parser.new
+    #
+    def new(body)
+      MailExtract::Parser.new(body)
+    end
+  end
+end

data/lib/mail_extract/line.rb

@@ -0,0 +1,36 @@
+module MailExtract
+  class Line
+    attr_reader :body, :type
+    
+    PATTERNS = {
+      /^[>]+\s?/    => :quote,
+      /^--/         => :signature,
+      /^-- /        => :signature,
+      /^[_]{2,}\n?/ => :signature,
+      /^[-]{2,}\n?/ => :signature
+    }
+    
+    def initialize(str)
+      @body = str
+      detect_type(str)
+    end
+    
+    private
+    
+    def detect_type(line)
+      # Detects the start line of quote text
+      if line.strip =~ /^On/ && line =~ /at [\d:]+/ && line.strip =~ /wrote:?\z/
+        @type = :quote
+        return
+      end
+      
+      @type = :text
+      PATTERNS.each_pair do |p,t|
+        if line =~ p
+          @type = t
+          break
+        end
+      end
+    end
+  end
+end

data/lib/mail_extract/parser.rb

@@ -0,0 +1,60 @@
+require 'strscan'
+
+module MailExtract
+  class Parser
+    attr_reader :body
+    
+    # Initialize a new MailExtract::Parser object
+    #   text - Email message body
+    #
+    def initialize(text)
+      @lines     = []
+      @text      = text.strip
+      @body      = ""
+      @last_type = :text
+      @type      = :text
+      parse
+    end
+    
+    private
+    
+    # Process email message body
+    #
+    def parse
+      scanner = StringScanner.new(@text)
+      while str = scanner.scan_until(/\n/)
+        parse_line(str)
+      end
+      if (last_line = scanner.rest.to_s).size > 0
+        parse_line(last_line)
+      end
+      @body = @lines.join("\n").strip
+    end
+    
+    # Process a single line
+    #
+    def parse_line(str)
+      line = MailExtract::Line.new(str)
+      if line.type == :quote
+        if @last_type == :text
+          @type = :quote
+        end
+      elsif line.type == :text
+        if @last_type == :quote
+          @type = :text
+        end
+        if @last_type == :signature
+          @type = :signature
+        end
+      elsif line.type == :signature
+        if @last_type == :text
+          @type = :signature
+        elsif @last_type == :quote
+          @type = :quote
+        end
+      end
+      @last_type = line.type
+      @lines << line.body.strip if @type == :text
+    end
+  end
+end

data/lib/mail_extract/version.rb

@@ -0,0 +1,3 @@
+module MailExtract
+  VERSION = "0.1.0".freeze unless defined? ::MailExtract::VERSION
+end

data/mail_extract.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/mail_extract/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'mail_extract'
+  gem.version     = MailExtract::VERSION.dup
+  gem.author      = 'Dan Sosedoff'
+  gem.email       = 'dan.sosedoff@gmail.com'
+  gem.homepage    = 'https://github.com/sosedoff/mail_extract'
+  gem.summary     = %q{Extracts email message body}
+  gem.description = %q{Email body parser that strips out all quotes and signatures.}
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{|f| File.basename(f)}
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'rspec', '~> 2.6'
+end

data/spec/fixtures/result_simple.txt

@@ -0,0 +1,7 @@
+Hi folks
+
+What is the best way to clear a Riak bucket of all key, values after
+running a test?
+I am currently using the Java HTTP API.
+
+-Abhishek Kona

data/spec/fixtures/result_simple_with_quotes.txt

@@ -0,0 +1,28 @@
+Hi,
+
+You can list the keys for the bucket and call delete for each. Or if you
+put the keys (and kept track of them in your test) you can delete them
+one at a time (without incurring the cost of calling list first.)
+
+Something like:
+
+String bucket = "my_bucket";
+BucketResponse bucketResponse = riakClient.listBucket(bucket);
+RiakBucketInfo bucketInfo = bucketResponse.getBucketInfo();
+
+for(String key : bucketInfo.getKeys()) {
+riakClient.delete(bucket, key);
+}
+
+
+would do it.
+
+See also
+
+http://wiki.basho.com/REST-API.html#Bucket-operations
+
+which says
+
+"At the moment there is no straightforward way to delete an entire
+Bucket. There is, however, an open ticket for the feature. To delete all
+the keys in a bucket, you’ll need to delete them all individually."

data/spec/fixtures/simple.txt

@@ -0,0 +1,13 @@
+Hi folks
+
+What is the best way to clear a Riak bucket of all key, values after 
+running a test?
+I am currently using the Java HTTP API.
+
+-Abhishek Kona
+
+
+_______________________________________________
+riak-users mailing list
+riak-users@lists.basho.com
+http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com

data/spec/fixtures/simple_with_quotes.txt

@@ -0,0 +1,51 @@
+Hi,
+On Tue, 2011-03-01 at 18:02 +0530, Abhishek Kona wrote:
+> Hi folks
+> 
+> What is the best way to clear a Riak bucket of all key, values after 
+> running a test?
+> I am currently using the Java HTTP API.
+
+You can list the keys for the bucket and call delete for each. Or if you
+put the keys (and kept track of them in your test) you can delete them
+one at a time (without incurring the cost of calling list first.)
+
+Something like:
+
+        String bucket = "my_bucket";
+        BucketResponse bucketResponse = riakClient.listBucket(bucket);
+        RiakBucketInfo bucketInfo = bucketResponse.getBucketInfo();
+        
+        for(String key : bucketInfo.getKeys()) {
+            riakClient.delete(bucket, key);
+        }
+
+
+would do it.
+
+See also 
+
+http://wiki.basho.com/REST-API.html#Bucket-operations
+
+which says 
+
+"At the moment there is no straightforward way to delete an entire
+Bucket. There is, however, an open ticket for the feature. To delete all
+the keys in a bucket, you’ll need to delete them all individually."
+
+> 
+> -Abhishek Kona
+> 
+> 
+> _______________________________________________
+> riak-users mailing list
+> riak-users@lists.basho.com
+> http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
+
+
+
+
+_______________________________________________
+riak-users mailing list
+riak-users@lists.basho.com
+http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com

data/spec/line_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe 'MailExtract::Line' do
+  def line(str)
+    MailExtract::Line.new(str)
+  end
+  
+  it 'detects quote start by date' do
+    line('On Tue, 2011-03-01 at 18:02 +0530, somebody wrote:').type.should == :quote
+    line('On 2011-03-01 at 18:02 somebody wrote').type.should == :quote
+    line('On some day somebody wrote').type.should == :text
+  end
+  
+  it 'detects quote' do
+    line('> this is a quote').type.should == :quote
+    line('> >> this is a quote').type.should == :quote
+  end
+  
+  it 'detects signature' do
+    lines = [
+     "--\nUsername",
+     "-- \nUsername",
+     "_______\nSome text"
+    ]
+    
+    lines.each do |l|
+      line(l).type.should == :signature
+    end
+  end
+end

data/spec/parser_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe 'MailExtract::Parser' do
+  it 'parses an email' do
+    body = MailExtract.new(fixture('simple.txt')).body
+    body.should == result_fixture('simple.txt')
+  end
+  
+  it 'parses an email with quotes' do
+    body = MailExtract.new(fixture('simple_with_quotes.txt')).body
+    body.should == result_fixture('simple_with_quotes.txt')
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+$:.unshift File.expand_path("../..", __FILE__)
+
+require 'mail_extract'
+
+def fixture_path(file=nil)
+  path = File.expand_path("../fixtures", __FILE__)
+  path = File.join(path, file) unless file.nil?
+  path
+end
+
+def fixture(file)
+  File.read(File.join(fixture_path, file))
+end
+
+def result_fixture(file)
+  fixture("result_#{file}")
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification 
+name: mail_extract
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Dan Sosedoff
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-21 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 6
+        version: "2.6"
+  type: :development
+  version_requirements: *id001
+description: Email body parser that strips out all quotes and signatures.
+email: dan.sosedoff@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- README.md
+- Rakefile
+- lib/mail_extract.rb
+- lib/mail_extract/line.rb
+- lib/mail_extract/parser.rb
+- lib/mail_extract/version.rb
+- mail_extract.gemspec
+- spec/fixtures/result_simple.txt
+- spec/fixtures/result_simple_with_quotes.txt
+- spec/fixtures/simple.txt
+- spec/fixtures/simple_with_quotes.txt
+- spec/line_spec.rb
+- spec/parser_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: https://github.com/sosedoff/mail_extract
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Extracts email message body
+test_files: 
+- spec/fixtures/result_simple.txt
+- spec/fixtures/result_simple_with_quotes.txt
+- spec/fixtures/simple.txt
+- spec/fixtures/simple_with_quotes.txt
+- spec/line_spec.rb
+- spec/parser_spec.rb
+- spec/spec_helper.rb