astrotrain 0.5.4 → 0.6.0

data/lib/astrotrain.rb

@@ -1,56 +1,52 @@
 module Astrotrain
-  class ProcessingCancelled < StandardError; end
-
-  CALLBACK_TYPES = [:pre_mapping, :pre_processing, :post_processing]
-  class << self
-    attr_accessor :root, :lib_root, :callbacks
-  end
-
-  def self.load(root = Dir.pwd)
-    self.root     = File.expand_path(root)
-    self.lib_root = File.expand_path(File.dirname(__FILE__))
-    load_dependencies
-    yield if block_given?
-    %w(tmail message mapping logged_mail mapping/transport mapping/http_post mapping/jabber).each do |lib|
-      require "astrotrain/#{lib}"
-    end
-    Astrotrain::Mail::ALLOW_MULTIPLE['delivered-to'] = true
-  end
-
-  def self.callback(name, *args, &block)
-    found = callbacks[name]
-    if block
-      found << block
-    else
-      found.each { |cback| cback.call(*args) }
-    end
-    found
+  VERSION = '0.6.0'
+
+  require 'utf8'
+  require 'charlock_holmes'
+  require 'addressable/uri'
+  require 'faraday'
+  require 'astrotrain/attachment'
+  require 'astrotrain/message'
+
+  # Processes an Astrotrain message.
+  #
+  # message     - Astrotrain::Message instance.
+  # destination - String URL to deliver the message.  The scheme selects
+  #               which Transport module to use (http://, resque://)
+  # options     - Optional hash of options:
+  #               :recipient - The main String recipient of the email.
+  #               :payload   - Optional hash to be sent with the request.
+  #
+  # Returns nothing.
+  def self.deliver(message, destination, options = {})
+    uri   = Addressable::URI.parse(destination.to_s)
+    klass = Transports.load(uri.scheme)
+    klass.deliver(message, destination, 
+                  :recipient => options[:recipient],
+                  :extra     => options[:payload])
   end
 
-  def self.clear_callbacks
-    self.callbacks = CALLBACK_TYPES.inject({}) { |memo, ctype| memo.update(ctype => []) }
-  end
-
-  clear_callbacks
-
-private
-  # help me ryan tomayko, you're my only help
-  def self.load_dependencies
-    require 'rubygems'
-    gem 'addressable',   '2.0.2'
-
-    dm_ver = "0.9.11"
-    gem "data_objects",   dm_ver
-    gem "dm-core",        dm_ver # The datamapper ORM
-    gem "dm-aggregates",  dm_ver # Provides your DM models with count, sum, avg, min, max, etc.
-    gem "dm-timestamps",  dm_ver # Automatically populate created_at, created_on, etc. when those properties are present.
-    gem "dm-types",       dm_ver # Provides additional types, including csv, json, yaml.
-    gem "dm-validations", dm_ver # Validation framework
-
-    $LOAD_PATH.unshift File.join(lib_root, 'vendor', 'rest-client', 'lib')
-
-    %w(dm-core dm-aggregates dm-timestamps dm-types dm-validations tmail rest_client).each do |lib|
-      require lib
+  # Transports are responsible for getting this email where it is supposed
+  # to go.
+  #
+  # All Transports should conform to this API:
+  #
+  #   Transports::HttpPost.process(address, message, main_recipient, extra_payload={})
+  #
+  module Transports
+    MAP = {:http => :http_post, :resque => :resque}
+
+    def self.load(key)
+      key   = key.to_sym if key
+      value = MAP[key]
+      if !value
+        raise ArgumentError, "No transport #{key.inspect} found in #{MAP.keys.inspect}"
+      elsif value.is_a?(Module)
+        value
+      else
+        require "astrotrain/transports/#{value}"
+        MAP[key]
+      end
     end
   end
-end
+end

data/lib/astrotrain/attachment.rb

@@ -0,0 +1,55 @@
+module Astrotrain
+  # Simple class that wraps a TMail part attachment in the IO API for 
+  # Faraday
+  class Attachment
+    def initialize(part)
+      @data    = nil
+      @part    = part
+      @is_read = false
+    end
+
+    def content_type
+      @part.content_type
+    end
+
+    def filename
+      @part.filename
+    end
+
+    alias local_path filename
+    alias original_filename filename
+
+    def read(value = nil)
+      if read?
+        nil
+      else
+        @is_read = true
+        data
+      end
+    end
+
+    def read?
+      @is_read == true
+    end
+
+    def data
+      @data ||= @part.body.to_s
+    end
+
+    def length
+      data.size
+    end
+
+    def attached?
+      !filename.nil?
+    end
+
+    def ==(other)
+      super || (filename == other.filename && content_type == other.content_type)
+    end
+
+    def inspect
+      %(#<Message::Attachment filename=#{filename.inspect} content_type=#{content_type.inspect}>)
+    end
+  end
+end

data/lib/astrotrain/message.rb

@@ -1,313 +1,290 @@
-require 'digest/sha1'
-require 'fileutils'
-require 'tempfile'
+# encoding: UTF-8
+
+require 'mail'
 require 'set'
+require 'iconv'
+
 
 module Astrotrain
   # Wrapper around a TMail object
   class Message
-    attr_accessor :body
+    EMAIL_REGEX = /[\w\.\_\%\+\-]+[^\.]@[\w\-\_\.]+/
+
+    # Reference to the internal Mail object that parsed the raw email.
     attr_reader :mail
 
+    # Refebrence to the original file that this Mail came from.
+    attr_reader :path
+
     class << self
-      attr_reader   :queue_path, :archive_path
       attr_accessor :recipient_header_order, :skipped_headers
     end
 
-    def self.queue_path=(path)
-      if path
-        path = File.expand_path(path)
-        FileUtils.mkdir_p path
-      end
-      @queue_path = path
-    end
-
-    def self.archive_path=(path)
-      if path
-        path = File.expand_path(path)
-        FileUtils.mkdir_p path
-      end
-      @archive_path = path
-    end
+    # Astrotrain::Message#headers does not show these headers
+    self.skipped_headers = Set.new %w(to cc from subject delivered-to
+      x-original-to received)
 
-    self.skipped_headers        = Set.new %w(date from subject delivered-to x-original-to received)
+    # This is the default order that Astrotrain will search for a matching
+    # recipient.
     self.recipient_header_order = %w(original_to delivered_to to)
-    self.queue_path             = File.join(Astrotrain.root, 'queue')
-
-    # Dumps the raw text into the queue_path.  Not really recommended, since you should
-    # set the queue_path to the directory your incoming emails are dumped into.
-    def self.queue(raw)
-      filename = nil
-      digest   = Digest::SHA1.hexdigest(raw)
-      while filename.nil? || File.exist?(filename)
-        filename = File.join(queue_path, Digest::SHA1.hexdigest(digest + rand.to_s))
-      end
-      File.open filename, 'wb' do |f|
-        f.write raw
-      end
-      filename
-    end
 
-    # Parses the given raw email text and processes it with a matching Mapping.
-    def self.receive(raw, file = nil)
-      message = parse(raw)
-      Astrotrain.callback(:pre_mapping, message)
-      Mapping.process(message, file)
-      message
-    rescue Astrotrain::ProcessingCancelled
+    # Public: Parses the raw email headers into a Astrotrain::Message instance.
+    #
+    # path - String path to the file.
+    #
+    # Returns Astrotrain::Message instance.
+    def self.read(path)
+      new(::Mail.read(path), path)
     end
 
-    # Processes the given file.  It parses it by reading the contents, and optionally
-    # archives or removes the original file.
-    def self.receive_file(path)
-      raw         = IO.read(path)
-      logged_path = path
-      if archive_path
-        daily_archive_path = archive_path / Time.now.year.to_s / Time.now.month.to_s / Time.now.day.to_s
-        FileUtils.mkdir_p(daily_archive_path)
-        logged_path = daily_archive_path / File.basename(path)
-        FileUtils.mv path, logged_path if path != logged_path
-      end
-      receive(raw, logged_path)
-    end
-
-    # Parses the raw email headers into a Astrotrain::Message instance.
+    # Public: Parses the raw email headers into a Astrotrain::Message instance.
+    #
+    # raw - String of the email content
+    #
+    # Returns Astrotrain::Message instance.
     def self.parse(raw)
-      new Mail.parse(raw)
-    end
-
-    def self.parse_email_addresses(value)
-      emails     = value.split(",")
-      collection = []
-      emails.each do |addr|
-        addr.strip!
-        next if addr.blank?
-        header = parse_email_address(addr.to_s)
-        collection << unescape(header[:email]) if !header[:email].blank?
-      end
-      collection
+      new(::Mail.new(raw))
     end
 
-    def self.parse_email_address(email)
-      return {} if email.blank?
-      begin
-        header = TMail::Address.parse(email)
-        parsed = {:name => header.name}
-        if header.is_a?(TMail::AddressGroup)
-          header = header[0]
-        end
-        if !header.blank?
-          parsed[:email] = header.address
-        end
-        parsed
-      rescue SyntaxError, TMail::SyntaxError
-        email = email.scan(/\<([^\>]+)\>/)[0]
-        if email.blank?
-          return {:name => nil, :email => nil}
-        else
-          email = email[0]
-          retry
-        end
-      end
-    end
-
-    # Stolen from Rack/Camping, remove the "+" => " " translation
-    def self.unescape(s)
-      s.gsub!(/((?:%[0-9a-fA-F]{2})+)/n){
-        [$1.delete('%')].pack('H*')
-      }
-      s
-    end
-
-    def initialize(mail)
+    def initialize(mail, path = nil)
+      @body = @html = @attachments = nil
+      @path        = path
       @mail        = mail
-      @mapping     = nil
-      @attachments = []
       @recipients  = {}
     end
 
-    # Gets the recipients of an email using the To/Delivered-To/X-Original-To headers.
-    # It's not always straightforward which email we want when dealing with filters
-    # and forward rules.
+    # Public: Gets the recipients of an email using the
+    # To/Delivered-To/X-Original-To headers.  It's not always straightforward
+    # which email we want when dealing with filters and forward rules.
+    #
+    # order - Array of email header names that specifies the order that the
+    #         list of recipient emails is assembled.  Valid strings are:
+    #         'original_to', 'delivered_to', and 'to'.
+    #
+    # Returns Array of possible recipients.
     def recipients(order = nil)
       if !@recipients.key?(order)
         order = self.class.recipient_header_order if order.blank?
+        order.push :body
         recipients = []
 
-        order.each do |key|
-          parse_email_headers(send("recipients_from_#{key}"), recipients)
+        emails = order.inject([]) do |memo, key|
+          memo.push *send("recipients_from_#{key}")
         end
-        parse_email_headers recipients_from_body, recipients
 
-        recipients.flatten!
-        recipients.uniq!
-        @recipients[order] = recipients
-      else
-        @recipients[order]
+        @recipients[order] = emails.map! { |em| em.address }
+        @recipients[order].uniq!
       end
+      @recipients[order]
     end
 
-    def recipients_from_to
-      @recipient_from_to ||= [@mail['to'].to_s]
+    # Public: Unquotes and converts the From header to UTF-8.
+    #
+    # Returns Array of Mail::Address objects
+    def from
+      @from ||= unquoted_address_header(:from)
     end
+    alias sender from
 
-    def recipients_from_delivered_to
-      @recipient_from_delivered_to ||= begin
-        delivered = @mail['Delivered-To']
-        if delivered.respond_to?(:first)
-          delivered.map! { |a| a.to_s }
-        else
-          [delivered.to_s]
-        end
-      end
+    # Public: Unquotes and converts the To header to UTF-8.
+    #
+    # Returns Array of Mail::Address objects
+    def to
+      @to ||= unquoted_address_header(:to)
     end
 
-    def recipients_from_original_to
-      @recipient_from_original_to ||= [@mail['X-Original-To'].to_s]
-    end
-
-    def recipients_from_body
-      @recipients_from_body ||= body.scan(/<[\w\.\_\%\+\-]+@[\w\-\_\.]+>/)
-    end
-
-    def sender
-      @sender ||= TMail::Unquoter.unquote_and_convert_to(@mail['from'].to_s, "utf-8")
+    # Public: Unquotes and converts the Cc header to UTF-8.
+    #
+    # Returns Array of Mail::Address objects
+    def cc
+      @cc ||= unquoted_address_header(:cc)
     end
 
+    # Public: Unquotes and converts the Subject header to UTF-8.
+    #
+    # Returns String
     def subject
       @mail.subject
-    rescue Iconv::InvalidCharacter
-      @mail.quoted_subject
     end
 
+    # Public: Gets the unique message-id for the email, with the surrounding
+    # `<` and `>` parsed out.
+    #
+    # Returns String
     def message_id
-      @message_id ||= header('message-id').to_s.gsub(/^<|>$/, '')
+      @mail.message_id
     end
 
+    # Public: Gets the plain/text body of the email.
+    #
+    # Returns String
     def body
-      @body ||= process_message_body(:body)
+      process_message_body if !@body
+      @body
     end
 
+    # Public: Gets the html body of the email.
+    #
+    # Returns String
     def html
-      @html ||= process_message_body(:html)
+      process_message_body if !@html
+      @html
     end
 
+    # Public: Gets the attachments in the email.
+    #
+    # Returns Array of Astrotrain::Attachment objects.
     def attachments
-      @attachments ||= process_message_body(:attachments)
-    end
-
-    def raw
-      @mail.port.to_s
-    end
-
-    def header(key)
-      headers[key]
+      process_message_body if !@attachments
+      @attachments
     end
 
+    # Public: Builds a hash of headers, skipping the keys specified in
+    # #skipped_headers.  If header values cannot be parsed, the original
+    # raw value is provided.
+    #
+    # Returns Hash of the headers with String keys and values.
     def headers
       @headers ||= begin
-        h = {}
-        @mail.header.each do |key, value|
-          next if self.class.skipped_headers.include?(key)
-          h[key] = read_header(key) 
+        @mail.header.fields.inject({}) do |memo, field|
+          name = field.name.downcase.to_s
+          header = unquoted_header(name)
+          self.class.skipped_headers.include?(name) ?
+            memo :
+            memo.update(name => self.class.unescape(unquoted_header(name)))
         end
-        h
       end
     end
 
-    class Attachment
-      def initialize(part)
-        @part    = part
-        @is_read = false
-      end
-
-      def content_type
-        @part.content_type
-      end
-
-      def filename
-        @filename ||= @part.type_param("name") || @part.disposition_param('filename')
-      end
-
-      # For IO API compatibility when used with Rest-Client
-      def close
-      end
+    # UTILITY METHODS
 
-      alias path filename
+    # Parses the 'To' header for email address.
+    #
+    # Returns Array of Mail::Address objects
+    def recipients_from_to
+      to
+    end
 
-      def read(value = nil)
-        if read?
-          nil
-        else
-          @is_read = true
-          data
-        end
-      end
+    # Parses the 'Delivered-To' header for email address.
+    #
+    # Returns Array of Mail::Address objects
+    def recipients_from_delivered_to
+      @recipients_from_delivered_to ||= unquoted_address_header('delivered-to')
+    end
 
-      def read?
-        @is_read == true
-      end
+    # Parses the 'X-Original-To' header for email address.
+    #
+    # Returns Array of Mail::Address objects
+    def recipients_from_original_to
+      @recipients_from_original_to ||= unquoted_address_header('x-original-to')
+    end
 
-      def data
-        @part.body
+    # Parses out all email addresses from the body of the email.
+    #
+    # Returns Array of Mail::Address objects
+    def recipients_from_body
+      @recipients_from_body ||= begin
+        emails_from_body = body.scan(EMAIL_REGEX)
+        address_list_for(emails_from_body)
       end
+    end
 
-      def attached?
-        !filename.nil?
+    # Parses the quoted header values: `=?...?=`.
+    #
+    # key - String or Symbol header name
+    #
+    # Returns unquoted String.
+    def unquoted_header(key)
+      if header = @mail[key]
+        value = header.respond_to?(:map) ?
+          header.map { |h| h.value }.join("\n") :
+          header.value
+        Mail::Encodings.value_decode(value)
+      else
+        ''
       end
+    end
 
-      def ==(other)
-        super || (filename == other.filename && content_type == other.content_type)
+    # Parses the given header for email addresses. Handles the case where some
+    # keys return arrays if there are multiple values.
+    #
+    # key - String or Symbol header name
+    #
+    # Returns Array of Mail::Address objects
+    def unquoted_address_header(key)
+      if header = @mail[key]
+        emails = if header.respond_to?(:value)
+          [header.value]
+        else
+          header.map { |h| h.value }
+        end
+        address_list_for(emails)
+      else
+        []
       end
+    end
 
-      def inspect
-        %(#<Message::Attachment filename=#{filename.inspect} content_type=#{content_type.inspect}>)
-      end
+    # Uses Mail::AddressList to parse the given comma separated emails.
+    #
+    # emails - Array of String emails (foo@example.com, Bar <bar@example.com...)
+    #
+    # Returns Array of Mail::Address objects
+    def address_list_for(emails)
+      emails = emails * ", "
+      list   = Mail::AddressList.new(self.class.unescape(emails))
+      addrs  = list.addresses.each { |a| a.decoded }
+      addrs.uniq!
+      addrs
+    rescue Mail::Field::ParseError
+      address_list_for(emails.scan(EMAIL_REGEX))
     end
 
-  protected
-    def read_header(key)
-      header = @mail.header[key]
-      begin
-        header.to_s
-      rescue
-        header.raw_body
-      end
+    # Stolen from Rack/Camping, remove the "+" => " " translation
+    def self.unescape(s)
+      s.gsub!(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      }
+      s
     end
 
-    def process_message_body(var = nil)
+    # Parses the mail's parts, assembling the plain/HTML Strings, as well as
+    # any attachments.
+    #
+    # Returns nothing.
+    def process_message_body
+      @attachments = []
       if @mail.multipart?
-        @attachments.clear
         @body, @html = [], []
         scan_parts(@mail)
         @body = @body.join("\n")
         @html = @html.join("\n")
       else
-        if @mail.content_type == 'text/html'
-          @html = @mail.body
+        if @mail.content_type =~ /text\/html/
+          @html = @mail.body.to_s
           @body = ''
         else
-          @body = @mail.body
+          @body = @mail.body.to_s
           @html = ''
         end
       end
-      if !@mail.charset
-        @body = convert_to_utf8(@body)
-        @html = convert_to_utf8(@html)
-      end
-      instance_variable_get "@#{var}" if var
+
+      @body = convert_to_utf8(@body)
+      @html = convert_to_utf8(@html)
     end
 
+    # Recursive method to scan all the parts of the given part.
+    #
+    # Returns nothing.
     def scan_parts(message)
       message.parts.each do |part|
         if part.multipart?
           scan_parts(part)
         else
           case part.content_type
-            when 'text/plain'
-              @body << part.body
-            when 'text/html'
-              @html << part.body
+            when /text\/plain/
+              @body << part.body.to_s
+            when /text\/html/
+              @html << part.body.to_s
             else
               att = Attachment.new(part)
               @attachments << att if att.attached?
@@ -316,27 +293,36 @@ module Astrotrain
       end
     end
 
-    def parse_email_headers(values, collection)
-      values.each do |value|
-        if !value.blank?
-          collection.push *self.class.parse_email_addresses(value)
-        end
+    # Converts a given String to UTF-8.
+    # If the message has no charset assigned, we'll attempt to detect it
+    # then convert it to UTF-8.
+    #
+    # s - unconverted String in the wrong character set
+    #
+    # Returns converted String.
+    def convert_to_utf8(s)
+      # If this string is already valid UTF-8 just hand it back
+      return s if s.as_utf8.valid?
+
+      # First lets try to detect the encoding if the message didn't specify
+      if !@mail.charset && detection = CharlockHolmes::EncodingDetector.detect(s)
+        @mail.charset = detection[:encoding]
       end
-    end
 
-    # Attempts to run iconv conversions in common charsets to UTF-8.  Needed for 
-    # those crappy emails that don't properly specify a charset in the headers. 
-    ICONV_CONVERSIONS = %w(utf-8 ISO-8859-1 ISO-8859-2 ISO-8859-3 ISO-8859-4 ISO-8859-5 ISO-8859-6 ISO-8859-7 ISO-8859-8 ISO-8859-9
-      ISO-8859-15 GB2312)
-    def convert_to_utf8(s)
-      ICONV_CONVERSIONS.each do |from|
-        begin
-          return Iconv.iconv(ICONV_CONVERSIONS[0], from, s).to_s
-        rescue Iconv::IllegalSequence
-        ensure
-          s
-        end
+      # if the encoding was already set or we just detected it AND it's not already
+      # set to UTF-8 - try to transcode the body into UTF-8
+      if @mail.charset && @mail.charset != 'UTF-8'
+        s = CharlockHolmes::Converter.convert s, @mail.charset, 'UTF-8'
       end
+
+      # By the time we get here, `s` is either UTF-8 or we need to force it to be
+      # But, even if it's UTF-8 we could be in the case where the charset on the
+      # message was set to UTF-8 but is in fact invalid.
+      # So for either case, we want to make sure the output is valid UTF-8 - even
+      # if it means mutating the invalid string.
+      # Also we're not reusing the String::UTF8 version of `s` from above here
+      # because by this point, it may be a new string.
+      s.as_utf8.clean.as_raw
     end
   end
-end
+end