astrotrain 0.3.1

data/lib/astrotrain/logged_mail.rb

@@ -0,0 +1,46 @@
+module Astrotrain
+  # Logs details of each incoming message.  
+  class LoggedMail
+    include DataMapper::Resource
+
+    class << self
+      attr_accessor :log_path, :log_processed
+    end
+
+    # Enabling this will save records for every processed email, not just the errored emails.
+    self.log_processed = false
+    self.log_path      = File.join(Astrotrain.root, 'messages')
+
+    property :id,            Serial
+    property :mapping_id,    Integer, :index => true
+    property :sender,        String,  :index => true, :size => 255, :length => 1..255
+    property :recipient,     String,  :index => true, :size => 255, :length => 1..255
+    property :subject,       String,  :index => true, :size => 255, :length => 1..255
+    property :mail_file,     String,  :size => 255, :length => 1..255
+    property :created_at,    DateTime
+    property :delivered_at,  DateTime
+    property :error_message, Text
+
+    belongs_to :mapping
+
+    def self.from(message, file = nil)
+      logged = new
+      begin
+        logged.sender    = Message.parse_email_addresses(message.sender).first
+        logged.subject   = message.subject
+        logged.mail_file = file if file
+      end
+      if !block_given? || yield(logged)
+        begin
+          logged.save
+          if logged.delivered_at && File.exist?(logged.mail_file.to_s)
+            FileUtils.rm_rf logged.mail_file
+          end
+        rescue
+          puts $!.inspect
+        end
+      end
+      logged
+    end
+  end
+end

data/lib/astrotrain/mapping.rb

@@ -0,0 +1,157 @@
+module Astrotrain
+  class Mapping
+    include DataMapper::Resource
+
+    class << self
+      attr_accessor :default_domain
+      attr_accessor :transports
+    end
+
+    self.transports     = {"HTTP Post" => 'http_post', "Jabber" => 'jabber'}
+    self.default_domain = 'astrotrain.com'
+
+    property :id,           Serial
+    property :email_user,   String, :size => 255, :length => 1..255, :index => :email, :format => /^[\w\.\_\%\+\-]*\*?$/
+    property :email_domain, String, :size => 255, :lenght => 1..255, :index => :email, :format => /^[\w\-\_\.]+$/, :default => lambda { default_domain }
+    property :destination,  String, :size => 255, :length => 1..255
+    property :transport,    String, :size => 255, :set => transports.values, :default => 'http_post'
+    property :separator,    String, :size => 255
+
+    validates_is_unique :email_user, :scope => :email_domain
+    validates_format :destination, :as => /^(https?:)\/\/[^\/]+\/?/i, :if => :destination_uses_url?
+    validates_format :destination, :as => :email_address, :if => :destination_uses_email?
+
+    has n, :logged_mails, :order => [:created_at.desc]
+
+    # returns a mapping for the given array of email addresses
+    def self.match(email_addresses)
+      email_addresses.each do |email_address|
+        email_address.strip!
+        email_address.downcase!
+        name, domain = email_address.split("@")
+        if mapping = match_by_address(name, domain) || match_by_wildcard(name, domain)
+          return [mapping, email_address]
+        end
+      end
+      nil
+    end
+
+    # Processes a given message.  It finds a mapping, creates a LoggedMail record,
+    # and attempts to process the message.
+    def self.process(message, file = nil)
+      LoggedMail.from(message, file) do |logged|
+        save_logged = begin
+          mapping, recipient = match(message.recipients)
+          if mapping
+            logged.recipient = recipient
+            logged.mapping   = mapping
+            mapping.process(message, recipient)
+            logged.delivered_at = Time.now.utc
+          end
+          LoggedMail.log_processed # save successfully processed messages?
+        rescue
+          logged.error_message = "#{$!.message}\n#{$!.backtrace.join("\n")}"
+        end
+        Astrotrain.callback(:post_processing, message, mapping, logged)
+        save_logged
+      end
+    end
+
+    # Processes a given message and recipient against the mapping's transport.
+    def process(message, recipient)
+      Astrotrain.callback(:pre_processing, message, self)
+      Transport.process(message, self, recipient)
+    end
+
+    # returns true if the email matches this mapping.  Wildcards in the name are allowed.
+    # A mapping with foo*@bar.com will match foo@bar.com and food@bar.com, but not foo@baz.com.
+    def match?(name, domain)
+      email_domain == domain && name =~ email_user_regex
+    end
+
+    def destination_uses_url?
+      transport == 'http_post'
+    end
+
+    def destination_uses_email?
+      transport == 'jabber'
+    end
+
+    def full_email
+      "#{email_user}@#{email_domain}"
+    end
+
+    # Looks for the mapping's separator in the message body and pulls only the content
+    # above it.  Assuming a separator of '===='...
+    #
+    #   This will be kept
+    #   
+    #   On Thu, Sep 3, 2009 at 12:34 AM... (this will be removed)
+    #   ====
+    #
+    #   > Everything here will be removed.
+    #
+    def find_reply_from(body)
+      return    if separator.blank?
+      return '' if body.blank?
+      lines = body.split("\n")
+      delim_line = found_empty = nil
+
+      (lines.size - 1).downto(0) do |i|
+        line = lines[i]
+        if !delim_line && line.include?(separator)
+          delim_line = i
+        elsif delim_line && !found_empty
+          delim_line = i
+          found_empty = line.strip.blank?
+        elsif delim_line && found_empty
+          if date_reply_line?(line) || line.strip.blank?
+            delim_line = i
+          else
+            break
+          end
+        end
+      end
+
+      if delim_line
+        body = if delim_line.zero?
+          []
+        elsif lines.size >= delim_line
+          lines[0..delim_line-1]
+        else
+          lines
+        end.join("\n")
+      elsif body.frozen?
+        body = body.dup
+      end
+      body.strip!
+      body
+    end
+
+  protected
+    def self.match_by_address(name, domain)
+      first(:email_user => name, :email_domain => domain)
+    end
+
+    def self.match_by_wildcard(name, domain)
+      wildcards = all(:email_domain => domain, :email_user.like => "%*")
+      wildcards.sort! { |x, y| y.email_user.size <=> x.email_user.size }
+      wildcards.detect { |w| w.match?(name, domain) }
+    end
+
+    DATE_LANGUATE_REGEXES = [/^on\b.*wrote\b?:$/i, /^am\b.*schrieb [\w\d\s]+:$/i, /^le\b.*a écrit\b?:$/i]
+    def date_reply_line?(line)
+      DATE_LANGUATE_REGEXES.any? { |re| line =~ re }
+    end
+
+    def email_user_regex
+      @email_user_regex ||= begin
+        if email_user['*']
+          /^#{email_user.sub /\*/, '(.*)'}$/
+        else
+          /^#{email_user}$/
+        end
+      end
+    end
+  end
+end

data/lib/astrotrain/mapping/http_post.rb

@@ -0,0 +1,18 @@
+module Astrotrain
+  class Mapping
+    class HttpPost < Transport
+      def process
+        return unless Transport.processing
+        RestClient.post @mapping.destination, fields.merge(:emails => fields[:emails].join(","))
+      end
+
+      def fields
+        super
+        @message.attachments.each_with_index do |att, index|
+          @fields[:"attachments_#{index}"] = att
+        end
+        @fields
+      end
+    end
+  end
+end

data/lib/astrotrain/mapping/jabber.rb

@@ -0,0 +1,28 @@
+begin
+  require 'xmpp4r-simple'
+  module Astrotrain
+    class Mapping
+      # This is experimental.  No attachments are supported.
+      class Jabber < Transport
+        class << self
+          attr_accessor :login, :password
+        end
+
+        def process
+          return unless Transport.processing
+          connection.deliver(@mapping.destination, content)
+        end
+
+        def connection
+          @connection ||= ::Jabber::Simple.new(self.class.login, self.class.password)
+        end
+
+        def content
+          @content ||= "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [fields[:from], fields[:to], fields[:subject], fields[:emails] * ", ", fields[:body]]
+        end
+      end
+    end
+  end
+rescue LoadError
+  puts "Install xmpp4r-simple for Jabber support."
+end

data/lib/astrotrain/mapping/transport.rb

@@ -0,0 +1,55 @@
+module Astrotrain
+  class Mapping
+    class Transport
+      class << self
+        attr_accessor :processing
+      end
+
+      # Enable this turn on processing.
+      self.processing = false
+
+      attr_reader :message, :mapping
+
+      # process a given message against the mapping.  The mapping transport is checked,
+      # and the appropirate transport class handles the request.
+      def self.process(message, mapping, recipient)
+        case mapping.transport
+          when 'http_post' then HttpPost.process(message, mapping, recipient)
+          when 'jabber'    then Jabber.process(message, mapping, recipient)
+        end
+      end
+
+      def initialize(message, mapping, recipient)
+        message.body = mapping.find_reply_from(message.body)
+        @message     = message
+        @mapping     = mapping
+        @recipient   = recipient
+      end
+
+      def process
+        raise UnimplementedError
+      end
+
+      def fields
+        @fields ||= begin
+          all_emails = @message.recipients - [@recipient]
+          f = {:subject => @message.subject, :to => @recipient, :from => @message.sender, :body => @message.body, :emails => all_emails, :html => @message.html}
+          @message.headers.each do |key, value|
+            f["headers[#{key}]"] = value
+          end
+          f
+        end
+      end
+
+      # defines custom #process class methods that instantiate the class and calls a #process instance method
+      def self.inherited(child)
+        super
+        class << child
+          def process(message, mapping, recipient)
+            new(message, mapping, recipient).process
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrotrain/message.rb

@@ -0,0 +1,330 @@
+require 'digest/sha1'
+require 'fileutils'
+require 'tempfile'
+require 'set'
+
+module Astrotrain
+  # Wrapper around a TMail object
+  class Message
+    attr_accessor :body
+    attr_reader :mail, :attachments
+
+    class << self
+      attr_reader   :queue_path, :archive_path
+      attr_accessor :recipient_header_order, :skipped_headers
+    end
+
+    def self.queue_path=(path)
+      if path
+        path = File.expand_path(path)
+        FileUtils.mkdir_p path
+      end
+      @queue_path = path
+    end
+
+    def self.archive_path=(path)
+      if path
+        path = File.expand_path(path)
+        FileUtils.mkdir_p path
+      end
+      @archive_path = path
+    end
+
+    self.skipped_headers        = Set.new %w(date from subject delivered-to x-original-to received)
+    self.recipient_header_order = %w(original_to delivered_to to)
+    self.queue_path             = File.join(Astrotrain.root, 'queue')
+
+    # Dumps the raw text into the queue_path.  Not really recommended, since you should
+    # set the queue_path to the directory your incoming emails are dumped into.
+    def self.queue(raw)
+      filename = nil
+      digest   = Digest::SHA1.hexdigest(raw)
+      while filename.nil? || File.exist?(filename)
+        filename = File.join(queue_path, Digest::SHA1.hexdigest(digest + rand.to_s))
+      end
+      File.open filename, 'wb' do |f|
+        f.write raw
+      end
+      filename
+    end
+
+    # Parses the given raw email text and processes it with a matching Mapping.
+    def self.receive(raw, file = nil)
+      message = parse(raw)
+      Astrotrain.callback(:pre_mapping, message)
+      Mapping.process(message, file)
+      message
+    end
+
+    # Processes the given file.  It parses it by reading the contents, and optionally
+    # archives or removes the original file.
+    def self.receive_file(path)
+      raw         = IO.read(path)
+      logged_path = path
+      if archive_path
+        daily_archive_path = archive_path / Time.now.year.to_s / Time.now.month.to_s / Time.now.day.to_s
+        FileUtils.mkdir_p(daily_archive_path)
+        logged_path = daily_archive_path / File.basename(path)
+        FileUtils.mv path, logged_path if path != logged_path
+      end
+      receive(raw, logged_path)
+    end
+
+    # Parses the raw email headers into a Astrotrain::Message instance.
+    def self.parse(raw)
+      new Mail.parse(raw)
+    end
+
+    def self.parse_email_addresses(value)
+      emails     = value.split(",")
+      collection = []
+      emails.each do |addr|
+        addr.strip!
+        next if addr.blank?
+        header = parse_email_address(addr.to_s)
+        collection << unescape(header[:email]) if !header[:email].blank?
+      end
+      collection
+    end
+
+    def self.parse_email_address(email)
+      return {} if email.blank?
+      begin
+        header = TMail::Address.parse(email)
+        {:name => header.name, :email => header.address}
+      rescue SyntaxError, TMail::SyntaxError
+        email = email.scan(/\<([^\>]+)\>/)[0]
+        if email.blank?
+          return {:name => nil, :email => nil}
+        else
+          email = email[0]
+          retry
+        end
+      end
+    end
+
+    # Stolen from Rack/Camping, remove the "+" => " " translation
+    def self.unescape(s)
+      s.gsub!(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      }
+      s
+    end
+
+    def initialize(mail)
+      @mail        = mail
+      @mapping     = nil
+      @attachments = []
+      @recipients  = {}
+    end
+
+    # Gets the recipients of an email using the To/Delivered-To/X-Original-To headers.
+    # It's not always straightforward which email we want when dealing with filters
+    # and forward rules.
+    def recipients(order = nil)
+      if !@recipients.key?(order)
+        order = self.class.recipient_header_order if order.blank?
+        recipients = []
+
+        parse_email_headers recipients_from_body, recipients
+        order.each do |key|
+          parse_email_headers(send("recipients_from_#{key}"), recipients)
+        end
+
+        recipients.flatten!
+        recipients.uniq!
+        @recipients[order] = recipients
+      else
+        @recipients[order]
+      end
+    end
+
+    def recipients_from_to
+      @recipient_from_to ||= [@mail['to'].to_s]
+    end
+
+    def recipients_from_delivered_to
+      @recipient_from_delivered_to ||= begin
+        delivered = @mail['Delivered-To']
+        if delivered.respond_to?(:first)
+          delivered.map! { |a| a.to_s }
+        else
+          [delivered.to_s]
+        end
+      end
+    end
+
+    def recipients_from_original_to
+      @recipient_from_original_to ||= [@mail['X-Original-To'].to_s]
+    end
+
+    def recipients_from_body
+      @recipients_from_body ||= body.scan(/<[\w\.\_\%\+\-]+@[\w\-\_\.]+>/)
+    end
+
+    def sender
+      @sender ||= TMail::Unquoter.unquote_and_convert_to(@mail['from'].to_s, "utf-8")
+    end
+
+    def subject
+      @mail.subject
+    rescue Iconv::InvalidCharacter
+      @mail.quoted_subject
+    end
+
+    def body
+      @body ||= begin
+        process_message_body
+        @body
+      end
+    end
+
+    def html
+      @html ||= begin
+        process_message_body
+        @html
+      end
+    end
+
+    def raw
+      @mail.port.to_s
+    end
+
+    def header(key)
+      @headers ||= {}
+      if !@headers.key?(key)
+        @headers[key] = if self.class.skipped_headers.include?(key)
+          nil
+        else
+          header = @mail.header[key]
+          begin
+            header.to_s
+          rescue
+            header.raw_body
+          end
+        end
+      end
+      @headers[key]
+    end
+
+    def headers
+      @headers ||= begin
+        h = {}
+        @mail.header.each do |key, value|
+          header_value = header(key)
+          h[key] = header_value if header_value
+        end
+        h
+      end
+    end
+
+    class Attachment
+      def initialize(part)
+        @part    = part
+        @is_read = false
+      end
+
+      def content_type
+        @part.content_type
+      end
+
+      def filename
+        @filename ||= @part.type_param("name") || @part.disposition_param('filename')
+      end
+
+      # For IO API compatibility when used with Rest-Client
+      def close
+      end
+
+      alias path filename
+
+      def read(value = nil)
+        if read?
+          nil
+        else
+          @is_read = true
+          data
+        end
+      end
+
+      def read?
+        @is_read == true
+      end
+
+      def data
+        @part.body
+      end
+
+      def attached?
+        !filename.nil?
+      end
+
+      def ==(other)
+        super || (filename == other.filename && content_type == other.content_type)
+      end
+
+      def inspect
+        %(#<Message::Attachment filename=#{filename.inspect} content_type=#{content_type.inspect}>)
+      end
+    end
+
+  protected
+    def process_message_body
+      if @mail.multipart?
+        @attachments.clear
+        @body, @html = [], []
+        scan_parts(@mail)
+        @body = @body.join("\n")
+        @html = @html.join("\n")
+      else
+        @body = @mail.body
+        @html = ''
+      end
+      if !@mail.charset
+        @body = convert_to_utf8(@body)
+        @html = convert_to_utf8(@html)
+      end
+    end
+
+    def scan_parts(message)
+      message.parts.each do |part|
+        if part.multipart?
+          scan_parts(part)
+        else
+          case part.content_type
+            when 'text/plain'
+              @body << part.body
+            when 'text/html'
+              @html << part.body
+            else
+              att = Attachment.new(part)
+              @attachments << att if att.attached?
+          end
+        end
+      end
+    end
+
+    def parse_email_headers(values, collection)
+      values.each do |value|
+        if !value.blank?
+          collection.push *self.class.parse_email_addresses(value)
+        end
+      end
+    end
+
+    # Attempts to run iconv conversions in common charsets to UTF-8.  Needed for 
+    # those crappy emails that don't properly specify a charset in the headers. 
+    ICONV_CONVERSIONS = %w(utf-8 ISO-8859-1 ISO-8859-2 ISO-8859-3 ISO-8859-4 ISO-8859-5 ISO-8859-6 ISO-8859-7 ISO-8859-8 ISO-8859-9
+      ISO-8859-15 GB2312)
+    def convert_to_utf8(s)
+      ICONV_CONVERSIONS.each do |from|
+        begin
+          return Iconv.iconv(ICONV_CONVERSIONS[0], from, s).to_s
+        rescue Iconv::IllegalSequence
+        ensure
+          s
+        end
+      end
+    end
+  end
+end