entp-astrotrain 0.2.0

data/lib/astrotrain/mapping.rb

@@ -0,0 +1,157 @@
+module Astrotrain
+  class Mapping
+    include DataMapper::Resource
+
+    class << self
+      attr_accessor :default_domain
+      attr_accessor :transports
+    end
+
+    self.transports     = {"HTTP Post" => 'http_post', "Jabber" => 'jabber'}
+    self.default_domain = 'astrotrain.com'
+
+    property :id,           Serial
+    property :email_user,   String, :size => 255, :length => 1..255, :index => :email, :format => /^[\w\.\_\%\+\-]*\*?$/
+    property :email_domain, String, :size => 255, :lenght => 1..255, :index => :email, :format => /^[\w\-\_\.]+$/, :default => lambda { default_domain }
+    property :destination,  String, :size => 255, :length => 1..255
+    property :transport,    String, :size => 255, :set => transports.values, :default => 'http_post'
+    property :separator,    String, :size => 255
+
+    validates_is_unique :email_user, :scope => :email_domain
+    validates_format :destination, :as => /^(https?:)\/\/[^\/]+\/?/i, :if => :destination_uses_url?
+    validates_format :destination, :as => :email_address, :if => :destination_uses_email?
+
+    has n, :logged_mails, :order => [:created_at.desc]
+
+    # returns a mapping for the given array of email addresses
+    def self.match(email_addresses)
+      email_addresses.each do |email_address|
+        email_address.strip!
+        email_address.downcase!
+        name, domain = email_address.split("@")
+        if mapping = match_by_address(name, domain) || match_by_wildcard(name, domain)
+          return [mapping, email_address]
+        end
+      end
+      nil
+    end
+
+    # Processes a given message.  It finds a mapping, creates a LoggedMail record,
+    # and attempts to process the message.
+    def self.process(message)
+      LoggedMail.from(message) do |logged|
+        save_logged = begin
+          mapping, recipient = match(message.recipients)
+          if mapping
+            logged.recipient = recipient
+            logged.mapping   = mapping
+            mapping.process(message, recipient)
+            logged.delivered_at = Time.now.utc
+          end
+          LoggedMail.log_processed # save successfully processed messages?
+        rescue
+          logged.error_message = "#{$!.class}: #{$!}"
+        end
+        Astrotrain.callback(:post_processing, message, mapping, logged)
+        save_logged
+      end
+    end
+
+    # Processes a given message and recipient against the mapping's transport.
+    def process(message, recipient)
+      Astrotrain.callback(:pre_processing, message, self)
+      Transport.process(message, self, recipient)
+    end
+
+    # returns true if the email matches this mapping.  Wildcards in the name are allowed.
+    # A mapping with foo*@bar.com will match foo@bar.com and food@bar.com, but not foo@baz.com.
+    def match?(name, domain)
+      email_domain == domain && name =~ email_user_regex
+    end
+
+    def destination_uses_url?
+      transport == 'http_post'
+    end
+
+    def destination_uses_email?
+      transport == 'jabber'
+    end
+
+    def full_email
+      "#{email_user}@#{email_domain}"
+    end
+
+    # Looks for the mapping's separator in the message body and pulls only the content
+    # above it.  Assuming a separator of '===='...
+    #
+    #   This will be kept
+    #   
+    #   On Thu, Sep 3, 2009 at 12:34 AM... (this will be removed)
+    #   ====
+    #
+    #   > Everything here will be removed.
+    #
+    def find_reply_from(body)
+      return    if separator.blank?
+      return '' if body.blank?
+      lines = body.split("\n")
+      delim_line = found_empty = nil
+
+      (lines.size - 1).downto(0) do |i|
+        line = lines[i]
+        if !delim_line && line.include?(separator)
+          delim_line = i
+        elsif delim_line && !found_empty
+          delim_line = i
+          found_empty = line.strip.blank?
+        elsif delim_line && found_empty
+          if date_reply_line?(line) || line.strip.blank?
+            delim_line = i
+          else
+            break
+          end
+        end
+      end
+
+      if delim_line
+        body = if delim_line.zero?
+          []
+        elsif lines.size >= delim_line
+          lines[0..delim_line-1]
+        else
+          lines
+        end.join("\n")
+      elsif body.frozen?
+        body = body.dup
+      end
+      body.strip!
+      body
+    end
+
+  protected
+    def self.match_by_address(name, domain)
+      first(:email_user => name, :email_domain => domain)
+    end
+
+    def self.match_by_wildcard(name, domain)
+      wildcards = all(:email_domain => domain, :email_user.like => "%*")
+      wildcards.sort! { |x, y| y.email_user.size <=> x.email_user.size }
+      wildcards.detect { |w| w.match?(name, domain) }
+    end
+
+    DATE_LANGUATE_REGEXES = [/^on\b.*wrote\b?:$/i, /^am\b.*schrieb [\w\d\s]+:$/i, /^le\b.*a écrit\b?:$/i]
+    def date_reply_line?(line)
+      DATE_LANGUATE_REGEXES.any? { |re| line =~ re }
+    end
+
+    def email_user_regex
+      @email_user_regex ||= begin
+        if email_user['*']
+          /^#{email_user.sub /\*/, '(.*)'}$/
+        else
+          /^#{email_user}$/
+        end
+      end
+    end
+  end
+end

data/lib/astrotrain/message.rb

@@ -0,0 +1,313 @@
+require 'digest/sha1'
+require 'fileutils'
+require 'tempfile'
+require 'set'
+
+module Astrotrain
+  # Wrapper around a TMail object
+  class Message
+    attr_accessor :body
+    attr_reader :mail, :attachments
+
+    class << self
+      attr_reader   :queue_path, :archive_path
+      attr_accessor :recipient_header_order, :skipped_headers
+    end
+
+    def self.queue_path=(path)
+      if path
+        path = File.expand_path(path)
+        FileUtils.mkdir_p path
+      end
+      @queue_path = path
+    end
+
+    def self.archive_path=(path)
+      if path
+        path = File.expand_path(path)
+        FileUtils.mkdir_p path
+      end
+      @archive_path = path
+    end
+
+    self.skipped_headers        = Set.new %w(date from subject delivered-to x-original-to received)
+    self.recipient_header_order = %w(original_to delivered_to to)
+    self.queue_path             = File.join(Astrotrain.root, 'queue')
+
+    # Dumps the raw text into the queue_path.  Not really recommended, since you should
+    # set the queue_path to the directory your incoming emails are dumped into.
+    def self.queue(raw)
+      filename = nil
+      digest   = Digest::SHA1.hexdigest(raw)
+      while filename.nil? || File.exist?(filename)
+        filename = File.join(queue_path, Digest::SHA1.hexdigest(digest + rand.to_s))
+      end
+      File.open filename, 'wb' do |f|
+        f.write raw
+      end
+      filename
+    end
+
+    # Parses the given raw email text and processes it with a matching Mapping.
+    def self.receive(raw)
+      message = parse(raw)
+      Astrotrain.callback(:pre_mapping, message)
+      Mapping.process(message)
+      message
+    end
+
+    # Processes the given file.  It parses it by reading the contents, and optionally
+    # archives or removes the original file.
+    def self.receive_file(path, raw = nil)
+      message = receive IO.read(path)
+      if archive_path
+        daily_archive_path = archive_path / Time.now.year.to_s / Time.now.month.to_s / Time.now.day.to_s
+        FileUtils.mkdir_p(daily_archive_path)
+        FileUtils.mv path, daily_archive_path / File.basename(path)
+      else
+        FileUtils.rm_rf path
+      end
+      message
+    end
+
+    # Parses the raw email headers into a Astrotrain::Message instance.
+    def self.parse(raw)
+      new Mail.parse(raw)
+    end
+
+    def self.parse_email_addresses(value)
+      emails     = value.split(",")
+      collection = []
+      emails.each do |addr|
+        addr.strip!
+        next if addr.blank?
+        header = parse_email_address(addr.to_s)
+        collection << unescape(header[:email]) if !header[:email].blank?
+      end
+      collection
+    end
+
+    def self.parse_email_address(email)
+      return {} if email.blank?
+      begin
+        header = TMail::Address.parse(email)
+        {:name => header.name, :email => header.address}
+      rescue SyntaxError, TMail::SyntaxError
+        email = email.scan(/\<([^\>]+)\>/)[0]
+        if email.blank?
+          return {:name => nil, :email => nil}
+        else
+          email = email[0]
+          retry
+        end
+      end
+    end
+
+    # Stolen from Rack/Camping, remove the "+" => " " translation
+    def self.unescape(s)
+      s.gsub!(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      }
+      s
+    end
+
+    def initialize(mail)
+      @mail        = mail
+      @mapping     = nil
+      @attachments = []
+      @recipients  = {}
+    end
+
+    # Gets the recipients of an email using the To/Delivered-To/X-Original-To headers.
+    # It's not always straightforward which email we want when dealing with filters
+    # and forward rules.
+    def recipients(order = nil)
+      if !@recipients.key?(order)
+        order = self.class.recipient_header_order if order.blank?
+        recipients = []
+
+        parse_email_headers recipients_from_body, recipients
+        order.each do |key|
+          parse_email_headers(send("recipients_from_#{key}"), recipients)
+        end
+
+        recipients.flatten!
+        recipients.uniq!
+        @recipients[order] = recipients
+      else
+        @recipients[order]
+      end
+    end
+
+    def recipients_from_to
+      @recipient_from_to ||= [@mail['to'].to_s]
+    end
+
+    def recipients_from_delivered_to
+      @recipient_from_delivered_to ||= begin
+        delivered = @mail['Delivered-To']
+        if delivered.respond_to?(:first)
+          delivered.map! { |a| a.to_s }
+        else
+          [delivered.to_s]
+        end
+      end
+    end
+
+    def recipients_from_original_to
+      @recipient_from_original_to ||= [@mail['X-Original-To'].to_s]
+    end
+
+    def recipients_from_body
+      @recipients_from_body ||= body.scan(/<[\w\.\_\%\+\-]+@[\w\-\_\.]+>/)
+    end
+
+    def sender
+      @sender ||= TMail::Unquoter.unquote_and_convert_to(@mail['from'].to_s, "utf-8")
+    end
+
+    def subject
+      @mail.subject
+    rescue Iconv::InvalidCharacter
+      @mail.quoted_subject
+    end
+
+    def body
+      @body ||= begin
+        process_message_body
+        @body
+      end
+    end
+
+    def html
+      @html ||= begin
+        process_message_body
+        @html
+      end
+    end
+
+    def raw
+      @mail.port.to_s
+    end
+
+    def headers
+      @headers ||= begin
+        h = {}
+        @mail.header.each do |key, value|
+          next if self.class.skipped_headers.include?(key)
+          h[key] = value.to_s
+        end
+        h
+      end
+    end
+
+    class Attachment
+      def initialize(part)
+        @part    = part
+        @is_read = false
+      end
+
+      def content_type
+        @part.content_type
+      end
+
+      def filename
+        @filename ||= @part.type_param("name") || @part.disposition_param('filename')
+      end
+
+      # For IO API compatibility when used with Rest-Client
+      def close
+      end
+
+      alias path filename
+
+      def read(value = nil)
+        if read?
+          nil
+        else
+          @is_read = true
+          data
+        end
+      end
+
+      def read?
+        @is_read == true
+      end
+
+      def data
+        @part.body
+      end
+
+      def attached?
+        !filename.nil?
+      end
+
+      def ==(other)
+        super || (filename == other.filename && content_type == other.content_type)
+      end
+
+      def inspect
+        %(#<Message::Attachment filename=#{filename.inspect} content_type=#{content_type.inspect}>)
+      end
+    end
+
+  protected
+    def process_message_body
+      if @mail.multipart?
+        @attachments.clear
+        @body, @html = [], []
+        scan_parts(@mail)
+        @body = @body.join("\n")
+        @html = @html.join("\n")
+      else
+        @body = @mail.body
+        @html = ''
+      end
+      if !@mail.charset
+        @body = convert_to_utf8(@body)
+        @html = convert_to_utf8(@html)
+      end
+    end
+
+    def scan_parts(message)
+      message.parts.each do |part|
+        if part.multipart?
+          scan_parts(part)
+        else
+          case part.content_type
+            when 'text/plain'
+              @body << part.body
+            when 'text/html'
+              @html << part.body
+            else
+              att = Attachment.new(part)
+              @attachments << att if att.attached?
+          end
+        end
+      end
+    end
+
+    def parse_email_headers(values, collection)
+      values.each do |value|
+        if !value.blank?
+          collection.push *self.class.parse_email_addresses(value)
+        end
+      end
+    end
+
+    # Attempts to run iconv conversions in common charsets to UTF-8.  Needed for 
+    # those crappy emails that don't properly specify a charset in the headers. 
+    ICONV_CONVERSIONS = %w(utf-8 ISO-8859-1 ISO-8859-2 ISO-8859-3 ISO-8859-4 ISO-8859-5 ISO-8859-6 ISO-8859-7 ISO-8859-8 ISO-8859-9
+      ISO-8859-15 GB2312)
+    def convert_to_utf8(s)
+      ICONV_CONVERSIONS.each do |from|
+        begin
+          return Iconv.iconv(ICONV_CONVERSIONS[0], from, s).to_s
+        rescue Iconv::IllegalSequence
+        ensure
+          s
+        end
+      end
+    end
+  end
+end

data/lib/astrotrain/tmail.rb

@@ -0,0 +1,48 @@
+module Astrotrain
+  # custom subclass of TMail::Mail that fixes some bugs.  The fixes were pushed upstream,
+  # and this class will go away once the gem is released.
+  class Mail < TMail::Mail
+    def charset( default = nil )
+      if h = @header['content-type']
+        h['charset'] || mime_version_charset || default
+      else
+        mime_version_charset || default
+      end
+    end
+
+    # some weird emails come with the charset specified in the mime-version header:
+    #
+    #  #<TMail::MimeVersionHeader "1.0\n charset=\"gb2312\"">
+    #
+    def mime_version_charset
+      if header['mime-version'].inspect =~ /charset=('|\\")?([^\\"']+)/
+        $2
+      end
+    end
+
+    # copied from TMail::Mail, uses #charset instead of #sub_header
+    def unquoted_body(to_charset = 'utf-8')
+      from_charset = charset
+      case (content_transfer_encoding || "7bit").downcase
+        when "quoted-printable"
+          # the default charset is set to iso-8859-1 instead of 'us-ascii'.
+          # This is needed as many mailer do not set the charset but send in ISO. This is only used if no charset is set.
+          if !from_charset.blank? && from_charset.downcase == 'us-ascii'
+            from_charset = 'iso-8859-1'
+          end
+
+          TMail::Unquoter.unquote_quoted_printable_and_convert_to(quoted_body,
+            to_charset, from_charset, true)
+        when "base64"
+          TMail::Unquoter.unquote_base64_and_convert_to(quoted_body, to_charset,
+            from_charset)
+        when "7bit", "8bit"
+          TMail::Unquoter.convert_to(quoted_body, to_charset, from_charset)
+        when "binary"
+          quoted_body
+        else
+          quoted_body
+      end
+    end
+  end
+end

data/lib/astrotrain.rb

@@ -0,0 +1,55 @@
+module Astrotrain
+  CALLBACK_TYPES = [:pre_mapping, :pre_processing, :post_processing]
+  class << self
+    attr_accessor :root, :lib_root, :callbacks
+  end
+
+  def self.load(root = Dir.pwd)
+    self.root     = File.expand_path(root)
+    self.lib_root = File.expand_path(File.dirname(__FILE__))
+    load_dependencies
+    yield if block_given?
+    %w(tmail message mapping logged_mail mapping/transport mapping/http_post mapping/jabber).each do |lib|
+      require "astrotrain/#{lib}"
+    end
+    Astrotrain::Mail::ALLOW_MULTIPLE['delivered-to'] = true
+  end
+
+  def self.callback(name, *args, &block)
+    found = callbacks[name]
+    if block
+      found << block
+    else
+      found.each { |cback| cback.call(*args) }
+    end
+    found
+  end
+
+  def self.clear_callbacks
+    self.callbacks = CALLBACK_TYPES.inject({}) { |memo, ctype| memo.update(ctype => []) }
+  end
+
+  clear_callbacks
+
+private
+  # help me ryan tomayko, you're my only help
+  def self.load_dependencies
+    require 'rubygems'
+    gem 'addressable',   '2.0.2'
+    gem "tmail",         "1.2.3.1"
+    gem "xmpp4r-simple", "0.8.8"
+
+    dm_ver = "0.9.11"
+    gem "dm-core",        dm_ver # The datamapper ORM
+    gem "dm-aggregates",  dm_ver # Provides your DM models with count, sum, avg, min, max, etc.
+    gem "dm-timestamps",  dm_ver # Automatically populate created_at, created_on, etc. when those properties are present.
+    gem "dm-types",       dm_ver # Provides additional types, including csv, json, yaml.
+    gem "dm-validations", dm_ver # Validation framework
+
+    $LOAD_PATH.unshift File.join(lib_root, 'vendor', 'rest-client', 'lib')
+
+    %w(dm-core dm-aggregates dm-timestamps dm-types dm-validations xmpp4r-simple tmail rest_client).each do |lib|
+      require lib
+    end
+  end
+end

data/lib/vendor/rest-client/README.rdoc

@@ -0,0 +1,104 @@
+= REST Client -- simple DSL for accessing REST resources
+
+A simple REST client for Ruby, inspired by the Sinatra's microframework style
+of specifying actions: get, put, post, delete.
+
+== Usage: Raw URL
+
+  require 'rest_client'
+
+  RestClient.get 'http://example.com/resource'
+  RestClient.get 'https://user:password@example.com/private/resource'
+
+  RestClient.post 'http://example.com/resource', :param1 => 'one', :nested => { :param2 => 'two' }
+
+  RestClient.delete 'http://example.com/resource'
+  
+== Multipart
+
+Yeah, that's right!  This does multipart sends for you!
+
+  RestClient.post '/data', :myfile => File.new("/path/to/image.jpg")
+
+This does two things for you:
+
+* Auto-detects that you have a File value sends it as multipart
+* Auto-detects the mime of the file and sets it in the HEAD of the payload for each entry
+
+If you are sending params that do not contain a File object but the payload needs to be multipart then:
+
+  RestClient.post '/data', :foo => 'bar', :multipart => true
+
+== Streaming downloads
+
+RestClient.get('http://some/resource/lotsofdata') do |res|
+  res.read_body do |chunk|
+    .. do something with chunk ..
+  end
+end
+
+See RestClient module docs for more details.
+
+== Usage: ActiveResource-Style
+
+  resource = RestClient::Resource.new 'http://example.com/resource'
+  resource.get
+
+  private_resource = RestClient::Resource.new 'https://example.com/private/resource', 'user', 'pass'
+  private_resource.put File.read('pic.jpg'), :content_type => 'image/jpg'
+
+See RestClient::Resource module docs for details.
+
+== Usage: Resource Nesting
+
+  site = RestClient::Resource.new('http://example.com')
+  site['posts/1/comments'].post 'Good article.', :content_type => 'text/plain'
+
+See RestClient::Resource docs for details.
+
+== Shell
+
+The restclient shell command gives an IRB session with RestClient already loaded:
+
+  $ restclient
+  >> RestClient.get 'http://example.com'
+
+Specify a URL argument for get/post/put/delete on that resource:
+
+  $ restclient http://example.com
+  >> put '/resource', 'data'
+
+Add a user and password for authenticated resources:
+
+  $ restclient https://example.com user pass
+  >> delete '/private/resource'
+
+Create ~/.restclient for named sessions:
+
+  sinatra:
+    url: http://localhost:4567
+  rack:
+    url: http://localhost:9292
+  private_site:
+    url: http://example.com
+    username: user
+    password: pass
+
+Then invoke:
+
+  $ restclient private_site
+  
+== Meta
+
+Written by Adam Wiggins (adam at heroku dot com)
+
+Major modifications by Blake Mizerany
+
+Patches contributed by: Chris Anderson, Greg Borenstein, Ardekantur, Pedro Belo, Rafael Souza, Rick Olson, and Aman Gupta
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://rest-client.heroku.com
+
+http://github.com/adamwiggins/rest-client
+