entp-astrotrain 0.2.0

data/.gitignore

@@ -0,0 +1,26 @@
+.DS_Store
+.rake_tasks
+log/*
+tmp/*
+TAGS
+*~
+.#*
+pkg
+schema/schema.rb
+schema/*_structure.sql
+schema/*.sqlite3
+schema/*.sqlite
+schema/*.db
+*.sqlite
+*.sqlite3
+*.db
+src/*
+.hgignore
+.hg/*
+.svn/*
+doc
+config/database.yml
+merb/custom.rb
+test/fixtures/queue
+messages
+test/messages

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008-* ENTP
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,47 @@
+astrotrain
+==========
+
+NOTE: Astrotrain is a full gem now.  If you're looking for the old Astrotrain on merb:
+http://github.com/entp/astrotrain/tree/merb
+git://github.com/entp/astrotrain.git (merb branch)
+
+Scans incoming emails for mapped recipients and sends an HTTP POST somewhere.
+
+  # setup a config file.
+  # Point the queue_path at a directory that your mail server dumps each raw incoming mail.
+  require 'astrotrain'
+
+  Astrotrain.load path do
+    DataMapper.setup(:default, {
+      :adapter  => "mysql",
+      :database => "astrotrain",
+      :username => "root",
+      :host     => "localhost"
+    })
+  end
+  Astrotrain::Message.queue_path = "/path/to/maildir"
+  Astrotrain::Mapping::Transport.processing = true
+
+  # start up IRB
+  irb -I /var/astrotrain/lib -r config.rb
+
+  # manage mappings
+  LIB=/var/astrotrain/lib CONFIG=config.rb rake at:mappings
+  LIB=/var/astrotrain/lib CONFIG=config.rb rake at:map EMAIL=support@foo.com DEST=http://foo.com/email
+  LIB=/var/astrotrain/lib CONFIG=config.rb rake at:unmap MAP=123
+
+  # start the server that runs over the queue directory
+  LIB=/var/astrotrain/lib CONFIG=config.rb rake at:process
+
+  # start the sinatra API
+  ruby -I /var/astrotrain/lib /var/astrotrain/lib/astrotrain/api.rb config.rb
+
+A single Astrotrain process currently handles email for two production applications, processing thousands daily.
+It's far from perfect, but definitely usable.
+
+TODO
+====
+
+Gem package - I realize the load paths could be simplified a bunch, so a gem package will come soon.
+Docs
+bounced emails

data/Rakefile

@@ -0,0 +1,145 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "astrotrain"
+    gem.summary = %Q{email => http post}
+    gem.email = "technoweenie@gmail.com"
+    gem.homepage = "http://github.com/entp/astrotrain"
+    gem.authors = ["technoweenie"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+namespace :at do
+  task :init do
+    $LOAD_PATH.unshift File.expand_path(ENV['LIB']) if ENV['LIB']
+    require ENV['CONFIG'] if ENV['CONFIG']
+
+    if !Object.const_defined?(:Astrotrain)
+      require 'astrotrain'
+      Astrotrain.load
+    end
+
+    Astrotrain::Message.queue_path = ENV['QUEUE'] if ENV['QUEUE']
+    if !File.exist?(Astrotrain::Message.queue_path)
+      puts "No Queue: #{Astrotrain.queue_path.inspect}"
+      exit
+    end
+  end
+
+  desc "List Mappings"
+  task :mappings => :init do
+    Astrotrain::Mapping.all.each do |map|
+      puts "##{map.id}: #{map.full_email} => #{map.destination}"
+      puts "Separated by: #{map.separator.inspect}" if !map.separator.blank?
+      puts
+    end
+  end
+
+  desc "Add mapping EMAIL=ticket*@foo.com DEST=http://foo.com/email [TRANS=(http_post|jabber)] [SEP=REPLYABOVEHERE] [HEADERS=delivered_to,to,original_to]"
+  task :map => :init do
+    map = Astrotrain::Mapping.new
+    map.email_user, map.email_domain = ENV['EMAIL'].to_s.split('@')
+    map.destination = ENV['DEST']
+    map.transport   = ENV['TRANS'] if ENV['TRANS']
+    map.separator   = ENV['SEP']   if ENV['SEP']
+    if map.save
+      puts "Mapping created for #{map.full_email} => #{map.destination}"
+      Rake::Task['at:mappings'].invoke
+    else
+      puts map.inspect
+      puts map.errors.inspect
+    end
+  end
+
+  desc "Remove mapping MAP=123"
+  task :unmap => :init do
+    map = Astrotrain::Mapping.get(ENV["MAP"])
+    map.destroy if map
+    Rake::Task['at:mappings'].invoke
+  end
+
+  desc "Start astrotrain DRb server."
+  task :process => :init do
+    pid_filename = File.join(Astrotrain.root, 'log', 'astrotrain_job.pid')
+
+    FileUtils.mkdir_p File.dirname(pid_filename)
+    require 'benchmark'
+
+    begin
+      File.open(pid_filename, 'w') { |f| f << Process.pid.to_s }
+      SLEEP = 5
+       
+      trap('TERM') { puts 'Exiting...'; $exit = true }
+      trap('INT')  { puts 'Exiting...'; $exit = true }
+
+      loop do
+        count = nil
+
+        realtime = Benchmark.realtime do
+          files = Dir["#{Astrotrain::Message.queue_path}/*"]
+          files.each do |mail|
+            Astrotrain::Message.receive_file(mail)
+          end
+          count = files.size
+        end
+
+        break if $exit
+
+        if count.zero?
+          sleep(SLEEP)
+        else
+          puts "#{count} mails processed at %.4f m/s ..." % [count / realtime]
+        end
+      
+        break if $exit
+      end
+    ensure
+      FileUtils.rm(pid_filename) rescue nil
+    end
+  end
+end
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "astrotrain #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+desc 'Default: run test examples'
+task :default => 'test'

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/astrotrain.gemspec

@@ -0,0 +1,96 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{astrotrain}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["technoweenie"]
+  s.date = %q{2009-09-21}
+  s.email = %q{technoweenie@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README",
+     "Rakefile",
+     "VERSION",
+     "astrotrain.gemspec",
+     "config/sample.rb",
+     "lib/astrotrain.rb",
+     "lib/astrotrain/api.rb",
+     "lib/astrotrain/logged_mail.rb",
+     "lib/astrotrain/mapping.rb",
+     "lib/astrotrain/mapping/http_post.rb",
+     "lib/astrotrain/mapping/jabber.rb",
+     "lib/astrotrain/mapping/transport.rb",
+     "lib/astrotrain/message.rb",
+     "lib/astrotrain/tmail.rb",
+     "lib/vendor/rest-client/README.rdoc",
+     "lib/vendor/rest-client/Rakefile",
+     "lib/vendor/rest-client/bin/restclient",
+     "lib/vendor/rest-client/foo.diff",
+     "lib/vendor/rest-client/lib/rest_client.rb",
+     "lib/vendor/rest-client/lib/rest_client/net_http_ext.rb",
+     "lib/vendor/rest-client/lib/rest_client/payload.rb",
+     "lib/vendor/rest-client/lib/rest_client/request_errors.rb",
+     "lib/vendor/rest-client/lib/rest_client/resource.rb",
+     "lib/vendor/rest-client/rest-client.gemspec",
+     "lib/vendor/rest-client/spec/base.rb",
+     "lib/vendor/rest-client/spec/master_shake.jpg",
+     "lib/vendor/rest-client/spec/payload_spec.rb",
+     "lib/vendor/rest-client/spec/request_errors_spec.rb",
+     "lib/vendor/rest-client/spec/resource_spec.rb",
+     "lib/vendor/rest-client/spec/rest_client_spec.rb",
+     "tasks/doc.thor",
+     "tasks/merb.thor",
+     "test/api_test.rb",
+     "test/fixtures/apple_multipart.txt",
+     "test/fixtures/basic.txt",
+     "test/fixtures/custom.txt",
+     "test/fixtures/fwd.txt",
+     "test/fixtures/gb2312_encoding.txt",
+     "test/fixtures/gb2312_encoding_invalid.txt",
+     "test/fixtures/html.txt",
+     "test/fixtures/iso-8859-1.txt",
+     "test/fixtures/mapped.txt",
+     "test/fixtures/multipart.txt",
+     "test/fixtures/multipart2.txt",
+     "test/fixtures/multiple.txt",
+     "test/fixtures/multiple_delivered_to.txt",
+     "test/fixtures/multiple_with_body_recipients.txt",
+     "test/fixtures/reply.txt",
+     "test/fixtures/utf-8.txt",
+     "test/logged_mail_test.rb",
+     "test/mapping_test.rb",
+     "test/message_test.rb",
+     "test/test_helper.rb",
+     "test/transport_test.rb"
+  ]
+  s.homepage = %q{http://github.com/entp/astrotrain}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{email => http post}
+  s.test_files = [
+    "test/api_test.rb",
+     "test/logged_mail_test.rb",
+     "test/mapping_test.rb",
+     "test/message_test.rb",
+     "test/test_helper.rb",
+     "test/transport_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/config/sample.rb

@@ -0,0 +1,12 @@
+path = File.join(File.dirname(__FILE__), '..')
+$LOAD_PATH.unshift File.join(path, 'lib')
+require 'astrotrain'
+
+Astrotrain.load path do
+  DataMapper.setup(:default, {
+    :adapter  => "mysql",
+    :database => "astrotrain",
+    :username => "root",
+    :host     => "localhost"
+  })
+end

data/lib/astrotrain/api.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'sinatra'
+
+before do
+  response['Content-Type'] = 'text/plain'
+end
+
+get '/queue_size' do
+  if File.exist?(Astrotrain::Message.queue_path)
+    (Dir.entries(Astrotrain::Message.queue_path).size - 2).to_s
+  else
+    '0'
+  end
+end
+
+get '/queue/*' do
+  path = params[:splat].first
+  path.gsub! /^\/+/, ''
+  file = File.join(Astrotrain::Message.queue_path, path)
+  if File.exist?(file)
+    IO.read(file)
+  else
+    halt 404, "#{path.inspect} was not found."
+  end
+end
+
+get '/queue' do
+  data = []
+  Dir.entries(Astrotrain::Message.queue_path).each do |e|
+    data << e unless e =~ /^\.{1,2}$/
+  end
+  data * "\n"
+end
+
+unless $testing
+  configure do
+    # the idea being, you pass the name of the config file
+    #
+    #   # loads File.join(Dir.pwd, 'config.rb')
+    #   CONFIG=config ruby lib/astrotrain/api.rb
+    #
+    #   # loads File.join(Dir.pwd, 'config.rb')
+    #   ruby lib/astrotrain/api.rb config
+    #
+    # That file contains the Greymalkin.load block that initializes Sequel
+    #
+    path = ENV['CONFIG'] || ARGV.shift
+    if path[0..0] != '/'
+      path = File.join(Dir.pwd, path)
+    end
+    require File.expand_path(path)
+  end
+end

data/lib/astrotrain/logged_mail.rb

@@ -0,0 +1,41 @@
+module Astrotrain
+  # Logs details of each incoming message.  
+  class LoggedMail
+    include DataMapper::Resource
+
+    class << self
+      attr_accessor :log_path, :log_processed
+    end
+
+    # Enabling this will save records for every processed email, not just the errored emails.
+    self.log_processed = false
+    self.log_path      = File.join(Astrotrain.root, 'messages')
+
+    property :id,            Serial
+    property :mapping_id,    Integer, :index => true
+    property :sender,        String,  :index => true, :size => 255, :length => 1..255
+    property :recipient,     String,  :index => true, :size => 255, :length => 1..255
+    property :subject,       String,  :index => true, :size => 255, :length => 1..255
+    property :created_at,    DateTime
+    property :delivered_at,  DateTime
+    property :error_message, String, :size => 255, :length => 1..255
+
+    belongs_to :mapping
+
+    def self.from(message)
+      logged = new
+      begin
+        logged.sender  = Message.parse_email_addresses(message.sender).first
+        logged.subject = message.subject
+      end
+      if !block_given? || yield(logged)
+        begin
+          logged.save
+        rescue
+          puts $!.inspect
+        end
+      end
+      logged
+    end
+  end
+end

data/lib/astrotrain/mapping/http_post.rb

@@ -0,0 +1,18 @@
+module Astrotrain
+  class Mapping
+    class HttpPost < Transport
+      def process
+        return unless Transport.processing
+        RestClient.post @mapping.destination, fields.merge(:emails => fields[:emails].join(","))
+      end
+
+      def fields
+        super
+        @message.attachments.each_with_index do |att, index|
+          @fields[:"attachments_#{index}"] = att
+        end
+        @fields
+      end
+    end
+  end
+end

data/lib/astrotrain/mapping/jabber.rb

@@ -0,0 +1,23 @@
+module Astrotrain
+  class Mapping
+    # This is experimental.  No attachments are supported.
+    class Jabber < Transport
+      class << self
+        attr_accessor :login, :password
+      end
+
+      def process
+        return unless Transport.processing
+        connection.deliver(@mapping.destination, content)
+      end
+
+      def connection
+        @connection ||= ::Jabber::Simple.new(self.class.login, self.class.password)
+      end
+
+      def content
+        @content ||= "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [fields[:from], fields[:to], fields[:subject], fields[:emails] * ", ", fields[:body]]
+      end
+    end
+  end
+end

data/lib/astrotrain/mapping/transport.rb

@@ -0,0 +1,55 @@
+module Astrotrain
+  class Mapping
+    class Transport
+      class << self
+        attr_accessor :processing
+      end
+
+      # Enable this turn on processing.
+      self.processing = false
+
+      attr_reader :message, :mapping
+
+      # process a given message against the mapping.  The mapping transport is checked,
+      # and the appropirate transport class handles the request.
+      def self.process(message, mapping, recipient)
+        case mapping.transport
+          when 'http_post' then HttpPost.process(message, mapping, recipient)
+          when 'jabber'    then Jabber.process(message, mapping, recipient)
+        end
+      end
+
+      def initialize(message, mapping, recipient)
+        message.body = mapping.find_reply_from(message.body)
+        @message     = message
+        @mapping     = mapping
+        @recipient   = recipient
+      end
+
+      def process
+        raise UnimplementedError
+      end
+
+      def fields
+        @fields ||= begin
+          all_emails = @message.recipients - [@recipient]
+          f = {:subject => @message.subject, :to => @recipient, :from => @message.sender, :body => @message.body, :emails => all_emails, :html => @message.html}
+          @message.headers.each do |key, value|
+            f["headers[#{key}]"] = value
+          end
+          f
+        end
+      end
+
+      # defines custom #process class methods that instantiate the class and calls a #process instance method
+      def self.inherited(child)
+        super
+        class << child
+          def process(message, mapping, recipient)
+            new(message, mapping, recipient).process
+          end
+        end
+      end
+    end
+  end
+end