entp-astrotrain 0.2.1 → 0.3.0

data/Rakefile

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'rake'
 
 begin
@@ -9,7 +8,19 @@ begin
     gem.email = "technoweenie@gmail.com"
     gem.homepage = "http://github.com/entp/astrotrain"
     gem.authors = ["technoweenie"]
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+
+    dm_ver = "0.9.11"
+    gem.add_runtime_dependency 'addressable',   '2.0.2'
+    gem.add_runtime_dependency "tmail",         "1.2.3.1"
+    gem.add_runtime_dependency "dm-core",        dm_ver # The datamapper ORM
+    gem.add_runtime_dependency "dm-aggregates",  dm_ver # Provides your DM models with count, sum, avg, min, max, etc.
+    gem.add_runtime_dependency "dm-timestamps",  dm_ver # Automatically populate created_at, created_on, etc. when those properties are present.
+    gem.add_runtime_dependency "dm-types",       dm_ver # Provides additional types, including csv, json, yaml.
+    gem.add_runtime_dependency "dm-validations", dm_ver # Validation framework
+    gem.add_development_dependency "context"
+    gem.add_development_dependency "rr"
+    gem.add_development_dependency "sinatra"
+    gem.add_development_dependency "xmppr4-simple"
   end
 
 rescue LoadError
@@ -67,42 +78,8 @@ namespace :at do
 
   desc "Start astrotrain DRb server."
   task :process => :init do
-    pid_filename = File.join(Astrotrain.root, 'log', 'astrotrain_job.pid')
-
-    FileUtils.mkdir_p File.dirname(pid_filename)
-    require 'benchmark'
-
-    begin
-      File.open(pid_filename, 'w') { |f| f << Process.pid.to_s }
-      SLEEP = 5
-       
-      trap('TERM') { puts 'Exiting...'; $exit = true }
-      trap('INT')  { puts 'Exiting...'; $exit = true }
-
-      loop do
-        count = nil
-
-        realtime = Benchmark.realtime do
-          files = Dir["#{Astrotrain::Message.queue_path}/*"]
-          files.each do |mail|
-            Astrotrain::Message.receive_file(mail)
-          end
-          count = files.size
-        end
-
-        break if $exit
-
-        if count.zero?
-          sleep(SLEEP)
-        else
-          puts "#{count} mails processed at %.4f m/s ..." % [count / realtime]
-        end
-      
-        break if $exit
-      end
-    ensure
-      FileUtils.rm(pid_filename) rescue nil
-    end
+    require 'astrotrain/worker'
+    Astrotrain::Worker.start
   end
 end
 

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.3.0

data/astrotrain.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{astrotrain}
-  s.version = "0.2.1"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["technoweenie"]
-  s.date = %q{2009-09-23}
+  s.date = %q{2009-09-25}
   s.email = %q{technoweenie@gmail.com}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
      "lib/astrotrain/mapping/transport.rb",
      "lib/astrotrain/message.rb",
      "lib/astrotrain/tmail.rb",
+     "lib/astrotrain/worker.rb",
      "lib/vendor/rest-client/README.rdoc",
      "lib/vendor/rest-client/Rakefile",
      "lib/vendor/rest-client/bin/restclient",
@@ -45,10 +46,9 @@ Gem::Specification.new do |s|
      "lib/vendor/rest-client/spec/request_errors_spec.rb",
      "lib/vendor/rest-client/spec/resource_spec.rb",
      "lib/vendor/rest-client/spec/rest_client_spec.rb",
-     "tasks/doc.thor",
-     "tasks/merb.thor",
      "test/api_test.rb",
      "test/fixtures/apple_multipart.txt",
+     "test/fixtures/bad_content_type.txt",
      "test/fixtures/basic.txt",
      "test/fixtures/custom.txt",
      "test/fixtures/fwd.txt",
@@ -89,8 +89,41 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<addressable>, ["= 2.0.2"])
+      s.add_runtime_dependency(%q<tmail>, ["= 1.2.3.1"])
+      s.add_runtime_dependency(%q<dm-core>, ["= 0.9.11"])
+      s.add_runtime_dependency(%q<dm-aggregates>, ["= 0.9.11"])
+      s.add_runtime_dependency(%q<dm-timestamps>, ["= 0.9.11"])
+      s.add_runtime_dependency(%q<dm-types>, ["= 0.9.11"])
+      s.add_runtime_dependency(%q<dm-validations>, ["= 0.9.11"])
+      s.add_development_dependency(%q<context>, [">= 0"])
+      s.add_development_dependency(%q<rr>, [">= 0"])
+      s.add_development_dependency(%q<sinatra>, [">= 0"])
+      s.add_development_dependency(%q<xmppr4-simple>, [">= 0"])
     else
+      s.add_dependency(%q<addressable>, ["= 2.0.2"])
+      s.add_dependency(%q<tmail>, ["= 1.2.3.1"])
+      s.add_dependency(%q<dm-core>, ["= 0.9.11"])
+      s.add_dependency(%q<dm-aggregates>, ["= 0.9.11"])
+      s.add_dependency(%q<dm-timestamps>, ["= 0.9.11"])
+      s.add_dependency(%q<dm-types>, ["= 0.9.11"])
+      s.add_dependency(%q<dm-validations>, ["= 0.9.11"])
+      s.add_dependency(%q<context>, [">= 0"])
+      s.add_dependency(%q<rr>, [">= 0"])
+      s.add_dependency(%q<sinatra>, [">= 0"])
+      s.add_dependency(%q<xmppr4-simple>, [">= 0"])
     end
   else
+    s.add_dependency(%q<addressable>, ["= 2.0.2"])
+    s.add_dependency(%q<tmail>, ["= 1.2.3.1"])
+    s.add_dependency(%q<dm-core>, ["= 0.9.11"])
+    s.add_dependency(%q<dm-aggregates>, ["= 0.9.11"])
+    s.add_dependency(%q<dm-timestamps>, ["= 0.9.11"])
+    s.add_dependency(%q<dm-types>, ["= 0.9.11"])
+    s.add_dependency(%q<dm-validations>, ["= 0.9.11"])
+    s.add_dependency(%q<context>, [">= 0"])
+    s.add_dependency(%q<rr>, [">= 0"])
+    s.add_dependency(%q<sinatra>, [">= 0"])
+    s.add_dependency(%q<xmppr4-simple>, [">= 0"])
   end
 end

data/lib/astrotrain.rb

@@ -36,8 +36,6 @@ private
   def self.load_dependencies
     require 'rubygems'
     gem 'addressable',   '2.0.2'
-    gem "tmail",         "1.2.3.1"
-    gem "xmpp4r-simple", "0.8.8"
 
     dm_ver = "0.9.11"
     gem "dm-core",        dm_ver # The datamapper ORM
@@ -48,7 +46,7 @@ private
 
     $LOAD_PATH.unshift File.join(lib_root, 'vendor', 'rest-client', 'lib')
 
-    %w(dm-core dm-aggregates dm-timestamps dm-types dm-validations xmpp4r-simple tmail rest_client).each do |lib|
+    %w(dm-core dm-aggregates dm-timestamps dm-types dm-validations tmail rest_client).each do |lib|
       require lib
     end
   end

data/lib/astrotrain/api.rb

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'sinatra'
 
 before do

data/lib/astrotrain/logged_mail.rb

@@ -16,21 +16,26 @@ module Astrotrain
     property :sender,        String,  :index => true, :size => 255, :length => 1..255
     property :recipient,     String,  :index => true, :size => 255, :length => 1..255
     property :subject,       String,  :index => true, :size => 255, :length => 1..255
+    property :mail_file,     String,  :size => 255, :length => 1..255
     property :created_at,    DateTime
     property :delivered_at,  DateTime
-    property :error_message, String, :size => 255, :length => 1..255
+    property :error_message, Text
 
     belongs_to :mapping
 
-    def self.from(message)
+    def self.from(message, file = nil)
       logged = new
       begin
-        logged.sender  = Message.parse_email_addresses(message.sender).first
-        logged.subject = message.subject
+        logged.sender    = Message.parse_email_addresses(message.sender).first
+        logged.subject   = message.subject
+        logged.mail_file = file if file
       end
       if !block_given? || yield(logged)
         begin
           logged.save
+          if logged.delivered_at && File.exist?(logged.mail_file.to_s)
+            FileUtils.rm_rf logged.mail_file
+          end
         rescue
           puts $!.inspect
         end

data/lib/astrotrain/mapping.rb

@@ -38,8 +38,8 @@ module Astrotrain
 
     # Processes a given message.  It finds a mapping, creates a LoggedMail record,
     # and attempts to process the message.
-    def self.process(message)
-      LoggedMail.from(message) do |logged|
+    def self.process(message, file = nil)
+      LoggedMail.from(message, file) do |logged|
         save_logged = begin
           mapping, recipient = match(message.recipients)
           if mapping
@@ -50,7 +50,7 @@ module Astrotrain
           end
           LoggedMail.log_processed # save successfully processed messages?
         rescue
-          logged.error_message = "#{$!.class}: #{$!}"
+          logged.error_message = "#{$!.message}\n#{$!.backtrace.join("\n")}"
         end
         Astrotrain.callback(:post_processing, message, mapping, logged)
         save_logged

data/lib/astrotrain/mapping/jabber.rb

@@ -1,23 +1,28 @@
-module Astrotrain
-  class Mapping
-    # This is experimental.  No attachments are supported.
-    class Jabber < Transport
-      class << self
-        attr_accessor :login, :password
-      end
+begin
+  require 'xmpp4r-simple'
+  module Astrotrain
+    class Mapping
+      # This is experimental.  No attachments are supported.
+      class Jabber < Transport
+        class << self
+          attr_accessor :login, :password
+        end
 
-      def process
-        return unless Transport.processing
-        connection.deliver(@mapping.destination, content)
-      end
+        def process
+          return unless Transport.processing
+          connection.deliver(@mapping.destination, content)
+        end
 
-      def connection
-        @connection ||= ::Jabber::Simple.new(self.class.login, self.class.password)
-      end
+        def connection
+          @connection ||= ::Jabber::Simple.new(self.class.login, self.class.password)
+        end
 
-      def content
-        @content ||= "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [fields[:from], fields[:to], fields[:subject], fields[:emails] * ", ", fields[:body]]
+        def content
+          @content ||= "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [fields[:from], fields[:to], fields[:subject], fields[:emails] * ", ", fields[:body]]
+        end
       end
     end
   end
+rescue LoadError
+  puts "Install xmpp4r-simple for Jabber support."
 end

data/lib/astrotrain/message.rb

@@ -49,25 +49,25 @@ module Astrotrain
     end
 
     # Parses the given raw email text and processes it with a matching Mapping.
-    def self.receive(raw)
+    def self.receive(raw, file = nil)
       message = parse(raw)
       Astrotrain.callback(:pre_mapping, message)
-      Mapping.process(message)
+      Mapping.process(message, file)
       message
     end
 
     # Processes the given file.  It parses it by reading the contents, and optionally
     # archives or removes the original file.
-    def self.receive_file(path, raw = nil)
-      message = receive IO.read(path)
+    def self.receive_file(path)
+      raw         = IO.read(path)
+      logged_path = path
       if archive_path
         daily_archive_path = archive_path / Time.now.year.to_s / Time.now.month.to_s / Time.now.day.to_s
         FileUtils.mkdir_p(daily_archive_path)
-        FileUtils.mv path, daily_archive_path / File.basename(path)
-      else
-        FileUtils.rm_rf path
+        logged_path = daily_archive_path / File.basename(path)
+        FileUtils.mv path, logged_path if path != logged_path
       end
-      message
+      receive(raw, logged_path)
     end
 
     # Parses the raw email headers into a Astrotrain::Message instance.

data/lib/astrotrain/worker.rb

@@ -0,0 +1,61 @@
+require 'benchmark'
+module Astrotrain
+  class Worker
+    attr_accessor :logger, :sleep_duration, :name
+
+    def self.start(options = {})
+      new(options).run
+    end
+
+    def initialize(options = {})
+      @name            = options[:name]  || "pid:#{Process.pid}"
+      @pid             = options[:pid]   || File.join(Astrotrain.root, 'log', 'astrotrain_job.pid')
+      @sleep_duration  = options[:sleep] || 5
+      @logger          = options.key?(:logger) ? options[:logger] : STDOUT
+    end
+
+    def run
+      setup do
+        files = 0
+        Dir.foreach(Message.queue_path) do |f|
+          next if f =~ /^\.{1,2}$/
+          files += 1
+          file = File.join(Message.queue_path, f)
+          Message.receive_file(file)
+        end
+        files
+      end
+    end
+
+    def say(s)
+      @logger << "#{s}\n" if @logger
+    end
+
+    def setup
+      say "*** Starting Astrotrain::Worker #{@name}"
+      FileUtils.mkdir_p File.dirname(@pid)
+
+      File.open(@pid, 'w') { |f| f << Process.pid.to_s }
+       
+      trap('TERM') { puts 'Exiting...'; $exit = true }
+      trap('INT')  { puts 'Exiting...'; $exit = true }
+
+      loop do
+        count    = nil
+        realtime = Benchmark.realtime { count = yield }
+
+        break if $exit
+
+        if count.zero?
+          sleep(@sleep_duration)
+        else
+          puts "#{count} mails processed at %.4f m/s ..." % [count / realtime]
+        end
+
+        break if $exit
+      end
+    ensure
+      FileUtils.rm(@pid) rescue nil
+    end
+  end
+end

data/lib/vendor/rest-client/lib/rest_client.rb

@@ -137,7 +137,6 @@ module RestClient
 
 			net = Net::HTTP.new(uri.host, uri.port)
 			net.use_ssl = uri.is_a?(URI::HTTPS)
-
 			net.start do |http|
 				## Ok. I know this is weird but it's a hack for now
 				## this lets process_result determine if it should read the body

data/lib/vendor/rest-client/lib/rest_client/net_http_ext.rb

@@ -5,6 +5,8 @@
 # Taken from:	
 # http://www.missiondata.com/blog/ruby/29/streaming-data-to-s3-with-ruby/
 
+Net::HTTP.ssl_context_accessor(:tmp_dh_callback)
+
 module Net
 	class HTTP
 		alias __request__ request

data/test/fixtures/bad_content_type.txt

@@ -0,0 +1,27 @@
+Delivered-To: "Processor" <processor@astrotrain.com>
+Message-ID: <a16be7390810161014n52b603e9k1aa6bb803c6735aa@mail.gmail.com>
+Date: Thu, 16 Oct 2008 10:14:18 -0700
+From: Bob <user@example.com>
+To: Processor <processor@astrotrain.com>
+Subject: Fwd: blah blah
+MIME-Version: 1.0
+X-Custom: reply
+Content-Type: multipart/mixed; boundary="====boundary===="
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline
+
+--====boundary====
+Content-Type: text/plain; charset="us-ascii"
+
+This message is being generated automatically to notify you
+that PowerMTA has crashed on mtasv.net.
+
+As the information below is likely to be essential for debugging
+the problem, please forward this message to <support@port25.com>.
+Thank you.
+
+--====boundary====
+Content-Type: text/plain; charset="us-ascii"
+
+Yo
+--====boundary====--

data/test/logged_mail_test.rb

@@ -7,7 +7,7 @@ class Astrotrain::LoggedMailTest < Astrotrain::TestCase
       @mapping = Astrotrain::Mapping.create!(:email_user => '*')
       @raw     = mail(:custom)
       @message = Astrotrain::Message.parse(@raw)
-      @logged  = Astrotrain::LoggedMail.from(@message) do |l|
+      @logged  = Astrotrain::LoggedMail.from(@message, 'foo/bar') do |l|
         l.recipient = @message.recipients(%w(delivered_to)).first
         l.mapping   = @mapping
       end
@@ -17,6 +17,10 @@ class Astrotrain::LoggedMailTest < Astrotrain::TestCase
       assert_equal @message.recipients(%w(delivered_to)).first, @logged.recipient
     end
 
+    it "sets mail_file" do
+      assert_equal 'foo/bar', @logged.mail_file
+    end
+
     it "sets sender" do
       assert_equal 'user@example.com', @logged.sender
     end

data/test/test_helper.rb

@@ -18,6 +18,9 @@ module Astrotrain
     })
   end
 
+  LoggedMail.auto_migrate!
+  Mapping.auto_migrate!
+
   LoggedMail.log_path = Astrotrain.root / 'messages'
   FileUtils.rm_rf   LoggedMail.log_path
   FileUtils.mkdir_p LoggedMail.log_path