ses_machine 0.0.1.dev

data/lib/ses_machine/db.rb

@@ -0,0 +1,92 @@
+# -*- encoding : utf-8 -*-
+
+
+module SesMachine #:nodoc:
+  class DB #:nodoc:
+    class << self
+
+      def get_keywords(array)
+        array.map{|w| w.downcase.to_s}.uniq.delete_if{|w| w.size < 3}
+      end
+
+      def update_daily_stats
+        map = <<-JS
+    function() {
+      var self = this;
+      var key = this.date - (this.date % 86400000);
+      var value = {total: 1};
+
+      types.forEach(function(type) {
+        value[type] = self.bounce_type == type ? 1 : 0;
+      });
+
+      emit(key, value);
+    }
+    JS
+
+        reduce = <<-JS
+    function(key, values) {
+      var sum = {};
+      var total = 0;
+      types.forEach(function(type) {
+        sum[type] = 0;
+      });
+
+      values.forEach(function(value) {
+        for(var type in value) {
+          sum[type] += value[type];
+        }
+      });
+
+      types.forEach(function(type) {
+        total += sum[type];
+      });
+      sum['total'] = total;
+
+      return sum;
+    }
+    JS
+        t = SesMachine.database['mails']
+        t.map_reduce(map, reduce, :out => {'merge' => 'daily_stats'}, :raw => true, :scope => {'types' => SesMachine::Bounce::TYPES.values})
+      end
+
+      def update_monthly_stats
+        map = <<-JS
+    function() {
+
+      var date = new Date(this['_id'])
+
+      var key = {
+        year: date.getFullYear(),
+        month: date.getMonth() + 1
+      };
+      var value = this.value;
+
+      emit(key, value);
+    }
+    JS
+
+        reduce = <<-JS
+    function(key, values) {
+      var sum = {total: 0};
+
+      types.forEach(function(type) {
+        sum[type] = 0;
+      });
+
+      values.forEach(function(value) {
+        for(var type in value) {
+          sum[type] += value[type];
+        }
+      });
+
+      return sum;
+    }
+    JS
+        t = SesMachine.database['daily_stats']
+        t.map_reduce(map, reduce, :out => {'merge' => 'monthly_stats'}, :raw => true, :scope => {'types' => SesMachine::Bounce::TYPES.values})
+      end
+
+    end
+  end
+end

data/lib/ses_machine/errors.rb

@@ -0,0 +1,24 @@
+# -*- encoding : utf-8 -*-
+
+
+module SesMachine #:nodoc
+  module Errors #:nodoc
+
+    # Raised when the database connection has not been set up properly, either
+    # by attempting to set an object on the db that is not a +Mongo::DB+, or
+    # not setting anything at all.
+    #
+    # Example:
+    #
+    # <tt>InvalidDatabase.new("Not a DB")</tt>
+    class InvalidDatabase < StandardError
+      def initialize(database)
+        @database = database
+      end
+
+      def message
+        "Database should be a Mongo::DB, not #{@database.class.name}"
+      end
+    end
+  end
+end

data/lib/ses_machine/mailer.rb

@@ -0,0 +1,68 @@
+# -*- encoding : utf-8 -*-
+
+
+module SesMachine #:nodoc:
+  class Mailer #:nodoc:
+
+    def initialize(*args); end
+
+    def perform_delivery_ses_machine(mail)
+      raw_source = SesMachine.use_dkim? ? sign_mail(mail) : mail.encoded
+
+      begin
+        response = SesMachine.ses.send_raw_email(raw_source, :source => SesMachine.email_account)
+      rescue AWS::SES::ResponseError => e
+        response = e.response
+      end
+
+      doc = {
+        :address => mail.to,
+        :subject => mail.subject,
+        :raw_source => raw_source,
+        :date => response.headers['date'].to_time.utc,
+        :request_id => response.request_id,
+        :response => response.to_s,
+        :response_code => response.code,
+        :response_error => response.error.to_s,
+        :bounce_type => response.error? ? SesMachine::Bounce::TYPES[:unknown] : 0,
+        '_keywords' => SesMachine::DB.get_keywords([mail.to].flatten + mail.subject.split)
+      }
+
+      doc.merge!(:message_id => response.message_id) unless response.error?
+
+      SesMachine.database['mails'].insert(doc)
+    end
+
+    alias_method :deliver!, :perform_delivery_ses_machine
+
+    protected
+
+    def dkim_signer
+      key = File.readlines(SesMachine.dkim_private_key).join
+      DKIM::Signer.new(SesMachine.dkim_domain, SesMachine.dkim_selector, key)
+    end
+
+    def sign_mail(mail)
+      # Read more: http://docs.amazonwebservices.com/ses/latest/DeveloperGuide/DKIM.html
+      exclude_headers = ['Message-Id', 'Date', 'Return-Path', 'Bounces-To']
+      exclude_headers_rxp = /^(#{exclude_headers.join('|')}):/i
+      signer = dkim_signer
+      raw_source = ''
+      mail.encoded.each_line do |line|
+        unless line =~ exclude_headers_rxp
+          unless line.blank?
+            line.gsub!(/\n/, "\r\n")
+            line.gsub!(/\r+/, "\r")
+          end
+          signer.feed(line)
+          raw_source << line
+        end
+      end
+      signature = signer.finish
+      "#{signature.signature_header}\r\n#{raw_source}"
+    end
+
+  end
+end
+
+ActionMailer::Base.add_delivery_method :ses_machine, SesMachine::Mailer

data/lib/ses_machine/routing.rb

@@ -0,0 +1,8 @@
+# -*- encoding : utf-8 -*-
+
+
+Rails.application.routes.draw do
+  match "ses_machine" => "ses_machine#index", :as => :ses_machine
+  match "ses_machine/activity" => "ses_machine#activity", :as => :ses_machine_activity
+  match "ses_machine/message/:id" => "ses_machine#show_message", :as => :ses_machine_message
+end

data/lib/ses_machine/version.rb

@@ -0,0 +1,13 @@
+# -*- encoding : utf-8 -*-
+
+
+module SesMachine #:nodoc
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+    PRE   = 'dev'
+
+    STRING = [MAJOR, MINOR, TINY, PRE].compact.join('.')
+  end
+end

data/lib/ses_machine.rb

@@ -0,0 +1,88 @@
+# -*- encoding : utf-8 -*-
+
+
+lib = File.expand_path(File.dirname(__FILE__))
+$:.unshift lib unless $:.include?(lib)
+
+# require 'rubygems'
+
+# gem 'rails', '~> 2.3.8'
+# gem 'activesupport', '~> 2.3.8'
+# gem 'actionmailer', '~> 2.3.8'
+# gem 'will_paginate', '~> 2.3'
+
+require 'singleton'
+require 'mongo'
+require 'action_mailer'
+require 'action_mailer/version'
+require 'dkim'
+require 'mail'
+require 'aws/ses'
+require 'will_paginate'
+require 'ses_machine/bounce'
+require 'ses_machine/config'
+require 'ses_machine/db'
+require 'ses_machine/errors'
+require 'ses_machine/mailer'
+require 'ses_machine/routing'
+
+%w{ models controllers helpers}.each do |dir|
+  path = File.join(File.dirname(__FILE__), '..', 'app', dir)
+  $LOAD_PATH << path
+  # ActiveSupport::Dependencies.load_paths << path
+  # ActiveSupport::Dependencies.load_once_paths.delete(path)
+  # ActionController::Base.append_view_path(File.join(File.dirname(__FILE__), '..', 'app', 'views'))
+end
+
+# WillPaginate.enable_actionpack
+if defined?(::Rails::Railtie)
+  require 'will_paginate/railtie'
+end
+
+module SesMachine #:nodoc
+
+  module Hooks; end
+
+  class << self
+
+    # Sets the SesMachine configuration options. Best used by passing a block.
+    #
+    # Example:
+    #
+    #   SesMachine.configure do |config|
+    #     config.use_dkim = true
+    #     config.dkim_domain = 'example.com'
+    #     config.dkim_selector = 'ses'
+    #     config.dkim_private_key = '/path/to/private/key'
+    #   end
+    #
+    # Returns:
+    #
+    # The SesMachine +Config+ singleton instance.
+    def configure
+      SesMachine::Config.instance
+      config = SesMachine::Config.instance
+      block_given? ? yield(config) : config
+    end
+    alias :config :configure
+  end
+
+  # Take all the public instance methods from the Config singleton and allow
+  # them to be accessed through the SesMachine module directly.
+  #
+  # Example:
+  #
+  # <tt>SesMachine.database = Mongo::Connection.new.db("test")</tt>
+  SesMachine::Config.public_instance_methods(false).each do |name|
+    (class << self; self; end).class_eval <<-EOT
+      def #{name}(*args)
+        configure.send("#{name}", *args)
+      end
+    EOT
+  end
+end
+
+ENV['RAILS_ENV'] ||= Rails.env || 'development'
+ENV['RAILS_ROOT'] ||= Rails.root.to_s || File.dirname(__FILE__) + '/../../../..'
+
+SesMachine.config.load

data/lib/tasks/db.rake

@@ -0,0 +1,28 @@
+# -*- encoding : utf-8 -*-
+
+namespace :ses_machine do
+  namespace :db do
+
+    desc 'Update daily statistics'
+    task :update_daily_stats => :environment do
+      response = SesMachine::DB.update_daily_stats
+      puts "DAILY STATS (#{response['timeMillis']}ms) - #{response['ok'] ? 'Success' : 'Failure'}"
+    end
+
+    desc 'Update monthly statistics'
+    task :update_monthly_stats => :environment do
+      response = SesMachine::DB.update_monthly_stats
+      puts "MONTHLY STATS (#{response['timeMillis']}ms) - #{response['ok'] ? 'Success' : 'Failure'}"
+    end
+
+    desc 'Create database indices'
+    task :create_indices => :environment do
+      t = SesMachine.database['mails']
+      t.ensure_index([['message_id', Mongo::ASCENDING]])
+      t.ensure_index([['bounce_type', Mongo::ASCENDING]])
+      t.ensure_index([['_keywords', Mongo::ASCENDING]])
+      t.ensure_index([['date', Mongo::DESCENDING]])
+      t.ensure_index([['address', Mongo::ASCENDING]])
+    end
+  end
+end

data/lib/tasks/ses_machine.rake

@@ -0,0 +1,91 @@
+# -*- encoding : utf-8 -*-
+
+require 'net/imap'
+
+namespace :ses_machine do
+  desc 'Check email with IMAP for new bounced and spam messages'
+  task :check_email_imap => :environment do
+
+    imap = Net::IMAP.new(SesMachine.email_server, SesMachine.email_port, SesMachine.email_use_ssl)
+    imap.login(SesMachine.email_account, SesMachine.email_password)
+    t = SesMachine.database['mails']
+
+    folders = SesMachine::Bounce::TYPES.keys.map {|i| i.to_s.classify}
+    folders.delete('EmailSent')
+    folders << 'MessageNotFound'
+    folders.each do |folder|
+      imap.create("SesMachine/#{folder}") unless imap.list('SesMachine/', folder)
+    end
+
+    SesMachine.email_imap_folders.each do |folder|
+      imap.select(folder)
+      imap.search(['FROM', 'email-bounces.amazonses.com', 'NOT', 'SEEN']).each do |message|
+        data = imap.fetch(message, ['UID', 'ENVELOPE', 'RFC822'])[0]
+        mail = Mail.read_from_string(data.attr['RFC822'])
+        message_id = data.attr['ENVELOPE']['in_reply_to']
+        if message_id.blank?
+          message_id = mail['X-Original-To'].to_s.split('@').first.to_s
+        else
+          message_id = message_id.split('@').first[1..-1].to_s
+        end
+        doc = t.find_one('message_id' => message_id)
+        if doc
+          email = doc['address'].first
+          bounce = {
+            :uid => data.attr['UID'],
+            :message_id => data.attr['ENVELOPE']['message_id'].split('@').first[1..-1],
+            :raw_source => data.attr['RFC822'],
+            :date => data.attr['ENVELOPE']['date'].to_time.utc
+          }
+          if mail['auto-submitted'].to_s == 'auto-replied'
+              imap.copy(message, 'SesMachine/AutoResponder')
+              bounce_type = SesMachine::Bounce::TYPES[:auto_responder]
+              SesMachine::Hooks.auto_responder_hook(email) if SesMachine::Hooks.respond_to?(:auto_responder_hook)
+          elsif mail.bounced?
+            if mail.retryable?
+              imap.copy(message, 'SesMachine/SoftBounce')
+              bounce_type = SesMachine::Bounce::TYPES[:soft_bounce]
+              SesMachine::Hooks.soft_bounce_hook(email) if SesMachine::Hooks.respond_to?(:soft_bounce_hook)
+            else
+              imap.copy(message, 'SesMachine/HardBounce')
+              bounce_type = SesMachine::Bounce::TYPES[:hard_bounce]
+              SesMachine::Hooks.hard_bounce_hook(email) if SesMachine::Hooks.respond_to?(:hard_bounce_hook)
+            end
+            bounce.merge!(:details => mail.diagnostic_code.to_s)
+          else
+            imap.copy(message, 'SesMachine/Unknown')
+            bounce_type = SesMachine::Bounce::TYPES[:unknown]
+            SesMachine::Hooks.unknown_hook(email) if SesMachine::Hooks.respond_to?(:unknown_hook)
+          end
+          t.update({'_id' => doc['_id']}, {'$set' => {'bounce' => bounce, 'bounce_type' => bounce_type}})
+        else
+          imap.copy(message, 'SesMachine/MessageNotFound')
+        end
+      end
+    end
+    SesMachine.email_imap_folders.each do |folder|
+      imap.select(folder)
+      imap.search(['FROM', 'complaints@email-abuse.amazonses.com', 'NOT', 'SEEN']).each do |message|
+        data = imap.fetch(message, ['UID', 'ENVELOPE', 'RFC822'])[0]
+        message_id = data.attr['RFC822'].split("\r\n").grep(/\AMessage-ID: (.+)\z/ix)[1][/<(.+)@/, 1].to_s
+        doc = t.find_one('message_id' => message_id)
+        if doc
+          email = doc['address'].first
+          bounce = {
+            :uid => data.attr['UID'],
+            :message_id => data.attr['ENVELOPE']['message_id'].split('@').first[1..-1],
+            :raw_source => data.attr['RFC822'],
+            :date => data.attr['ENVELOPE']['date'].to_time.utc
+          }
+          imap.copy(message, 'SesMachine/SpamComplaint')
+          SesMachine::Hooks.spam_complaint_hook(email) if SesMachine::Hooks.respond_to?(:spam_complaint_hook)
+          t.update({'_id' => doc['_id']}, {'$set' => {'bounce' => bounce, 'bounce_type' => SesMachine::Bounce::TYPES[:spam_complaint]}})
+        else
+          imap.copy(message, 'SesMachine/MessageNotFound')
+        end
+      end
+    end
+    imap.logout()
+    imap.disconnect()
+  end
+end

data/ses_machine.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path('../lib/ses_machine/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = 'ses_machine'
+  s.version     = SesMachine::VERSION::STRING
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Kirill Nikitin']
+  s.email       = ['locke23rus@gmail.com']
+  s.homepage    = 'http://rubygems.org/gems/ses_machine'
+  s.summary     = 'SES Machine'
+  s.description = 'SES Machine description'
+  s.license     = 'MIT'
+
+  s.required_rubygems_version = '>= 1.3.6'
+
+  s.add_dependency 'mail', '>= 2.4.4'
+  s.add_dependency 'aws-ses', '>= 0.4.4'
+  s.add_dependency 'rubydkim', '>= 0.3.1'
+  s.add_dependency 'mongo', '>= 1.9.1'
+  s.add_dependency 'bson', '>= 1.9.1'
+  s.add_dependency 'bson_ext', '>= 1.9.1'
+  s.add_dependency 'will_paginate', '>= 3.0.4'
+
+  s.files        = `git ls-files`.split("\n")
+  s.test_files   = `git ls-files -- test/*`.split("\n")
+  s.require_path = 'lib'
+end

data/test/config_test.rb

@@ -0,0 +1,34 @@
+# -*- encoding : utf-8 -*-
+
+require 'test_helper'
+
+class ConfigTest < Test::Unit::TestCase
+
+  def setup
+    @config = SesMachine::Config.instance
+    @config.send(:remove_instance_variable, :@database) if @config.instance_variable_defined?(:@database)
+  end
+
+  def teardown
+    @config.send(:remove_instance_variable, :@database) if @config.instance_variable_defined?(:@database)
+  end
+
+  def test_load_config
+    assert_nil @config.database
+    filename = File.join(File.dirname(__FILE__), 'fixtures', 'config', 'ses_machine.yml')
+    @config.load(filename)
+    assert_not_nil @config.database
+  end
+
+  def test_set_database_with_invalid_database
+    assert_raise(SesMachine::Errors::InvalidDatabase) { @config.database = 'test' }
+  end
+
+  def test_load_config_from_hash
+    filename = File.join(File.dirname(__FILE__), 'fixtures', 'config', 'ses_machine.yml')
+    settings = YAML.load_file(filename)['test']
+    assert_nil @config.database
+    @config.from_hash(settings)
+    assert_equal 'ses_machine_config_test', @config.database.name
+  end
+end

data/test/db_test.rb

@@ -0,0 +1,50 @@
+# -*- encoding : utf-8 -*-
+
+require 'test_helper'
+
+class DBTest < ActiveSupport::TestCase
+
+  def setup
+    SesMachine.config.load(File.join(File.dirname(__FILE__), 'fixtures', 'config', 'ses_machine.yml'))
+    SesMachine.database['mails'].remove
+    SesMachine.database['daily_stats'].remove
+    SesMachine.database['monthly_stats'].remove
+  end
+
+  def teardown
+    SesMachine.database['mails'].remove
+    SesMachine.database['daily_stats'].remove
+    SesMachine.database['monthly_stats'].remove
+  end
+
+#  def test_get_keywords
+#    words = ['Q', 'qw', 'QWE', 'Й', 'йц', 'ЙЦУ', '1', '12', '123']
+#    assert_equal ['qwe', 'йцу', '123'], SesMachine::DB.get_keywords(words)
+#  end
+
+  def test_update_daily_stats
+    load_mails
+    response = SesMachine::DB.update_daily_stats
+    assert_equal 3, SesMachine.database['mails'].count
+    assert_equal 2, SesMachine.database['daily_stats'].count
+    assert_equal 1, response['ok']
+  end
+
+  def test_update_monthly_stats
+    load_mails
+    SesMachine::DB.update_daily_stats
+    assert_equal 0, SesMachine.database['monthly_stats'].count
+    response = SesMachine::DB.update_monthly_stats
+    assert_equal 2, SesMachine.database['monthly_stats'].count
+    assert_equal 1, response['ok']
+  end
+
+  private
+
+  def load_mails
+    SesMachine.database['mails'].insert('bounce_type' => SesMachine::Bounce::TYPES[:mail_sent], 'date' => 2.month.ago.to_time.utc)
+    SesMachine.database['mails'].insert('bounce_type' => SesMachine::Bounce::TYPES[:unknown], 'date' => Time.now.utc)
+    SesMachine.database['mails'].insert('bounce_type' => SesMachine::Bounce::TYPES[:hard_bounce], 'date' => Time.now.utc)
+  end
+
+end

data/test/fixtures/config/ses_machine.yml

@@ -0,0 +1,11 @@
+defaults: &defaults
+  database:
+    name: ses_machine_test
+    host: localhost
+    port: 27017
+
+
+test:
+  <<: *defaults
+  database:
+    name: ses_machine_config_test

data/test/routing_test.rb

@@ -0,0 +1,17 @@
+# -*- encoding : utf-8 -*-
+
+require 'test_helper'
+
+class RoutingTest < ActionController::TestCase
+  def setup
+    ActionController::Routing::Routes.draw do |map|
+      map.ses_machine
+    end
+  end
+
+  test 'map index ses machine' do
+    assert_recognizes({:controller => 'ses_machine', :action => 'index'}, {:path => 'ses_machine', :method => :get})
+    assert_recognizes({:controller => 'ses_machine', :action => 'activity'}, {:path => 'ses_machine/activity', :method => :get})
+    assert_recognizes({:controller => 'ses_machine', :action => 'show_message', :id => '1'}, {:path => 'ses_machine/message/1', :method => :get})
+  end
+end

data/test/ses_machine_controller_test.rb

@@ -0,0 +1,49 @@
+# -*- encoding : utf-8 -*-
+
+require 'test_helper'
+
+class SesMachineControllerTest < ActionController::TestCase
+  def setup
+    SesMachine.config.load(File.join(File.dirname(__FILE__), 'fixtures', 'config', 'ses_machine.yml'))
+    SesMachine.database['mails'].remove
+    SesMachine.database['mails'].insert('_id' => BSON::ObjectId('4d95f4ebf023ca49fa000001'),
+                                        'date' => Time.now.utc,
+                                        'address' => ['test@example.com'],
+                                        'subject' => 'Test mail',
+                                        'raw_source' => 'Test')
+    ActionController::Routing::Routes.draw { |map| map.ses_machine }
+    @controller = SesMachineController.new
+  end
+
+  def teardown
+    SesMachine.database['mails'].remove
+  end
+
+  def test_index
+    get :index
+    assert_response :success
+    assert_not_nil assigns(:count_mails_sent)
+    assert_not_nil assigns(:count_mails_bounced)
+    assert_not_nil assigns(:monthly_stats)
+    assert_not_nil assigns(:date)
+  end
+
+  def test_activity
+    get :activity
+    assert_response :success
+    assert_not_nil assigns(:messages)
+    assert_not_nil assigns(:bounce_types)
+  end
+
+  def test_show_message
+    get :show_message, :id => '4d95f4ebf023ca49fa000001'
+    assert_response :success
+    assert_not_nil assigns(:mail)
+  end
+
+  def test_show_message_without_id
+    get :show_message, :id => '12345'
+    assert_redirected_to ses_machine_path
+    assert_not_nil flash[:error]
+  end
+end

data/test/ses_machine_generator_test.rb

@@ -0,0 +1,38 @@
+# -*- encoding : utf-8 -*-
+
+require 'test_helper'
+require 'rails_generator'
+require 'rails_generator/scripts/generate'
+
+generators_path = File.join(File.dirname(__FILE__), '..', 'generators')
+Rails::Generator::Base.append_sources(Rails::Generator::PathSource.new(:plugin, generators_path))
+
+class SesMachineGeneratorTest < Test::Unit::TestCase
+
+  def setup
+    FileUtils.mkdir_p(fake_rails_root)
+  end
+
+  def teardown
+    FileUtils.rm_r(fake_rails_root)
+  end
+
+  def test_generates_correct_files
+    Rails::Generator::Scripts::Generate.new.run(['ses_machine'], :destination => fake_rails_root, :quiet => true)
+    assert File.exist?(File.join(fake_rails_root, 'config', 'ses_machine.yml'))
+    assert File.exist?(File.join(fake_rails_root, 'config', 'initializers', 'ses_machine.rb'))
+    assert File.exist?(File.join(fake_rails_root, 'config', 'initializers', 'ses_machine_hooks.rb'))
+    assert File.exist?(File.join(fake_rails_root, 'public', 'stylesheets', 'ses_machine.css'))
+  end
+
+  private
+
+  def fake_rails_root
+    File.join(File.dirname(__FILE__), 'rails_root')
+  end
+
+  def file_list
+    Dir.glob(File.join(fake_rails_root, '*'))
+  end
+
+end

data/test/ses_machine_test.rb

@@ -0,0 +1,10 @@
+# -*- encoding : utf-8 -*-
+
+require 'test_helper'
+
+class SesMachineTest < ActiveSupport::TestCase
+  # Replace this with your real tests.
+  test "the truth" do
+    assert true
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,7 @@
+# -*- encoding : utf-8 -*-
+
+ENV['RAILS_ENV'] = 'test'
+ENV['RAILS_ROOT'] = File.join(File.dirname(__FILE__), 'fixtures')
+
+require 'test/unit'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'ses_machine')

data/uninstall.rb

@@ -0,0 +1,3 @@
+# -*- encoding : utf-8 -*-
+
+# Uninstall hook code here