received 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+.yardoc
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--format progress

data/Gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gem 'rspec-core'
+gem 'rspec-mocks'
+gem 'rspec-expectations'
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Roman Shterenzon
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+ReceiveD
+========
+
+ReceiveD is yet another way for receiving mail with Rails.
+Why have yet another subsystem (like IMAP), when you can deliver the mail
+directly to your data store?
+
+ReceiveD is almost [RFC2033][1] compliant LMTP server built around
+[eventmachine][2] and as such should be quite fast.
+
+The receive daemon will listen on TCP or UNIX socket, and write the mail
+to the backend storage.
+
+Currently only [MongoDB][3] is supported, but writing another backend
+(MySQL, Redis, etc.) is trivial.
+
+
+Installation
+------------
+`sudo gem install received`
+
+Modify your [Postfix][4] configuration to deliver mail via LMTP to TCP or UNIX socket.
+
+Example main.cf:
+    virtual_transport = lmtp:192.168.2.106:1111
+    virtual_mailbox_domains = example.com
+
+Create a YAML configuration file which has the following parameters:
+    {'production'=>{'host'=>hostname, 'database'=>db, 'collection'=>col}}
+
+The mongoid.yml will do, just add the name of collection, i.e.
+
+    production:
+      <<: *defaults
+      database: foo_production
+      collection: inbox
+
+The default environment is *production*, but you can specify other environment
+using RAILS_ENV environment variable.
+In this case, make sure you have the relevant key in your configuration file.
+
+
+Running
+-------
+Check -h for help, port/unix socket path and config file are required.
+
+
+Bugs and missing features
+-------------------------
+
+* When using UNIX socket the permissions/ownership are not changed. Use -u and -g when running
+  as daemon or change the permissions/ownership manually.
+* ReceiveD wasn't really tested for compliance with RFC2033
+* It doesn't implement [RFC2034][5] (ENHANCEDSTATUSCODES), because Postfix doesn't seem to care
+* It doesn't perform any validation of the provided input, e.g. LHLO, MAIL FROM, RCPT TO
+
+[1]: http://tools.ietf.org/html/rfc2033
+[2]: http://rubyeventmachine.com/
+[3]: http://www.mongodb.org/
+[4]: http://www.postfix.org/
+[5]: http://tools.ietf.org/html/rfc2034
+
+Copyright (c) 2011 Roman Shterenzon, released under the MIT license

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new
+
+Bundler::GemHelper.install_tasks

data/bin/received

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'logger'
+require 'optparse'
+$:<< File.expand_path('../../lib', __FILE__)
+require 'received'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: received [options]'
+  opts.on('-c', '--config FILE', 'Config file name (required)') {|v| options[:config] = v}
+  opts.on('-b', '--backend BACKEND', [:mongodb], 'Backend (default: mongodb)') {|v| options[:backend] = v}
+  opts.on('-d', '--daemonize', 'Become a daemon') {|v| options[:daemon] = v}
+  opts.on('-s', '--unix-socket PATH', 'Use UNIX socket') {|v| options[:unix_socket] = v}
+  opts.on('-p', '--port NUM', 'Listen to TCP port') {|v| options[:port] = v.to_i}
+  opts.on('-i', '--host NAME', 'Bind to this IP (default: 127.0.0.1)') {|v| options[:host] = v}
+  opts.on('-a', '--piddir PATH', 'Directory for pid file (default: /var/tmp)') {|v| options[:dir] = v}
+  opts.on('-l', '--log FILE', 'Log file name (default: piddir/received.log)') {|v| options[:logfile] = v}
+  opts.on('-u', '--user NAME', 'Effective user when daemon (default: nobody)') {|v| options[:user] = v}
+  opts.on('-g', '--group NAME', 'Effective group when daemon (default: nobody)') {|v| options[:group] = v}
+  opts.on('-v', '--verbose', 'Verbose logging') {options[:level] = Logger::DEBUG}
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+raise "Config file is required, please provide with -c config.yml" unless options[:config]
+
+# Default backend
+options[:backend] ||= 'mongodb'
+
+options[:logger] = Logger.new(options[:logfile] || $stderr).tap do |logger|
+  logger.level = options[:level] || Logger::INFO
+end
+
+if options.delete(:daemon)
+  require 'daemons'
+  # Monkey patch to rename log file
+  class Daemons::Application
+    def output_logfile
+      (options[:log_output] && logdir) ? File.join(logdir, @group.app_name + '.log') : nil
+    end
+  end
+  params = {:app_name => 'received', :log_output => true, :dir_mode => :normal, :dir => options[:dir] || '/var/tmp'}
+  # Drop privileges if started as superuser
+  params.merge!({:user => options[:user] || 'nobody', :group => options[:group] || 'nobody'}) if Process.uid == 0
+  Daemons.daemonize(params)
+end
+
+server = Received::Server.new(options)
+%w(TERM INT).each do |sig|
+  Signal.trap(sig) {server.stop}
+end
+server.serve!

data/lib/received/backend/base.rb

@@ -0,0 +1,13 @@
+module Received
+  module Backend
+    class Base
+      # Stores the data
+      #
+      # @abstract
+      # @param [String] data
+      def store(data)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/received/backend/mongodb.rb

@@ -0,0 +1,26 @@
+require 'mongo'
+
+module Received
+  module Backend
+    class Mongodb < Base
+
+      # Initialize MongoDB storage backend
+      #
+      # @param [Hash] params
+      # @option params [String] host
+      # @option params [String] database
+      # @option params [String] collection
+      def initialize(params)
+        @db = Mongo::Connection.new(params['host']).db(params['database'])
+        @coll = @db.collection(params['collection'])
+      end
+
+      # Store mail in MongoDB
+      #
+      # @param [Hash] mail
+      def store(mail)
+        @coll.save(mail.merge({:ts => Time.now.to_i}), :safe => true)
+      end
+    end
+  end
+end

data/lib/received/connection.rb

@@ -0,0 +1,52 @@
+require 'eventmachine'
+require 'received/lmtp'
+
+module Received
+  class Connection < EM::Connection
+
+    def initialize(server, backend)
+      @server, @backend = server, backend
+      @proto = LMTP.new(self)
+    end
+
+    def post_init
+      logger.debug "new connection"
+      @proto.start!
+    end
+
+    def receive_data(data)
+      logger.debug {"receiving data: #{data.inspect}"}
+      @proto.on_data(data)
+    end
+
+    def send_data(data)
+      logger.debug {"sending data: #{data.inspect}"}
+      super
+    end
+
+    # Client disconnected
+    def unbind
+      logger.debug "connection closed"
+      @server.remove_connection(self)
+    end
+
+    # Callback, called by protocol handler
+    #
+    # @param [Hash] mail
+    # @option mail [String] :from
+    # @option mail [Array] :rcpt
+    # @option mail [String] :body
+    def mail_received(mail)
+      begin
+        @backend.store(mail)
+        logger.info "stored mail from: #{mail[:from]}"
+      rescue Exception => e
+        logger.error "saving failed with: #{e.message}"
+      end
+    end
+
+    def logger
+      @server.logger
+    end
+  end
+end

data/lib/received/lmtp.rb

@@ -0,0 +1,127 @@
+module Received
+  # RFC2033
+  class LMTP
+
+    def initialize(conn)
+      @conn = conn
+    end
+
+    def on_data(data)
+      @buf += data
+      while line = @buf.slice!(/.+\r\n/)
+        line.chomp! unless @state == :data
+        event(line)
+      end
+    end
+
+    def start!
+      @state = :start
+      @buf = ''
+      @from = nil
+      @rcpt = []
+      event(nil)
+    end
+
+    private
+    def event(ev)
+      @conn.logger.debug {"state was: #{@state.inspect}"}
+      @state = case @state
+      when :start
+        @body = []
+        banner
+        :banner_sent
+      when :banner_sent
+        if ev.start_with?('LHLO')
+          lhlo_response
+          extensions
+          :lhlo_received
+        else
+          error
+        end
+      when :lhlo_received
+        if ev =~ /MAIL FROM:<?([^>]+)/
+          @from = $1
+          ok
+          :mail_from_received
+        else
+          error
+        end
+      when :mail_from_received
+        if ev =~ /RCPT TO:<?([^>]+)/
+          @rcpt << $1
+          ok
+          :rcpt_to_received
+        else
+          error
+        end
+      when :rcpt_to_received
+        if ev =~ /RCPT TO:<?([^>]+)/
+          @rcpt << $1
+          ok
+        elsif ev == "DATA"
+          start_mail_input
+          :data
+        else
+          error
+        end
+      when :data
+        if ev == ".\r\n"
+          @rcpt.size.times {ok}
+          mail = {:from => @from, :rcpt => @rcpt, :body => @body.join}
+          @conn.mail_received(mail)
+          :data_received
+        else
+          @body << ev
+          :data
+        end
+      when :data_received
+        if ev == "QUIT"
+          closing_connection
+          :start
+        else
+          error
+        end
+      else
+        raise "Where am I? (#{@state.inspect})"
+      end || @state
+      @conn.logger.debug {"state now: #{@state.inspect}"}
+    end
+
+    def banner
+      emit "220 localhost LMTP server ready"
+    end
+
+    def lhlo_response
+      emit "250-localhost"
+    end
+
+    def start_mail_input
+      emit "354 End data with <CR><LF>.<CR><LF>"
+    end
+
+    def closing_connection
+      emit "221 Bye"
+      @conn.close_connection_after_writing
+    end
+
+    # FIXME: RFC2033 requires ENHANCEDSTATUSCODES,
+    # but it's not used in Postfix
+    def extensions
+       emit "250-8BITMIME\r\n250 PIPELINING"
+    end
+
+    def ok
+      emit "250 OK"
+    end
+
+    def error
+      emit "500 command unrecognized"
+    end
+
+    def emit(str)
+      @conn.send_data "#{str}\r\n"
+      # return nil, so there won't be implicit state transition
+      nil
+    end
+  end
+end

data/lib/received/server.rb

@@ -0,0 +1,104 @@
+#require 'active_support/core_ext/string/inflections'
+require 'yaml'
+require 'eventmachine'
+require 'received/connection'
+
+module Received
+  class Server
+    attr_reader :logger, :options
+
+    def initialize(options)
+      @options = options
+      @logger = options[:logger] || Logger.new($stderr)
+      @connections = []
+      # For how long the server will wait for connections to finish
+      @grace_period = options[:grace_period] || 10
+      create_backend
+    end
+
+    def serve!
+      EventMachine.run { start }
+    end
+
+    def start
+      unless options[:unix_socket] or options[:port]
+        raise "No port or UNIX socket path were provided"
+      end
+      set_title
+      if host = options[:unix_socket]
+        port = nil
+      else
+        host = options[:host] || '127.0.0.1'
+        port = options[:port]
+      end
+      logger.info "Starting server on #{host}#{port ? ":" + port.to_s : ''}"
+      @signature = EventMachine.start_server(host, port, Received::Connection, self, @backend) do |conn|
+        add_connection(conn)
+      end
+    end
+
+    def stop
+      return if stopping?
+      logger.info "Stopping server"
+      EventMachine.stop_server(@signature)
+      @stopped_at = Time.now
+      unless wait_for_connections_and_stop
+        # Still some connections running, schedule a check later
+        EventMachine.add_periodic_timer(1) { wait_for_connections_and_stop }
+      end
+    end
+
+    # Checks whether the server is in stopping mode
+    def stopping?
+      !!@stopped_at
+    end
+
+    # Checks if the server is processing any connections
+    def idle?
+      @connections.empty?
+    end
+
+    def remove_connection(conn)
+      @connections.delete(conn)
+      set_title
+    end
+
+    private
+
+    # Sets the process title as seen in ps
+    def set_title
+      $0 = "received (#{@connections.size} connections)"
+    end
+
+    # Whether grace period is over
+    def grace_ended?
+      Time.now - @stopped_at > @grace_period
+    end
+
+    def wait_for_connections_and_stop
+      if idle? or grace_ended?
+        EventMachine.stop
+        true
+      else
+        logger.info "Waiting for #{@connections.size} connection(s) to finish..."
+        false
+      end
+    end
+
+    def add_connection(conn)
+      @connections << conn
+      set_title
+    end
+
+    def create_backend
+      backend = options[:backend].to_s
+      require 'received/backend/' + backend
+      #klass = ('Received::Backend::' + backend.camelize).constantize
+      klass = eval('Received::Backend::' + backend.capitalize)
+      env = ENV['RAILS_ENV'] || 'production'
+      config = YAML.load(File.read(options[:config]))[env]
+      @backend = klass.new(config)
+    end
+
+  end
+end

data/lib/received/version.rb

@@ -0,0 +1,3 @@
+module Received
+  VERSION = "0.1.0"
+end

data/lib/received.rb

@@ -0,0 +1,5 @@
+module Received
+end
+
+require 'received/backend/base'
+require 'received/server'

data/received.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "received/version"
+
+Gem::Specification.new do |s|
+  s.name        = "received"
+  s.version     = Received::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Roman Shterenzon"]
+  s.email       = ["romanbsd@yahoo.com"]
+  s.homepage    = ""
+  s.summary     = %q{Receive mail from Postfix and store it somewhere}
+  s.description = %q{Currently stores received mail in MongoDB}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_runtime_dependency 'activesupport'
+  s.add_runtime_dependency 'daemons'
+  s.add_runtime_dependency 'eventmachine'
+  s.add_runtime_dependency 'mongo', '~>1.3.0'
+  s.add_runtime_dependency 'bson_ext', '~>1.3.0'
+end

data/spec/lmtp_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'logger'
+
+describe Received::LMTP do
+  before :each do
+    @mock = mock 'conn'
+    @mock.should_receive(:send_data).with("220 localhost LMTP server ready\r\n")
+    @mock.stub!(:logger).and_return(Logger.new($stderr))
+    @mock.logger.debug "*** Starting test ***"
+    @proto = Received::LMTP.new(@mock)
+    @proto.start!
+  end
+
+  it "does full receive flow" do
+    @mock.should_receive(:send_data).with("250-localhost\r\n")
+    @mock.should_receive(:send_data).with("250-8BITMIME\r\n250 PIPELINING\r\n")
+    @mock.should_receive(:send_data).with("250 OK\r\n").exactly(3).times
+    @mock.should_receive(:send_data).with("354 End data with <CR><LF>.<CR><LF>\r\n")
+    @mock.should_receive(:send_data).with("250 OK\r\n").exactly(2).times
+    @mock.should_receive(:send_data).with("221 Bye\r\n")
+    body = "Subject: spec\r\nspec\r\n"
+    @mock.should_receive(:mail_received).with({
+      :from => 'spec1@example.com',
+      :rcpt => ['spec2@example.com', 'spec3@example.com'],
+      :body => body
+    })
+    @mock.should_receive(:close_connection_after_writing)
+
+    ["LHLO", "MAIL FROM:<spec1@example.com>", "RCPT TO:<spec2@example.com>",
+      "RCPT TO:<spec3@example.com>", "DATA", "#{body}.", "QUIT"].each do |line|
+      @mock.logger.debug "client: #{line}"
+      @proto.on_data(line + "\r\n")
+    end
+
+  end
+
+  it "parses multiline" do
+    @mock.should_receive(:send_data).with("250-localhost\r\n")
+    @mock.should_receive(:send_data).with("250-8BITMIME\r\n250 PIPELINING\r\n")
+    @mock.should_receive(:send_data).with("250 OK\r\n")
+    @proto.on_data("LHLO\r\nMAIL FROM:<spec@example.com>\r\n")
+  end
+
+  it "buffers commands up to CR/LF" do
+    @mock.should_receive(:send_data).with("250-localhost\r\n")
+    @mock.should_receive(:send_data).with("250-8BITMIME\r\n250 PIPELINING\r\n")
+    @mock.should_receive(:send_data).with("250 OK\r\n")
+    @proto.on_data("LHLO\r\nMAIL FROM")
+    @proto.on_data(":<spec@example.com>\r\n")
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'received'

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification 
+name: received
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Roman Shterenzon
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-01 00:00:00 +03:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: daemons
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: mongo
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: bson_ext
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :runtime
+  version_requirements: *id005
+description: Currently stores received mail in MongoDB
+email: 
+- romanbsd@yahoo.com
+executables: 
+- received
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/received
+- lib/received.rb
+- lib/received/backend/base.rb
+- lib/received/backend/mongodb.rb
+- lib/received/connection.rb
+- lib/received/lmtp.rb
+- lib/received/server.rb
+- lib/received/version.rb
+- received.gemspec
+- spec/lmtp_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Receive mail from Postfix and store it somewhere
+test_files: 
+- spec/lmtp_spec.rb
+- spec/spec_helper.rb