eventmachine-email_server 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f8d810ad36c8d3b8515f5f209afaadbb9038391
-  data.tar.gz: d6f036b47bd9fedf1e566ad576ed02e6f8a02b71
+  metadata.gz: a217dda0576da20c2bace6cd1320f6bc6662a052
+  data.tar.gz: cf0f814c5b56a830300d89daeff7e43a457017bd
 SHA512:
-  metadata.gz: bc9f85702c714eca052dd7b5ee52c66f84d35866df288494917e4039c29c5b533e1edd118a65f2815b4208962ef3331796808fb3da16e8ec50d45c1f311b7abf
-  data.tar.gz: dcc7d410bdd476b78db32533a507f8812d10b97c57bb2c7b177f752c95770a972ccbd111af44b416e2e06521b3be342719e0b425e6af58df9a7a3ee9c9677698
+  metadata.gz: 03eb0a117e26b162c41b5a9068687d4224ca7fe39b97378004aab43f81aeb2551b3ee49bd97f346491470fc5c6ef27b0d8133052e5c28e3fa1c8a5a1208eb0a4
+  data.tar.gz: 6c091d1cc691535298fd2f296ac493a46cd3dacbd968a95080ea34ed25aeecb5ecc6804dc2ac836bd39c17524243ce55154dd5899a1cfb57122c9231388f9573

data/.gitignore

@@ -13,3 +13,4 @@
 *.a
 mkmf.log
 /test/*.sqlite3
+/test/test.classifier

data/README.md

@@ -1,13 +1,24 @@
 # EventMachine::EmailServer
 
-This provides an EventMachine-based implementation of POP3 and SMTP services--primarily for use within the Rubot framework.  However, as I add features, this might come in handy for other people as well.
+This provides an EventMachine-based implementation of POP3 and SMTP services--primarily for use within the Rubot framework.  However, as I add features, this might come in handy for other people as well (might make a good spamtrap).
  
-There are several email and user backends so that the POP3 and SMTP servers can share: Memory, Sqlite3, and Null.
+There are several email and user backends so that the POP3 and SMTP servers can share: Memory, Sqlite3, and Null. (In the future, I plan to move Sqlite3 support into an external module and add filesystem storage as an external module as well.)
 
-The SMTP server currently only receives mail as an end-host (no relay or sending) and does no fancy routing of email (e.g., aliases and procmail).  It does, however, have graylisting, DNS PTR checks, DNSBL checks, rate limiting, and simple filters.
+The SMTP server currently only receives mail as an end-host (no relay or sending) and does no fancy routing of email (e.g., aliases and procmail).  It does, however, have graylisting, DNS PTR checks, DNSBL checks, rate limiting, SPF checking, simple filters, and a baysian classifier.
 
-Writing a full-featured mail server is a multi-year, multi-person project.  I would need some help.
+Writing a full-featured mail server is a multi-year, multi-person project.  I would need some help.  If you're interested, let me know.
 
+Potential features if people request them:
+
+* SSL, StartTLS, Peer authentication (medium)
+* Cram-MD5-based authentication (medium)
+* IMAP (hard)
+* Create a launcher in bin/ and parse configuration files (easy)
+* Domain-Keys (hard, unless I find a lib that does it)
+* Relay (this is easy to implement, but hard to get right)
+* Aliases (easy)
+* Logging (easiest)
+ 
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -43,10 +54,10 @@ Everything turned on:
     require 'dnsbl/client'
     require 'sqlite3'
 	
-    s = SQLite3::Database.new("test/test.sqlite3")
+    s = SQLite3::Database.new("email_server.sqlite3")
     userstore = Sqlite3UserStore.new(s)
     emailstore = Sqlite3EmailStore.new(s)
-    userstore << User.new(1, "chris", "chris", "chris@example.org")
+    userstore << User.new(1, "chris", "password", "chris@example.org")
     
     config = {
       'default' => RateLimit::Config.new('default', 2, 2, -2, 1, 1, 1),
@@ -54,13 +65,17 @@ Everything turned on:
     storage = RateLimit::Memory.new
     rl = RateLimit::BucketBased.new(storage, config, 'default')
     
-    dnsbl = DNSBL::Client.new
+    classifier = EventMachine::EmailServer::Classifier.new("test/test.classifier", [:spam, :ham], [:spam])
+    classifier.train(:spam, "Amazing pillz viagra cialis levitra staxyn")
+    classifier.train(:ham, "Big pigs make great bacon")
     
     SMTPServer.reverse_ptr_check(true)
     SMTPServer.graylist(Hash.new)
     SMTPServer.ratelimiter(rl)
-    SMTPServer.dnsbl(dnsbl)  
+    SMTPServer.dnsbl_check(true)  
+    SMTPServer.spf_check(true)
     SMTPServer.reject_filters << /viagra/i
+    SMTPServer.classifier(classifier)
     
     EM.run {
       pop3 = EventMachine::start_server "0.0.0.0", 2110, POP3Server, "example.org", userstore, emailstore
@@ -70,19 +85,6 @@ Everything turned on:
 
 ## Contributing
 
-If we want this to be "professional":
-*EventMachine-based SPF Checking
-*EventMachine-based DNSBL::Client
-*Abstract filtering into a class with a callback so that all sorts of filtering (e.g., baysian) could be done
-*Create a launcher in bin/ and parse configuration files
-*StartTLS
-*SSL
-*CRAM-MD5
-*Domain-Keys
-*Relay (this is easy to implement, but hard to get right)
-*Aliases
-*Logging
-
 1. Fork it ( https://github.com/[my-github-username]/eventmachine-email_server/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)

data/eventmachine-email_server.gemspec

@@ -23,6 +23,10 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "ratelimit-bucketbased", ">= 0.0.1"
   spec.add_runtime_dependency "eventmachine-dnsbl", ">= 0.0.2"
   spec.add_runtime_dependency "spf", ">= 0.0.44"
+  spec.add_runtime_dependency "classifier", ">= 1.3.4"
+  spec.add_runtime_dependency "rb-gsl", ">= 1.16.0.4"
+  spec.add_runtime_dependency "fast-stemmer", ">= 1.0.2"
+  spec.add_runtime_dependency "madeleine", ">= 0.9.0"
   spec.add_development_dependency "minitest", "~> 5.5"
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/eventmachine/email_server.rb

@@ -3,6 +3,7 @@ require 'eventmachine/email_server/base'
 require 'eventmachine/email_server/memory'
 require 'eventmachine/email_server/null'
 require 'eventmachine/email_server/sqlite3'
+require 'eventmachine/email_server/classifier'
 require 'eventmachine/email_server/pop3_server'
 require 'eventmachine/email_server/smtp_server'
 require 'eventmachine/email_server/eventmachine_dns_monkeypatch'

data/lib/eventmachine/email_server/classifier.rb

@@ -0,0 +1,38 @@
+require 'eventmachine'
+require 'classifier'
+require 'madeleine'
+
+module EventMachine
+  module EmailServer
+    class Classifier
+      def initialize(datafile, categories, blocked_categories)
+        @categories = categories || [:good, :bad]
+        @blocked_categories = blocked_categories || [:bad]
+        @categories.map! { |c| c.prepare_category_name.to_s }
+        @blocked_categories.map! { |c| c.prepare_category_name.to_s }
+        @madeleine = SnapshotMadeleine.new(datafile) {
+            ::Classifier::Bayes.new(*categories)
+        }
+        @classifier = @madeleine.system
+      end
+      
+      def train(category, email)
+        @classifier.train(category, email)
+        @madeleine.take_snapshot
+      end
+      
+      def classify(email)
+        @classifier.classify(email)
+      end
+      
+      def block?(email)
+        c = classify(email)
+        if @blocked_categories.index(c).nil?
+          return false
+        end
+        true
+      end
+    end
+  end
+end
+    

data/lib/eventmachine/email_server/smtp_server.rb

@@ -11,6 +11,7 @@ module EventMachine
       @@reverse_ptr_check = false
       @@spf_check = false
       @@reject_filters = Array.new
+      @@classifier = nil
       
       def self.reset
         @@graylist = nil
@@ -19,6 +20,7 @@ module EventMachine
         @@reverse_ptr_check = false
         @@spf_check = false
         @@reject_filters = Array.new
+        @classifier = nil
       end
       
       def self.reverse_ptr_check(ptr=nil)
@@ -61,7 +63,14 @@ module EventMachine
           @@spf_check = spf
         end
         @@spf_check
-      end 
+      end
+      
+      def self.classifier(classifier=nil)
+        if not classifier.nil?
+          @@classifier = classifier
+        end
+        @@classifier
+      end
 
       def initialize(hostname, userstore, emailstore)
         @hostname = hostname
@@ -74,7 +83,9 @@ module EventMachine
         @dnsbl_ok = true
         @rate_ok = true
         @gray_ok = true
+        @spf_ok = true
         @reject_ok = true
+        @classifier_ok = true
         @pending_checks = Array.new
       end
       
@@ -145,6 +156,14 @@ module EventMachine
         end
       end
       
+      def check_classifier
+        if @@classifier
+          if @@classifier.block?(@email_body)
+            @classifier_ok = false
+          end
+        end
+      end
+      
       def check_spf(helo, client_ip, identity)
         if @@spf_check
           @spf_ok = false
@@ -196,7 +215,16 @@ module EventMachine
       end
       
       def send_answer
-        if @ptr_ok and @rate_ok and @gray_ok and @reject_ok and @dnsbl_ok and @spf_ok
+        if @debug
+          puts "ptr_ok        = #{@ptr_ok}"
+          puts "rate_ok       = #{@rate_ok}"
+          puts "gray_ok       = #{@gray_ok}"
+          puts "reject_ok     = #{@reject_ok}"
+          puts "dnsbl_ok      = #{@dnsbl_ok}"
+          puts "spf_ok        = #{@spf_ok}"
+          puts "classifier_ok = #{@classifier_ok}"
+        end
+        if @ptr_ok and @rate_ok and @gray_ok and @reject_ok and @dnsbl_ok and @spf_ok and @classifier_ok
           ans = "250 OK"
         else
           ans = "451 Requested action aborted: local error in processing"
@@ -209,6 +237,7 @@ module EventMachine
     		if (@data_mode) && (line.chomp =~ /^\.$/)
     			@data_mode = false
           check_reject
+          check_classifier
           if @pending_checks.length == 0
             send_answer
           end

data/lib/eventmachine/email_server/version.rb

@@ -1,5 +1,5 @@
 module EventMachine
   module EmailServer
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end

data/test/test_email_server.rb

@@ -27,21 +27,19 @@ class TestEmailServer < Minitest::Test
     @test_vector = Proc.new { |test_name|
       (test_name.to_s =~ /test/)
     }
-    if File.exist?("test/test.sqlite3")
-      File.unlink("test/test.sqlite3")
-    end
-    if File.exist?("email_server.sqlite3")
-      File.unlink("email_server.sqlite3")
-    end
+    remove_scraps
   end
   
-  def teardown
-    if File.exist?("test/test.sqlite3")
-      File.unlink("test/test.sqlite3")
-    end
-    if File.exist?("email_server.sqlite3")
-      File.unlink("email_server.sqlite3")
+  def remove_scraps
+    ["test.sqlite3", "email_server.sqlite3"].each do |f|
+      if File.exist?("test/#{f}")
+        File.unlink("test/#{f}")
+      end
     end
+  end    
+  
+  def teardown
+    remove_scraps
   end
   
   def setup_user(userstore)
@@ -69,6 +67,38 @@ Looks like we had fun!
     end
   end
   
+  def send_spam(expected_status="451")
+    from = "friend@example.org"
+    to = "chris@example.org"
+    msg = "From: friend@example.org
+To: chris@example.org
+Subject: What to do when you're not doing.
+
+Could I interest you in some cialis?
+"
+    Thread.new do
+      smtp = Net::SMTP.start('localhost', 2025)
+      ret = smtp.send_message msg, from, to
+      assert_equal(expected_status, ret.status)
+    end
+  end    
+  
+  def send_ham(expected_status="250")
+    from = "friend@example.org"
+    to = "chris@example.org"
+    msg = "From: friend@example.org
+To: chris@example.org
+Subject: Good show
+
+Have you seen the latest Peppa Pig?
+"
+    Thread.new do
+      smtp = Net::SMTP.start('localhost', 2025)
+      ret = smtp.send_message msg, from, to
+      assert_equal(expected_status, ret.status)
+    end
+  end    
+  
   def pop_some_email
     Thread.new do 
       pop = Net::POP3.APOP(true).new('localhost',2110)
@@ -277,6 +307,28 @@ Looks like we had fun!
     }
     
   end
+  
+  def test_classifier
+    return unless @test_vector.call(__method__)
+    userstore = MemoryUserStore.new
+    emailstore = MemoryEmailStore.new
+    setup_user(userstore)
+    SMTPServer.reset
+    classifier = EventMachine::EmailServer::Classifier.new("test/test.classifier", [:spam, :ham], [:spam])
+    classifier.train(:spam, "Amazing pillz viagra cialis levitra staxyn")
+    classifier.train(:ham, "Big pigs make great bacon")
+    SMTPServer.classifier(classifier)
+    EM.run {
+      smtp = EventMachine::start_server "0.0.0.0", 2025, SMTPServer, "example.org", userstore, emailstore
+      timer = EventMachine::Timer.new(0.1) do
+        send_spam("451")
+        send_ham("250")
+      end
+      timer2 = EventMachine::Timer.new(1) do
+        EM.stop
+      end
+    }
+  end
     
   def test_example
     return unless @test_vector.call(__method__)
@@ -297,13 +349,18 @@ Looks like we had fun!
     storage = RateLimit::Memory.new
     rl = RateLimit::BucketBased.new(storage, config, 'default')
   
+    classifier = EventMachine::EmailServer::Classifier.new("test/test.classifier", [:spam, :ham], [:spam])
+    classifier.train(:spam, "Amazing pillz viagra cialis levitra staxyn")
+    classifier.train(:ham, "Big pigs make great bacon")
   
     SMTPServer.reset
     SMTPServer.reverse_ptr_check(true)
     SMTPServer.graylist(Hash.new)
     SMTPServer.ratelimiter(rl)
     SMTPServer.dnsbl_check(true)  
+    SMTPServer.spf_check(true)
     SMTPServer.reject_filters << /viagra/i
+    SMTPServer.classifier(classifier)
   
     EM.run {
       pop3 = EventMachine::start_server "0.0.0.0", 2110, POP3Server, "example.org", userstore, emailstore
@@ -313,7 +370,5 @@ Looks like we had fun!
       end
     }
   end
-  
-  
    
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventmachine-email_server
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - chrislee35
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-07 00:00:00.000000000 Z
+date: 2015-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -80,6 +80,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.44
+- !ruby/object:Gem::Dependency
+  name: classifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+- !ruby/object:Gem::Dependency
+  name: rb-gsl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.16.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.16.0.4
+- !ruby/object:Gem::Dependency
+  name: fast-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: madeleine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -135,9 +191,11 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- email_server.sqlite3
 - eventmachine-email_server.gemspec
 - lib/eventmachine/email_server.rb
 - lib/eventmachine/email_server/base.rb
+- lib/eventmachine/email_server/classifier.rb
 - lib/eventmachine/email_server/eventmachine_dns_monkeypatch.rb
 - lib/eventmachine/email_server/memory.rb
 - lib/eventmachine/email_server/null.rb