mlist 0.1.9

data/rails/init.rb

@@ -0,0 +1,22 @@
+# Provides the mechinism to support applications that want to observe MList
+# models.
+#
+# ActiveRecord observers are reloaded at each request in development mode.
+# They will be registered with the MList models each time. Since the MList
+# models are required once at initialization, there will always only be one
+# instance of the model class, and therefore, many instances of the observer
+# class registered with it; all but the most recent are invalid, since they
+# were undefined when the dispatcher reloaded the application.
+#
+# Should we ever have observers in MList, this will likely need more careful
+# attention.
+#
+unless Rails.configuration.cache_classes
+  class << ActiveRecord::Base
+    def instantiate_observers_with_mlist_observers
+      subclasses.each(&:delete_observers)
+      instantiate_observers_without_mlist_observers
+    end
+    alias_method_chain :instantiate_observers, :mlist_observers
+  end
+end

data/spec/fixtures/schema.rb

@@ -0,0 +1,94 @@
+ActiveRecord::Schema.define(:version => 20081126181722) do
+  
+  # All MList required tables are prefixed with 'mlist_' to ease integration
+  # into other systems' databases.
+  
+  
+  # The table in which email content is stored.
+  # 
+  create_table :mlist_emails, :force => true do |t|
+    t.column :source, :text
+    t.column :created_at, :datetime
+  end
+  
+  # The table in which MList will store MList::Messages.
+  #
+  # The identifier is the 'message-id' header value of the finally delivered
+  # email.
+  #
+  # An MList::Message will store a reference to your application's subscriber
+  # instance if it is an ActiveRecord subclass. That subclass must respond to
+  # :email_address. If your subscriber is just a string, it is assumed to be
+  # an email address. Either way, that email address will be stored with the
+  # MList::Message, providing a way for you to associate messages by
+  # subscriber_address. This is less ideal, as you may allow subscribers to
+  # change their email addresses over time.
+  #
+  create_table :mlist_messages, :force => true do |t|
+    t.column :mail_list_id, :integer
+    t.column :thread_id, :integer
+    t.column :email_id, :integer
+    t.column :identifier, :string
+    t.column :parent_identifier, :string
+    t.column :parent_id, :integer
+    t.column :mailer, :string
+    t.column :subject, :string
+    t.column :subscriber_address, :string
+    t.column :subscriber_type, :string
+    t.column :subscriber_id, :integer
+    t.column :created_at, :datetime
+  end
+  add_index :mlist_messages, :mail_list_id
+  add_index :mlist_messages, :thread_id
+  add_index :mlist_messages, :identifier
+  add_index :mlist_messages, :parent_identifier
+  add_index :mlist_messages, :parent_id
+  add_index :mlist_messages, :subject
+  add_index :mlist_messages, :subscriber_address
+  add_index :mlist_messages, [:subscriber_type, :subscriber_id]
+  
+  # Every MList::Message is associated with an MList::Thread.
+  #
+  create_table :mlist_threads, :force => true do |t|
+    t.column :mail_list_id, :integer
+    t.column :messages_count, :integer
+    t.timestamps
+  end
+  add_index :mlist_threads, :mail_list_id
+  
+  # The table in which MList will store MList::MailLists.
+  #
+  # The manager_list_identifier column stores the MList::List#list_id value.
+  # This is a connection to the application's implementation of MList::List.
+  # These identifiers must be unique and never change for an MList::List.
+  #
+  # An MList::MailList will store a reference to your application's
+  # MList::List instance if it is an ActiveRecord subclass.
+  #
+  create_table :mlist_mail_lists, :force => true do |t|
+    t.column :manager_list_identifier, :string
+    t.column :manager_list_type, :string
+    t.column :manager_list_id, :integer
+    t.column :messages_count, :integer
+    t.column :threads_count, :integer
+    t.timestamps
+  end
+  add_index :mlist_mail_lists, :manager_list_identifier
+  add_index :mlist_mail_lists, [:manager_list_identifier, :manager_list_type, :manager_list_id],
+    :name => :index_mlist_mail_lists_on_manager_association
+  
+  
+  # Database list manager tables, used for testing purposes.
+  #
+  create_table :lists, :force => true do |t|
+    t.column :address, :string
+    t.column :label, :string
+    t.column :created_at, :datetime
+  end
+  
+  create_table :subscribers, :force => true do |t|
+    t.column :list_id, :integer
+    t.column :email_address, :string
+    t.column :created_at, :datetime
+  end
+end

data/spec/integration/date_formats_spec.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe MList, 'date formats' do
+  specify 'mlist_reply_timestamp should handle single digit days and months' do
+    Time.local(2009, 2, 3, 7).to_s(:mlist_reply_timestamp).should == 'Tue, Feb 3, 2009 at 7:00 AM'
+  end
+  
+  specify 'mlist_reply_timestamp should handle double digit days and months' do
+    Time.local(2009, 2, 13, 11).to_s(:mlist_reply_timestamp).should == 'Fri, Feb 13, 2009 at 11:00 AM'
+    Time.local(2009, 2, 13, 14).to_s(:mlist_reply_timestamp).should == 'Fri, Feb 13, 2009 at 2:00 PM'
+  end
+end

data/spec/integration/mlist_spec.rb

@@ -0,0 +1,232 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require 'mlist/manager/database'
+
+describe MList do
+  def forward_email(tmail)
+    simple_matcher('forward email') do |email_server|
+      lambda do
+        lambda do
+          email_server.receive(tmail)
+        end.should_not change(email_server.deliveries, :size)
+      end.should_not store_message
+    end
+  end
+  
+  def store_message
+    simple_matcher('store message') do |block|
+      thread_count, message_count = MList::Thread.count, MList::Message.count
+      block.call
+      MList::Thread.count == thread_count + 1 && MList::Message.count == message_count + 1
+    end
+  end
+  
+  def start_new_thread(tmail)
+    simple_matcher('start new thread') do |email_server|
+      delivery_count = email_server.deliveries.size
+      thread_count = MList::Thread.count
+      message_count = MList::Message.count
+      email_server.receive(tmail)
+      email_server.deliveries.size > delivery_count &&
+        MList::Thread.count == (thread_count + 1) &&
+        MList::Message.count == (message_count + 1)
+    end
+  end
+  
+  dataset do
+    @list_manager = MList::Manager::Database.new
+    @list_one = @list_manager.create_list('list_one@example.com')
+    @list_one.subscribe('adam@nomail.net')
+    @list_one.subscribe('tom@example.com')
+    @list_one.subscribe('dick@example.com')
+    
+    @list_two = @list_manager.create_list('list_two@example.com')
+    @list_two.subscribe('adam@nomail.net')
+    @list_two.subscribe('jane@example.com')
+    
+    @list_three = @list_manager.create_list('empty@example.com')
+    @list_three.subscribe('adam@nomail.net')
+  end
+  
+  before do
+    @email_server = MList::EmailServer::Fake.new
+    @server = MList::Server.new(
+      :list_manager => @list_manager,
+      :email_server => @email_server
+    )
+    
+    # TODO Move this stuff to Dataset
+    ActiveRecord::Base.connection.increment_open_transactions
+    ActiveRecord::Base.connection.begin_db_transaction
+  end
+  
+  after do
+    if ActiveRecord::Base.connection.open_transactions != 0
+      ActiveRecord::Base.connection.rollback_db_transaction
+      ActiveRecord::Base.connection.decrement_open_transactions
+    end
+    ActiveRecord::Base.clear_active_connections!
+  end
+  
+  it 'should have threads and mail_lists updated_at set to last message receive time' do
+    now = Time.now
+    stub(Time).now {now}
+    @email_server.receive(tmail_fixture('single_list'))
+    MList::MailList.last.updated_at.to_s.should == now.to_s
+    MList::Thread.last.updated_at.to_s.should == now.to_s
+    
+    later = 5.days.from_now
+    stub(Time).now {later}
+    @email_server.receive(tmail_fixture('single_list_reply'))
+    MList::MailList.last.updated_at.to_s.should == later.to_s
+    MList::Thread.last.updated_at.to_s.should == later.to_s
+  end
+  
+  it 'should associate manager lists to mlist mail lists when they are ActiveRecord instances' do
+    @email_server.receive(tmail_fixture('single_list'))
+    mail_list = MList::MailList.last
+    mail_list.manager_list.should == @list_one
+    mail_list.manager_list_identifier.should == @list_one.list_id
+  end
+  
+  it 'should not forward mail that has been on this server before' do
+    @email_server.should_not forward_email(tmail_fixture('x-beenthere'))
+  end
+  
+  it 'should store message/thread even when there are no recipient subscribers' do
+    tmail = tmail_fixture('single_list')
+    tmail.to = @list_three.address
+    @email_server.should start_new_thread(tmail)
+  end
+  
+  it 'should not forward mail from non-subscriber and notify manager list' do
+    tmail = tmail_fixture('single_list')
+    tmail.from = 'unknown@example.com'
+    stub(@list_manager).lists(is_a(MList::Email)) { [@list_one] }
+    mock(@list_one).non_subscriber_post(is_a(MList::Email))
+    @email_server.should_not forward_email(tmail)
+  end
+  
+  it 'should not forward mail from non-subscriber when inactive and notify as non-subscriber' do
+    tmail = tmail_fixture('single_list')
+    tmail.from = 'unknown@example.com'
+    do_not_call(@list_one).active?
+    stub(@list_manager).lists(is_a(MList::Email)) { [@list_one] }
+    mock(@list_one).non_subscriber_post(is_a(MList::Email))
+    @email_server.should_not forward_email(tmail)
+  end
+  
+  it 'should not forward mail from blocked subscriber, notify the subscriber using list manager notifier' do
+    subscriber = MList::Manager::Database::Subscriber.find_by_email_address('adam@nomail.net')
+    tmail = tmail_fixture('single_list')
+    mock(@list_one).active? {true}
+    mock(@list_one).blocked?(subscriber) { true }
+    stub(@list_manager).lists(is_a(MList::Email)) { [@list_one] }
+    lambda do
+      @email_server.receive(tmail)
+      response_tmail = @email_server.deliveries.last
+      response_tmail.header_string('from').should == 'mlist-list_one@example.com'
+      response_tmail.header_string('to').should == 'adam@nomail.net'
+      response_tmail.header_string('x-mlist-loop').should == 'notice'
+      response_tmail.header_string('x-mlist-notice').should == 'subscriber_blocked'
+    end.should_not store_message
+  end
+  
+  it 'should not forward mail to inactive list and notify manager list' do
+    tmail = tmail_fixture('single_list')
+    mock(@list_one).active? { false }
+    stub(@list_manager).lists(is_a(MList::Email)) { [@list_one] }
+    mock(@list_one).inactive_post(is_a(MList::Email))
+    @email_server.should_not forward_email(tmail)
+  end
+  
+  it 'should report unrecognized email to list manager' do
+    tmail = tmail_fixture('single_list')
+    stub(@list_manager).lists(is_a(MList::Email)) { [] }
+    mock(@list_manager).no_lists_found(is_a(MList::Email))
+    @email_server.should_not forward_email(tmail)
+  end
+  
+  it 'should report bounces to the list manager' do
+    stub(@list_manager).lists(is_a(MList::Email)) { [@list_one] }
+    mock(@list_one).bounce(is_a(MList::Email))
+    @email_server.should_not forward_email(tmail_fixture('bounces/1'))
+  end
+  
+  describe 'single list' do
+    before do
+      @tmail_post = tmail_fixture('single_list')
+      @email_server.receive(@tmail_post)
+    end
+    
+    it 'should forward emails that are sent to a mailing list' do
+      @email_server.deliveries.size.should == 1
+      email = @email_server.deliveries.first
+      email.should have_address(:to, 'list_one@example.com')
+      # bcc fields are not in the headers of delivered emails
+      email.should have_address(:'reply-to', 'list_one@example.com')
+    end
+    
+    it 'should start a new thread for a new email' do
+      thread = MList::Thread.last
+      thread.messages.first.email.tmail.should equal_tmail(@tmail_post)
+    end
+    
+    it 'should add to an existing thread when reply email' do
+      message = MList::Message.last
+      reply_tmail = tmail_fixture('single_list_reply', 'in-reply-to' => "<#{message.identifier}>")
+      @email_server.receive(reply_tmail)
+      thread = MList::Thread.last
+      thread.messages.size.should be(2)
+    end
+    
+    it 'should associate parent message when reply email' do
+      message = MList::Message.last
+      @email_server.receive(tmail_fixture('single_list_reply', 'in-reply-to' => "<#{message.identifier}>"))
+      reply = MList::Message.last
+      reply.parent_identifier.should == message.identifier
+      reply.parent.should == message
+    end
+    
+    it 'should not associate a parent when not a reply' do
+      tmail = tmail_fixture('single_list')
+      tmail['message-id'] = 'asdfasdfj'
+      tmail['subject'] = 'other thing'
+      @email_server.should start_new_thread(tmail)
+    end
+    
+    it 'should store subscriber address with messages' do
+      MList::Message.last.subscriber_address.should == 'adam@nomail.net'
+    end
+    
+    it 'should associate subscriber to messages when they are ActiveRecord instances' do
+      MList::Message.last.subscriber.should == MList::Manager::Database::Subscriber.find_by_email_address('adam@nomail.net')
+    end
+  end
+  
+  describe 'multiple lists' do
+    before do
+      @tmail_post = tmail_fixture('multiple_lists')
+      @email_server.receive(@tmail_post)
+    end
+    
+    it 'should forward emails that are sent to a mailing list' do
+      @email_server.deliveries.size.should == 2
+      
+      email = @email_server.deliveries.first
+      email.should have_address(:to, 'list_one@example.com')
+      # bcc fields are not in the headers of delivered emails
+      email.should have_address(:'reply-to', 'list_one@example.com')
+      
+      email = @email_server.deliveries.last
+      email.should have_address(:to, 'list_two@example.com')
+      # bcc fields are not in the headers of delivered emails
+      email.should have_address(:'reply-to', 'list_two@example.com')
+    end
+    
+    it 'should start a new thread for each list, both referencing the same email' do
+      threads = MList::Thread.find(:all)
+      threads[0].messages.first.email.should == threads[1].messages.first.email
+    end
+  end
+end

data/spec/integration/pop_email_server_spec.rb

@@ -0,0 +1,22 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'mlist/email_server/pop'
+
+describe MList::EmailServer::Pop, 'execute' do
+  before do
+    @mails = []
+    @pop = Object.new
+    stub(@pop).mails { @mails }
+    stub(@pop).start do |username, password, block|
+      block.call @pop
+    end
+    stub(Net::POP3).new { @pop }
+    @pop_server = MList::EmailServer::Pop.new({})
+  end
+  
+  it 'should delete email after successfully receiving' do
+    message = OpenStruct.new(:pop => email_fixture('single_list'))
+    mock(message).delete
+    @mails << message
+    @pop_server.execute
+  end
+end

data/spec/integration/proof_spec.rb

@@ -0,0 +1,74 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require 'mlist/manager/database'
+
+describe MList do
+  include MList::Util::EmailHelpers
+  
+  dataset do
+    subscriber_addresses = %w(adam@nomail.net adam@example.net)
+    
+    @list_manager = MList::Manager::Database.new
+    Dir[email_fixtures_path('integration/list*')].each do |list_path|
+      list = @list_manager.create_list("#{File.basename(list_path)}@example.com")
+      subscriber_addresses.each {|a| list.subscribe(a)}
+    end
+  end
+  
+  before do
+    @email_server = MList::EmailServer::Fake.new
+    @server = MList::Server.new(
+      :list_manager => @list_manager,
+      :email_server => @email_server
+    )
+    
+    # TODO Move this stuff to Dataset
+    ActiveRecord::Base.connection.increment_open_transactions
+    ActiveRecord::Base.connection.begin_db_transaction
+  end
+  
+  after do
+    if ActiveRecord::Base.connection.open_transactions != 0
+      ActiveRecord::Base.connection.rollback_db_transaction
+      ActiveRecord::Base.connection.decrement_open_transactions
+    end
+    ActiveRecord::Base.clear_active_connections!
+  end
+  
+  it 'should place messages in threads and threads in lists' do
+    Dir[email_fixtures_path('integration/list*')].each do |list_path|
+      list = MList::Manager::Database::List.find_by_address(File.basename(list_path) + '@example.com')
+      Dir[File.join(list_path, 'thread*')].each do |thread_path|
+        email_paths = Dir[File.join(thread_path, '*.eml')]
+        top_post_path = email_paths.shift
+        top_post_tmail = TMail::Mail.load(top_post_path)
+        stub(@email_server).generate_message_id { remove_brackets(top_post_tmail.message_id) }
+        @email_server.should start_thread(top_post_tmail)
+        email_paths.each do |email_path|
+          tmail = TMail::Mail.load(email_path)
+          stub(@email_server).generate_message_id { remove_brackets(tmail.message_id) }
+          expected = email_path =~ %r{\d+\.eml\Z} ? :should : :should_not
+          @email_server.send expected, accept_message(tmail)
+        end
+      end
+    end
+  end
+  
+  def accept_message(tmail)
+    simple_matcher("to receive message from #{tmail.header_string('from')}") do |email_server|
+      message_count_start = MList::Message.count
+      email_server.receive(tmail)
+      MList::Message.count == message_count_start + 1 && MList::Message.last.identifier == remove_brackets(@email_server.deliveries.last.header_string('message-id'))
+    end
+  end
+  
+  def start_thread(tmail)
+    simple_matcher("to begin thread from #{tmail.header_string('from')}") do |email_server|
+      message_count_start = MList::Message.count
+      thread_count_start = MList::Thread.count
+      email_server.receive(tmail)
+      MList::Message.count == message_count_start + 1
+      MList::Thread.count == thread_count_start + 1
+    end
+  end
+end

data/spec/matchers/equal_tmail.rb

@@ -0,0 +1,53 @@
+module Spec
+  module Matchers
+    
+    class EqualTmail
+      def initialize(expected)
+        @expected = expected
+      end
+      
+      def matches?(tmail)
+        @given = tmail
+        headers_match?
+      end
+      
+      def failure_message
+        @failure_message
+      end
+      
+      private
+        def comparable_value(value)
+          case value
+          when Array
+            value.collect {|e| e.to_s.strip}.join(" ").strip
+          when String
+            value.strip
+          end
+        end
+        
+        def headers_match?
+          missing = @expected.header.collect { |name, value| @given[name].nil? ? name : nil }.compact
+          extra = @given.header.collect { |name, value| @expected[name].nil? ? name : nil }.compact
+          if extra.empty? && missing.empty?
+            unequal = []
+            @expected.header.each do |name, value|
+              expected_value, given_value = comparable_value(value), comparable_value(@given[name])
+              unless expected_value == given_value
+                unequal << "expected header #{name.inspect} to be #{expected_value.inspect} but was #{given_value.inspect}"
+              end
+            end
+            @failure_message = unequal.join("\n") unless unequal.empty?
+          else
+            @failure_message = "expected tmail instances to be equal but headers were not"
+            @failure_message << "\nmissing in given: #{missing.inspect}" unless missing.empty?
+            @failure_message << "\nextra in given: #{extra.inspect}" unless extra.empty?
+          end
+          @failure_message.nil?
+        end
+    end
+    
+    def equal_tmail(tmail)
+      EqualTmail.new(tmail)
+    end
+  end
+end

data/spec/matchers/have_address.rb

@@ -0,0 +1,48 @@
+module Spec
+  module Matchers
+    
+    class HaveAddress
+      def initialize(field, expected)
+        @field, @expected = field, expected
+      end
+      
+      def matches?(email)
+        @actual = addresses(email)
+        missing = expected_addresses.reject {|e| @actual.include?(e)}
+        extra = @actual.reject {|e| expected_addresses.include?(e)}
+        extra.empty? && missing.empty?
+      end
+      
+      def failure_message
+        "expected #{@field} address to contain #{expected_addresses.inspect} but was #{@actual.inspect}"
+      end
+      
+      def negative_failure_message
+        "expected #{@field} address not to contain #{expected_addresses.inspect} but it did"
+      end
+      
+      private
+        def addresses(email)
+          email[@field.to_s].addrs.collect(&:address) rescue []
+        end
+        
+        def expected_addresses
+          case @expected
+          when Array
+            @expected.collect { |a| extract_address(a) }
+          when String
+            [extract_address(@expected)]
+          end
+        end
+        
+        def extract_address(string)
+          address = string.sub(/.*?<(.*?)>/, '\1')
+          address if address =~ /\A([^@\s]+)@(localhost|(?:[-a-z0-9]+\.)+[a-z]{2,})\Z/i
+        end
+    end
+    
+    def have_address(field, expected)
+      HaveAddress.new(field, expected)
+    end
+  end
+end

data/spec/matchers/have_header.rb

@@ -0,0 +1,104 @@
+module Spec
+  module Matchers
+    
+    class HaveHeader
+      def initialize(name, expected)
+        @name, @expected = name, expected
+      end
+      
+      def matches?(email)
+        @given = email
+        
+        if @expected.blank?
+          if !@given[@name.downcase]
+            @failure_message = "expected header #{@name.inspect} to be present but it was not"
+          end
+        else
+          missing = expected_values.reject {|e| header_values.include?(e)}
+          extra = header_values.reject {|e| expected_values.include?(e)}
+          unless extra.empty? && missing.empty?
+            @failure_message = "expected header #{@name.inspect} to be equal but was not"
+            @failure_message << "\nmissing in given: #{missing.inspect}" unless missing.empty?
+            @failure_message << "\nextra in given: #{extra.inspect}" unless extra.empty?
+          end
+        end
+        
+        @failure_message.nil?
+      end
+      
+      def failure_message
+        @failure_message
+      end
+      
+      def negative_failure_message
+        if @expected.blank? && @given[@name.downcase]
+          "expected header #{@name.inspect} to not be present but was"
+        else
+          "expected header #{@name.inspect} to not be equal but it was"
+        end
+      end
+      
+      private
+        def comparable_value(value)
+          case value
+          when Array
+            value.collect {|e| e.to_s.strip}.join(" ").strip
+          when String
+            value.strip
+          end
+        end
+        
+        def expected_values
+          case @expected
+          when Array
+            @expected.collect { |e| comparable_value(e) }
+          when String
+            [comparable_value(@expected)]
+          end
+        end
+        
+        def header_values
+          values = []
+          @given.each_header do |k,v|
+            values << v.to_s.strip if k.downcase == @name.downcase
+          end
+          values
+        end
+    end
+    
+    class HaveHeaders
+      def initialize(expected)
+        @matchers = case expected
+        when Array
+          expected.collect {|e| HaveHeader.new(e, nil)}
+        when Hash
+          expected.collect {|k,v| HaveHeader.new(k,v)}
+        end
+      end
+      
+      def matches?(given)
+        @matched, @failed = [], []
+        @matchers.each do |matcher|
+          (matcher.matches?(given) ? @matched : @failed) << matcher
+        end
+        @failed.empty?
+      end
+      
+      def failure_message
+        @failed.collect(&:failure_message).join("\n")
+      end
+      
+      def negative_failure_message
+        raise 'have_headers cannot be used in a negative'
+      end
+    end
+    
+    def have_header(name, expected = nil)
+      HaveHeader.new(name, expected)
+    end
+    
+    def have_headers(expected)
+      HaveHeaders.new(expected)
+    end
+  end
+end