kns_email_endpoint 0.1.0

data/lib/kns_email_endpoint/message_state.rb

@@ -0,0 +1,89 @@
+require 'digest/sha1'
+module KNSEmailEndpoint
+  class MessageState
+    
+    class << self 
+      attr_accessor :storage
+
+      def set_storage(storage, opts)
+        @storage = Storage.get_storage(storage, opts)
+        return @storage
+      end
+
+      def gen_unique_id(conn,msg)
+        raise "Message must have a valid message_id" unless msg.message_id.to_s != ""
+        Digest::SHA1.hexdigest "#{conn}::#{msg.message_id}::K-KEY"
+      end
+    end
+
+
+    attr_reader :message, :message_id, :retry_count, :unique_id, :state
+
+    def initialize(conn_name, message)
+      # setup the state
+      @conn_name = conn_name
+      @message = message
+      @message_id = get_message_id
+      @unique_id = get_unique_id 
+      @storage = self.class.storage
+      raise "Unknown Storage" unless @storage
+
+      # get from storage
+      @storage.find_or_create(:unique_id => @unique_id, :message_id => @message_id)
+      @state = @storage.state
+      @retry_count = @storage.retry_count
+
+      # Do not auto delete email. 
+      @message.mark_for_delete = false
+
+    end
+
+    def state=(s)
+      @state = s
+      @storage.state = @state if @storage.unique_id
+      @message.mark_for_delete = @state == :deleted
+      return @state
+    end
+
+    def retry
+      @retry_count += 1
+      @storage.retry_count = @retry_count
+      return @retry_count
+    end
+
+    def reset
+      @retry_count = 0
+      @storage.retry_count = @retry_count
+      return @retry_count
+    end
+
+    def reset_state
+      @storage.delete
+      @storage.create(:unique_id => @unique_id, :message_id => @message_id)
+      @retry_count = @storage.retry_count
+      @state = @storage.state
+    end
+
+    def delete
+      @storage.delete
+      self.state = :deleted
+    end
+
+
+    private 
+
+    def get_message_id
+      return @message_id if @message_id
+      if @message.has_message_id?
+        @message_id = @message.message_id
+      else
+        raise "The mail message does not have a valid message_id."
+      end
+      return @message_id
+    end
+
+    def get_unique_id
+      return @unique_id ||= self.class.gen_unique_id(@conn_name, @message)
+    end
+  end
+end

data/lib/kns_email_endpoint/process_email.rb

@@ -0,0 +1,123 @@
+require 'work_queue'
+module KNSEmailEndpoint
+
+  class ProcessEmail
+    # if in repeat mode, these states will be processed
+    $REPEATABLE_STATES = [:forwarded, :replied, :unprocessed, :error]
+    # if in single mode, these states will be processed
+    $SINGLE_STATES = [:unprocessed, :error] 
+
+    class << self
+
+      def go(conn)
+        raise "Need config" unless Configuration[conn.name]
+        log = conn.conn_log
+        3.times { log.debug "" }
+        log.info "Processing Email from connection: #{conn.name}"
+        endpoint_opts = {
+          :ruleset => conn.appid,
+          :environment => conn.environment,
+          :use_session => true,
+          :logging => log.debug? 
+        }
+
+        begin
+          queue = WorkQueue.new(Configuration.work_threads)
+          email_processed_count = 0
+          email_errors_count = 0
+          conn.retriever.find({
+            :count => :all,
+            :delete_after_find => true,
+            :what => :first,
+            :order => :asc
+          }) do |msg|
+            # worker enqueue
+            queue.enqueue_b {
+              begin
+                msg_state = MessageState.new(conn.name, msg)
+                log.debug "Processing Message #{msg_state.unique_id}"
+                log.debug "STATE: #{msg_state.state}"
+                if (conn.process_mode == :single && $SINGLE_STATES.include?(msg_state.state)) ||
+                   (conn.process_mode == :repeat && $REPEATABLE_STATES.include?(msg_state.state))
+
+                  ee = EmailEndpoint.new(conn.name, endpoint_opts, conn.sender)
+                  event_args = {
+                    :msg => msg,
+                    :unique_id => msg_state.unique_id
+                  }.merge! conn.event_args
+
+                  log.debug "Raising Event\n #{event_args.inspect}"
+                  result = ee.received(event_args)
+                  if log.debug?
+                    log.debug "--- Endpoint Log ---"
+                    log.debug ee.log.join("\n")
+                    log.debug "--------------------"
+                  end
+                  if ee.message_state.state == :processing
+                    # there was no directive returned or endpoint failed.
+                    log.debug "UNEXPECTED DIRECTIVE RECEIVED: \n#{result.inspect}"
+                    raise "No directive matched message (#{msg.message_id})"
+                    
+                  end
+                  log.debug "NEW STATE: " + ee.message_state.state.to_s
+                  log.debug "Delete message? #{msg.is_marked_for_delete?}"
+                else
+                  log.debug "Skipping #{msg.message_id} (#{msg_state.state})"
+                  log.debug "Delete message? #{msg.is_marked_for_delete?}"
+                end
+                email_processed_count += 1
+
+              rescue => e
+                rc = msg_state.retry
+                if rc >= (conn.max_retry_count - 1)
+                  msg_state.state = :failed
+                else
+                  msg_state.state = :error
+                end
+                log.error e.message
+                log.error "RETRY COUNT: #{rc}"
+                log.error "NEW STATE: #{msg_state.state}"
+                log.error "Delete message? #{msg.is_marked_for_delete?}"
+                email_errors_count += 1
+              end
+
+            }
+            queue.join
+          end
+          log.info "Number of email successfully processed for connection #{conn.name}: #{email_processed_count}"
+          log.info "Number of email unsuccesfully processed for connection #{conn.name}: #{email_errors_count}"
+        rescue => e
+          log.error "There was an error processing email for #{conn.name}: #{e.message}"
+        end
+        
+        
+      end
+
+      def go_async
+        threads = []
+        begin
+          Configuration.each_connection do |conn|
+            threads << Thread.new { go conn }
+          end
+          threads.each { |t| t.join }
+        rescue => e
+          Configuration.log.error e.message
+        end
+      end
+
+      def go_all
+        begin
+          Configuration.each_connection { |conn| go conn }
+        rescue => e
+          Configuration.log.error e.message
+        end
+      end
+
+      def flush
+        # flush all message states
+        Configuration.storage_engine.delete_all
+      end
+    end
+
+  end
+end

data/lib/kns_email_endpoint/storage/abstract_storage.rb

@@ -0,0 +1,93 @@
+module KNSEmailEndpoint
+  module Storage
+    class AbstractStorage
+      attr_reader :message_id, :unique_id, :retry_count, :state
+
+      def initialize(*args)
+
+        reset_storage
+      end
+
+      # delete should return either true if successful, or false if unsuccessful
+      # delete should also call reset_storage if successful
+      def delete
+        return false # override me
+
+      end
+
+      # create should call set_vars and save! if successful and return true
+      # otherwise, it should raise an exception
+      def create(opts={})
+        return false #override me
+
+      end
+
+      # find should return self if a record is found after calling set_vars
+      # it should return nil if not found and call reset_storage
+      def find(unique_id)
+        return nil #override me
+      end
+
+      def retry_count=(r)
+        raise "Unknown unique_id" unless @unique_id
+        @retry_count = r
+        save!
+      end
+
+      def state=(s)
+        raise "Unknown unique_id" unless @unique_id
+        @state = s
+        save!
+      end
+
+      def to_h
+        return {} unless @unique_id
+        return {
+          :unique_id => @unique_id,
+          :message_id => @message_id,
+          :retry_count => @retry_count,
+          :state => @state
+        }
+      end
+
+      def find_or_create(opts={})
+        unless opts[:unique_id] && opts[:message_id]
+          raise "Must provide at least a unique_id and message_id"
+        end
+
+        s = find(opts[:unique_id])
+        return s if s
+
+        return create(opts) ? self : nil
+      end
+
+      private
+
+      def save!
+        # override me
+      end
+
+      def set_vars(h)
+        h.symbolize_keys!
+        raise "Invalid unique_id" unless h[:unique_id]
+        raise "Invalid message_id" unless h[:message_id]
+        raise "Invalid retry_count" unless h[:retry_count]
+        raise "Invalid state" unless h[:state]
+        @unique_id = h[:unique_id]
+        @message_id = h[:message_id]
+        @retry_count = h[:retry_count]
+        @state = h[:state].to_sym
+        
+      end
+
+      def reset_storage
+        @unique_id = nil
+        @message_id = nil
+        @retry_count = nil
+        @state = nil
+        @current_file = nil
+      end
+
+    end
+  end
+end

data/lib/kns_email_endpoint/storage/file_storage.rb

@@ -0,0 +1,74 @@
+require 'json'
+require 'fileutils'
+module KNSEmailEndpoint
+  module Storage
+    class FileStorage < AbstractStorage
+      attr_reader :name
+
+      def initialize(settings={})
+        @dir = settings[:file_location]
+        raise "Unknown file_location" unless @dir
+        FileUtils.mkdir_p @dir
+
+        super(settings)
+        @name = "file"
+                
+      end
+
+      def delete
+        return false unless @current_file
+        FileUtils.rm @current_file if File.exists? @current_file
+        reset_storage
+        return true
+      end
+
+      def create(opts={})
+        options = {
+          :state => :unprocessed,
+          :retry_count => 0
+        }.merge! opts
+        raise ":unique_id is required" unless options[:unique_id]
+        raise ":message_id is required" unless options[:message_id]
+
+        @current_file = File.join(@dir, options[:unique_id]) 
+
+        if File.exists? @current_file
+          raise "unique_id #{options[:unique_id]} already exists"
+        end
+        
+        set_vars(options)
+        save!
+        
+        return true
+      end
+
+      def find(unique_id)
+        lookup_file = File.join(@dir, unique_id)
+        if File.exists? lookup_file
+          @current_file = lookup_file
+          set_vars JSON.parse(File.open(@current_file, 'r').read)
+          return self
+        else
+          reset_storage
+          return nil
+        end
+      end
+
+      def delete_all
+        FileUtils.remove_dir(@dir, true)
+        FileUtils.mkdir_p @dir
+      end
+
+
+      private
+
+      def save!
+        File.open(@current_file, "w") do |f|
+          f.write to_h.to_json
+        end
+      end
+
+
+    end
+  end
+end

data/lib/kns_email_endpoint/storage/memcache_storage.rb

@@ -0,0 +1,72 @@
+require 'dalli'
+module KNSEmailEndpoint
+  module Storage
+    class MemcacheStorage < AbstractStorage
+      class << self
+        attr_accessor :client
+      end
+
+      def initialize(settings)
+        options = {
+          :host => "localhost",
+          :port => 11211,
+          :ttl => nil
+        }.merge!(settings)
+
+
+        # This simple bit of magic allows usage of 
+        # a class connection rather than an instance connection
+        # so we never have more than one active connection to memcache
+        self.class.client ||= Dalli::Client.new("#{options[:host]}:#{options[:port]}")
+        @client = self.class.client
+        @ttl = options[:ttl]
+        
+      end
+
+      def create(opts={})
+        options = {
+          :state => :unprocessed,
+          :retry_count => 0
+        }.merge! opts
+        raise ":unique_id is required" unless options[:unique_id]
+        raise ":message_id is required" unless options[:message_id]
+        
+        set_vars(options)
+        save!
+        
+        return true
+      end
+
+      def find(unique_id)
+        s = @client.get(unique_id)
+        if s
+          set_vars(s)
+          return self 
+        else 
+          reset_storage
+          return nil
+        end
+      end
+
+      def delete
+        return false unless @unique_id
+        @client.delete @unique_id
+        reset_storage
+        return true
+      end
+
+      def delete_all
+        @client.flush
+        reset_storage
+        return true
+      end
+        
+      private
+
+      def save!
+        @client.set(@unique_id, to_h, @ttl)
+      end
+
+    end
+  end
+end

data/lib/kns_email_endpoint/storage/storage.rb

@@ -0,0 +1,15 @@
+require 'kns_email_endpoint/storage/abstract_storage'
+module KNSEmailEndpoint
+  module Storage
+    autoload :FileStorage, 'kns_email_endpoint/storage/file_storage'
+    autoload :MemcacheStorage, 'kns_email_endpoint/storage/memcache_storage'
+
+    def self.get_storage(engine, settings)
+      case engine.to_sym
+        when :file then return FileStorage.new(settings)
+        when :memcache then return MemcacheStorage.new(settings)
+        else return nil
+      end
+    end
+  end
+end

data/lib/kns_email_endpoint/version.rb

@@ -0,0 +1,3 @@
+module KnsEmailEndpoint
+  VERSION = "0.1.0"
+end

data/lib/kns_email_endpoint.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+
+# require some extensions
+require 'kns_email_endpoint/core_extensions/class'
+require 'kns_email_endpoint/core_extensions/hash'
+require 'kns_email_endpoint/core_extensions/imap'
+require 'kns_email_endpoint/core_extensions/message'
+require 'kns_email_endpoint/core_extensions/test_retriever'
+
+# all the gem pieces
+require 'kns_endpoint'
+require 'kns_email_endpoint/connection'
+require 'kns_email_endpoint/configuration'
+require 'kns_email_endpoint/email_endpoint'
+require 'kns_email_endpoint/storage/storage'
+require 'kns_email_endpoint/message_state'
+require 'kns_email_endpoint/process_email'
+
+

data/spec/a18x34/.app

@@ -0,0 +1,4 @@
+--- 
+:ruleset_id: a18x34
+:role: owner
+:name: Testing KRL CLI

data/spec/a18x34/a18x34.krl

@@ -0,0 +1,91 @@
+ruleset a18x34 {
+  meta {
+    name "Test App for Email Endpoint"
+    description <<
+      Testing application for the Email Endpoint
+    >>
+    author "Michael Farmer"
+    // Uncomment this line to require Marketplace purchase to use this app.
+    // authz require user
+    logging on
+  }
+
+
+  global {
+  
+  }
+
+  rule receive_new_email is active {
+    select when mail received test_rule "parts"
+    pre { 
+      envelope = event:param("msg");
+      from = event:param("from");
+      to = event:param("to");
+      subject = event:param("subject");
+      label = event:param("label");
+      unique_id = event:param("unique_id");
+      collection = {
+        "from": from,
+        "to": to,
+        "subject": subject,
+        "label": label,
+        "unique_id": unique_id
+      }
+    }
+    {
+      send_directive("processed");
+    } 
+
+    fired {
+      log collection.encode();
+    }
+
+  }
+
+  rule delete_mail is active {
+    select when mail received test_rule "delete_me"
+    {
+      email:delete();
+    }
+      
+  }
+
+  rule reply_mail is active {
+    select when mail received test_rule "reply"
+
+    {
+      email:reply() with body = "This is a reply message";
+    }
+  }
+
+  rule reply_and_delete_mail is active {
+    select when mail received test_rule "reply and delete"
+
+    {
+      email:reply() with body = "This is a reply message" and delete_message = true;
+    }
+  }
+
+
+  rule forward_mail is active {
+    select when mail received test_rule "forward"
+    pre {
+      fwd_to = event:param("forward_to");
+    }
+    {
+      email:forward() with to = fwd_to and body = "This is a forwarded message"
+    }
+      
+  }
+
+  rule forward_and_delete_mail is active {
+    select when mail received test_rule "forward and delete"
+    pre {
+      fwd_to = event:param("forward_to");
+    }
+    {
+      email:forward() with to = fwd_to and body = "This is a forwarded message" and delete_message = true;
+    }
+      
+  }
+}

data/spec/lib/kns_email_endpoint/configuration_spec.rb

@@ -0,0 +1,44 @@
+require 'lib/kns_email_endpoint'
+
+$CONFIG_FILE = File.join(File.dirname(__FILE__), '../..', 'test_config_file.yml') 
+
+module KNSEmailEndpoint
+  describe Configuration do
+
+    Configuration.load_from_file $CONFIG_FILE
+    let(:c) { Configuration }
+
+    it "should have a storage engine" do
+      c.storage_engine.should_not be_nil
+    end
+
+    describe "loaded" do
+
+      specify { c.work_threads.should eql 5}
+      specify { c.poll_delay.should eql 30}
+      specify { c.logdir.should eql "/tmp/email_endpoint"}
+      specify { c.connections.should_not be_empty }
+      specify { c.storage.should_not be_empty }
+      specify { c.log.class.should == Logger }
+      specify { c.storage_engine.class.should == KNSEmailEndpoint::Storage::MemcacheStorage }
+      specify { c.log.level.should == 0 }
+
+    end
+
+
+    describe "access connections" do
+
+      it 'should return a connection by name' do
+        c["test"]["name"].should eql "test"
+      end
+
+      it "should let me loop through the connections" do
+        c.each_connection do |conn|
+          ["test", "gmail"].should include(conn.name)
+        end
+
+      end
+    end
+
+  end
+end