mail_daemon 0.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 48feab4732334f1c8c9286c661a700b603ef6189
+  data.tar.gz: 410115f6ee8a2e51c3a46907b5639ceb160c8810
+SHA512:
+  metadata.gz: 6b41e1e61aaca95d8f32e5967b4d2a488dd292f8f211ebb3d33530045121307be75c667026a2a6dab1b1219eb172dfc02efda21a5d482333b7bf06801a9a02ca
+  data.tar.gz: f3458d0f73b2df26d6a119a89680aabc4cb7f8248769c8b46cf04d4eaa8941b6ef9c2d1ad761b4f56dc5031460ef54dbb6e8aa3af87660c4bf5ad5f7c99a3f5c

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.swp

data/.ruby-gemset

@@ -0,0 +1 @@
+mail_daemon

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.2.3

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mail_daemon.gemspec
+gemspec
+
+gem "mail_room", :path => "/Users/stewartmckee/code/mail_room"

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 EmergeAdapt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Ijonas Kisselbach
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# mail_daemon
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mail_daemon'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mail_daemon
+
+## Design
+
+###Gems
+
+*  mail_room - retrieves mail from multiple mailboxes at once and sends them through sidekiq or an http postback
+*  mail - mail parsing and sending gem
+*  talon - python code to extract the relevant body of the message
+
+
+### Notes
+
+ * config for office365 smtp needs authentication type 'login' gmail is 'plain'
+ * gmail apps authentication is by ip address only, suggest a feature of being able to host the send daemon on another box
+
+## Usage
+
+Ensure the following environment variables are set:
+
+    REDIS_URL=redis://localhost:6379
+    CB_API_ENDPOINT=https://login.caseblocks.com
+    CB_API_TOKEN=...
+
+And run the daemon from the CLI
+
+    mail_daemon
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/mail_daemon/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/mail_daemon

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby -Ilib
+require 'rubygems'
+require 'bundler/setup'
+# Prepares the $LOAD_PATH by adding to it lib directories of the gem and
+# its dependencies:
+require 'mail_daemon'
+require 'ap'
+
+
+mailboxes = [
+  {:host => "imap.gmail.com", :username => "stewart@emergeadapt.com", :port => 993, :start_tls => true, :password => "junction1", :ssl => true}
+]
+
+handler = MailDaemon::Handler.new(:connections => mailboxes)
+
+handler.start do |message, type|
+  if type == "status_update"
+    puts "#{message[:status][0].upcase}#{message[:status][1..-1]} #{message[:mailbox][:username]}"
+  else
+    ap message
+  end
+end

data/config.yml

@@ -0,0 +1,27 @@
+---
+:mailboxes:
+  -
+    :email: "stewart@activeinformationdesign.com"
+    :password: "junction1"
+    :name: "inbox"
+    :search_command: 'UNSEEN'
+    :delivery_method: sidekiq
+    :delivery_options:
+      :worker: EmailHandlerWorker
+  -
+    :email: "stewart@emergeadapt.com"
+    :password: "junction1"
+    :name: "inbox"
+    :search_command: 'UNSEEN'
+    :delivery_method: sidekiq
+    :delivery_options:
+      :worker: EmailHandlerWorker
+  -
+    :email: "stewartmckee@emergeadapttest.onmicrosoft.com"
+    :host: "outlook.office365.com"
+    :password: "Katr1n33"
+    :name: "inbox"
+    :search_command: 'UNSEEN'
+    :delivery_method: sidekiq
+    :delivery_options:
+      :worker: EmailHandlerWorker

data/lib/mail_daemon.rb

@@ -0,0 +1,71 @@
+require_relative 'mail_daemon/version'
+require_relative 'mail_daemon/helpers'
+require_relative 'mail_daemon/email_body_parser'
+require_relative 'mail_daemon/email_handler'
+require_relative 'mail_daemon/imap/statuses'
+require_relative 'mail_daemon/imap/connection'
+require_relative 'mail_daemon/imap/watcher'
+require_relative 'mail_daemon/encryption'
+require 'sinatra/base'
+require 'json'
+require 'open3'
+require 'mail'
+
+include MailDaemon::Helpers
+
+module MailDaemon
+  class Handler
+    def initialize(options)
+      setup_options(options)
+      default_option :connections, []
+
+      Signal.trap("INT") {
+        Thread.new {self.stop}.join
+      }
+      # Trap `Kill `
+      Signal.trap("TERM") {
+        Thread.new {self.stop}.join
+      }
+    end
+
+    def restart
+      self.stop if self.running?
+      self.reload unless @mailbox_config
+      self.start
+    end
+
+    def running?
+      @watchers && @watchers.detect{|watcher| watcher.running?}
+    end
+
+    # Signal catching
+    def stop
+      @watchers.map{|watcher| watcher.stop}
+      @threads.map{|t| t.kill }
+    end
+
+    def start(&block)
+      @watchers = []
+
+      @options[:connections].each do |mailbox|
+        mailbox[:ssl_options] = {:verify_mode => OpenSSL::SSL::VERIFY_NONE} if mailbox[:ssl]
+
+        puts "Setting up watcher for #{mailbox[:username]}"
+        @watchers << MailDaemon::Imap::Watcher.new(mailbox)
+      end
+      @threads = []
+      @watchers.each do |watcher|
+        @threads << Thread.new do
+          watcher.start do |message|
+            type = message.delete(:type)
+            yield message, type
+          end
+        end
+      end
+
+      @threads.map{|t| t.join}
+
+    end
+
+  end
+end

data/lib/mail_daemon/email_body_parser.rb

@@ -0,0 +1,28 @@
+require 'open3'
+
+module MailDaemon
+  class EmailBodyParser
+    def self.parse(text, type="text/plain")
+
+      data_hash = {:body => text, :type => type}
+
+      json_hash = data_hash.to_json
+
+      stdin, stdout, stderr, wait_thr = Open3.popen3('python', '/Users/stewartmckee/code/talon/test.py')
+      stdin.write(json_hash)
+      stdin.close
+      result = stdout.read
+      stdout.close
+      error = stderr.read
+      stderr.close
+      exit_code = wait_thr.value
+      if exit_code == 0
+        result
+      else
+        puts result
+        raise error
+      end
+    end
+
+  end
+end

data/lib/mail_daemon/email_handler.rb

@@ -0,0 +1,109 @@
+require 'fileutils'
+require 'mail'
+require_relative "email_body_parser"
+require 'nokogiri'
+require 'sanitize'
+
+module MailDaemon
+  class EmailHandler
+    def initialize(options)
+      @options = options
+      @email = Mail.read_from_string(options[:inbound_message])
+
+      puts "Handling inbound email from #{@email.from}"
+
+      if @email.multipart?
+        @email.parts.each do |part|
+          content_type = part.content_type.split(";")[0]
+          if content_type[0..3] == "text"
+            if content_type == "text/html"
+              @html_body = part.body.decoded.force_encoding("ASCII-8BIT").encode('UTF-8', undef: :replace, replace: '')
+            elsif content_type == "text/plain"
+              @text_body = part.body.decoded.force_encoding("ASCII-8BIT").encode('UTF-8', undef: :replace, replace: '')
+            end
+          end
+        end
+      else
+        @text_body = @email.body.decoded.force_encoding("ASCII-8BIT").encode('UTF-8', undef: :replace, replace: '')
+      end
+
+      @body = @html_body || @text_body
+    end
+
+    def generate_message_hash
+
+      message_hash = {
+        "from" => @email.from,
+        "to" => @email.to,
+        "subject" => @email.subject,
+        "body" => @body,
+        "attachements" => prepare_attachments,
+        "metadata" => fetch_metadata
+      }
+      message_hash.delete(:inbound_message)
+      message_hash
+    end
+
+    def fetch_metadata
+      fetch_fingerprints.merge({
+        "references" => references
+      })
+    end
+
+    def references
+      @email.header["References"] ? @email.header["References"].field.element.message_ids : []
+    end
+
+    def fetch_fingerprints
+      previous_id = references.last
+      match = /(.*?)\.(.*?)@emergeadapt.com/.match(previous_id)
+      if match
+        if match[2] == "case"
+          {"case_id" => match[1]}
+        else
+          {"message_id" => match[1]}
+        end
+      else
+        {}
+      end
+    end
+
+    def prepare_attachments
+      attachments = []
+      @email.attachments.each do | attachment |
+        filename = attachment.filename
+        random_tag = Random.rand(100000000000)
+        attachment_folder = File.join(["/", "tmp", "caseblocks", @options[:mailbox][:account_code], "attachments"])
+        FileUtils::mkdir_p(attachment_folder)
+        file_path = File.join([attachment_folder, "#{random_tag}.#{filename}"])
+        File.open(file_path, "w+b", 0644) {|f| f.write attachment.body.decoded}
+        attachments << {
+          "filename" => attachment.filename,
+          "filepath" => file_path
+        }
+      end
+      attachments
+    end
+
+
+    def content_type
+      @html_body.nil? ? "text/plain" : "text/html"
+    end
+
+    def sanitize!
+      @body = Sanitize.fragment(@body, Sanitize::Config::RELAXED)
+    end
+
+    def clean_replies!
+      @body = MailDaemon::EmailBodyParser.parse(@body, content_type)
+    end
+
+    def queue!
+      $redis
+    end
+
+    def body
+      @body
+    end
+  end
+end

data/lib/mail_daemon/email_watcher.rb

@@ -0,0 +1,155 @@
+require 'redis'
+require 'ap'
+require 'mysql2'
+
+module MailDaemon
+  class EmailWatcher
+
+    def initialize
+
+      raise "REDIS_URL environment variable is required (eg redis://localhost:6739)" unless ENV["REDIS_URL"]
+      raise "MYSQL_HOST environment variable is required" unless ENV["MYSQL_HOST"]
+      raise "MYSQL_DATABASE environment variable is required" unless ENV["MYSQL_DATABASE"]
+      raise "MYSQL_USERNAME environment variable is required" unless ENV["MYSQL_USERNAME"]
+      raise "MYSQL_PASSWORD environment variable is required" unless ENV["MYSQL_PASSWORD"]
+
+      ENV["MYSQL_PASSWORD"] = "" unless ENV["MYSQL_PASSWORD"]
+      ENV["MYSQL_PORT"] = "3306"
+
+      redis_url = URI.parse(ENV["REDIS_URL"])
+
+      $redis = Redis.new(:host => redis_url.host, :port => redis_url.port)
+
+      Signal.trap("INT") {
+        Thread.new {self.stop}.join
+      }
+      # Trap `Kill `
+      Signal.trap("TERM") {
+        Thread.new {self.stop}.join
+      }
+
+
+
+      start
+    end
+
+    def configuration
+      mailboxes = []
+      mysql_client do |mysql|
+        statement = mysql.prepare("SELECT case_blocks_email_accounts.id, case_blocks_accounts.nickname, case_blocks_email_accounts.imap_username, case_blocks_email_accounts.imap_encrypted_password, case_blocks_email_accounts.imap_host, case_blocks_email_accounts.imap_port, case_blocks_email_accounts.imap_ssl, case_blocks_email_accounts.imap_start_tls, case_blocks_email_accounts.imap_folder_name, case_blocks_email_accounts.imap_search_command, case_blocks_email_accounts.imap_messages_processed, case_blocks_email_accounts.imap_last_processed_at FROM case_blocks_email_accounts JOIN case_blocks_accounts ON case_blocks_email_accounts.account_id = case_blocks_accounts.id where imap_enabled=1")
+        result = statement.execute()
+        result.each do |row|
+          ssl_options = row["imap_ssl"]==1 ? {:verify_mode => OpenSSL::SSL::VERIFY_NONE} : false
+          decrypted_password = Encryption.new.decrypt(row["imap_encrypted_password"])
+          mailboxes << {:id => row["id"], :account_code => row["nickname"], :username => row["imap_username"], :host => row["imap_host"], :port => row["imap_port"], :password => decrypted_password, :ssl_options => ssl_options, :start_tls => row["imap_start_tls"]==1, :name => row["imap_folder_nam"], :search_command => row["imap_search_command"], :message_count => row["imap_messages_processed"], :last_delivered_at => row["imap_last_processed_at"]}
+        end
+      end
+      mailboxes
+    end
+
+    def restart
+      self.stop if self.running?
+      self.reload unless @mailbox_config
+      self.start
+    end
+
+    def running?
+      @watchers.detect{|watcher| watcher.running?}
+    end
+
+    # Signal catching
+    def stop
+      @watchers.map{|watcher| watcher.stop}
+      @threads.map{|t| t.kill }
+    end
+
+    def start
+      @watchers = []
+
+      configuration.each do |mailbox|
+        puts "Setting up watcher for #{mailbox[:username]}"
+        @watchers << MailDaemon::Imap::Watcher.new(mailbox)
+      end
+
+      @threads = []
+      @watchers.each do |watcher|
+        @threads << Thread.new do
+          watcher.start do |message|
+            if message[:type] == "status_update"
+              update_status(message)
+            elsif message[:type] == "incoming_email"
+              handle_incoming_message(message)
+            else
+              puts "Unknown message type"
+              ap message
+            end
+          end
+        end
+      end
+      @threads.map{|t| t.join}
+
+    end
+
+    def handle_incoming_message(options)
+      @handler = MailDaemon::EmailHandler.new(options)
+      @handler.sanitize!
+      @handler.clean_replies!
+
+      hash = @handler.generate_message_hash
+
+      $redis.lpush(redis_key(options), hash)
+      puts "Queued."
+    end
+
+    def redis_key(options)
+      "cb:comms:#{options[:mailbox][:account_code]}:inbox"
+    end
+
+    def update_status(options)
+      puts "Status Update: #{options[:status]} for #{options[:mailbox][:username]}"
+      mysql_client do |mysql|
+        status_message = options[:status]
+        statement = mysql.prepare("UPDATE case_blocks_email_accounts SET imap_status='#{options[:status]}', imap_status_message='#{status_message}' where id=?")
+        statement.execute(options[:mailbox][:id])
+      end
+    end
+
+    private
+
+    def mysql_client(&block)
+      client = Mysql2::Client.new(:host => ENV["MYSQL_HOST"], :port => ENV["MYSQL_PORT"], :username => ENV["MYSQL_USERNAME"], :password => ENV["MYSQL_PASSWORD"], :database => ENV["MYSQL_DATABASE"])
+      yield client
+      client.close
+    end
+
+    def mailbox_status_message(watcher)
+      if watcher.logging_in?
+        "Connecting and Logging into server"
+      elsif watcher.logged_in?
+        "Connected"
+      else
+        "Disconnected"
+      end
+    end
+    def mailbox_status(watcher)
+      if watcher.logging_in?
+        :logging_in
+      elsif watcher.logged_in?
+        :connected
+      else
+        :disconnected
+      end
+    end
+    def mailbox_status_colour(watcher)
+      if watcher.logging_in?
+        "orange"
+      elsif watcher.logged_in?
+        "green"
+      else
+        "red"
+      end
+    end
+
+
+  end
+end