smartmachine 1.2.3 → 1.3.0

data/lib/smart_machine/syncer.rb

@@ -64,6 +64,9 @@ module SmartMachine
         'grids/elasticsearch',
         'grids/elasticsearch/***',
 
+        'grids/emailer',
+        'grids/emailer/***',
+
         'grids/minio',
         'grids/minio/***',
 
@@ -103,11 +106,17 @@ module SmartMachine
         'bin/smartmachine',
 
         'config',
+        'config/emailer',
+        'config/emailer/***',
         'config/mysql',
         'config/mysql/schedule.rb',
         'config/phpmyadmin',
         'config/phpmyadmin/***',
+        'config/roundcube',
+        'config/roundcube/***',
         'config/credentials.yml.enc',
+        'config/emailer.yml',
+        'config/engine.yml',
         'config/environment.rb',
         'config/elasticsearch.yml',
         'config/minio.yml',
@@ -117,6 +126,7 @@ module SmartMachine
         'config/phpmyadmin.yml',
         'config/prereceiver.yml',
         'config/redis.yml',
+        'config/roundcube.yml',
         'config/terminal.yml',
 
         'grids',

data/lib/smart_machine/templates/dotsmartmachine/config/emailer/docker/command.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+STDOUT.sync = true
+
+class Services
+  def initialize
+    @services = %w(rsyslog haproxy spamassassin spamassassin.update opendkim postfix dovecot)
+  end
+
+  def start
+    puts "Starting Services..."
+    @services.each { |service| system("monit start #{service}") }
+    system("monit")
+
+    puts "Starting Logtailer..."
+    system("/usr/bin/logtailer.rb start")
+  end
+
+  def stop(signo)
+    puts "Stopping Logtailer..."
+    system("/usr/bin/logtailer.rb stop")
+
+    puts "Stopping Services... SIGNAL: SIG#{Signal.signame(signo)}."
+    system("monit quit")
+    sleep(3)
+    @services.reverse.each { |service| system("monit stop #{service}") }
+
+    puts "Flushing Logtailer..."
+    system("/usr/bin/logtailer.rb flush")
+
+    exit
+  end
+end
+
+trap('SIGINT') do |signo|
+  Services.new.stop(signo)
+end
+
+trap('SIGTERM') do |signo|
+  Services.new.stop(signo)
+end
+
+at_exit do
+  # Clean up.
+end
+
+Services.new.start
+
+sleep

data/lib/smart_machine/templates/dotsmartmachine/config/emailer/docker/entrypoint.rb

@@ -0,0 +1,196 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'logger'
+
+logger = Logger.new(STDOUT)
+STDOUT.sync = true
+
+def update_envkeys_in(filepaths, envkeys)
+  filepaths.each do |filepath|
+    str = File.read(filepath)
+    str = str.gsub(/%(?!<)/, '%%')
+    str = format(str, envkeys)
+    File.open(filepath, "w") { |file| file << str }
+  end
+end
+
+# initial setup
+unless File.exist?('/run/initial_container_start')
+  FileUtils.touch('/run/initial_container_start')
+
+  # EnvKeys
+  envkeys = {
+    container_name: ENV.delete('CONTAINER_NAME'),
+    fqdn: ENV.delete('FQDN'),
+    mailname: ENV.delete('MAILNAME'),
+    sysadmin_email: ENV.delete('SYSADMIN_EMAIL'),
+    mysql_host: ENV.delete('MYSQL_HOST'),
+    mysql_port: ENV.delete('MYSQL_PORT'),
+    mysql_user: ENV.delete('MYSQL_USER'),
+    mysql_password: ENV.delete('MYSQL_PASSWORD'),
+    mysql_database_name: ENV.delete('MYSQL_DATABASE_NAME'),
+    monit_smtp_email_name: ENV.delete('MONIT_SMTP_EMAIL_NAME'),
+    monit_smtp_email_address: ENV.delete('MONIT_SMTP_EMAIL_ADDRESS'),
+    monit_smtp_host: ENV.delete('MONIT_SMTP_HOST'),
+    monit_smtp_port: ENV.delete('MONIT_SMTP_PORT'),
+    monit_smtp_username: ENV.delete('MONIT_SMTP_USERNAME'),
+    monit_smtp_password: ENV.delete('MONIT_SMTP_PASSWORD'),
+    oracle_ips_allowed: ENV.delete('ORACLE_IPS_ALLOWED'),
+    oracle_deflect_url: ENV.delete('ORACLE_DEFLECT_URL'),
+    timezone: `cat /etc/timezone`.chomp
+  }
+
+  # rsyslog
+  # imklog module is commented in rsyslog.conf because rsyslog does not
+  # have privileges to run it and hence throws error on startup.
+  system("sed -i '/imklog/s/^/#/' /etc/rsyslog.conf")
+
+  # Postfix
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/main.cf', '/etc/postfix/main.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/master.cf', '/etc/postfix/master.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-sender-login-maps.cf', '/etc/postfix/mysql-sender-login-maps.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-virtual-alias-domains.cf', '/etc/postfix/mysql-virtual-alias-domains.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-virtual-alias-maps-domains.cf', '/etc/postfix/mysql-virtual-alias-maps-domains.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-virtual-alias-maps-masters.cf', '/etc/postfix/mysql-virtual-alias-maps-masters.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-virtual-alias-maps-users.cf', '/etc/postfix/mysql-virtual-alias-maps-users.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-virtual-alias-maps-userstothemselves.cf', '/etc/postfix/mysql-virtual-alias-maps-userstothemselves.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-virtual-mailbox-domains.cf', '/etc/postfix/mysql-virtual-mailbox-domains.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix/mysql-virtual-mailbox-maps.cf', '/etc/postfix/mysql-virtual-mailbox-maps.cf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/postfix-policyd-spf-python/policyd-spf.conf', '/etc/postfix-policyd-spf-python/policyd-spf.conf'
+  filepaths = [
+    '/etc/postfix/main.cf',
+    '/etc/postfix/mysql-sender-login-maps.cf',
+    '/etc/postfix/mysql-virtual-alias-domains.cf',
+    '/etc/postfix/mysql-virtual-alias-maps-domains.cf',
+    '/etc/postfix/mysql-virtual-alias-maps-masters.cf',
+    '/etc/postfix/mysql-virtual-alias-maps-users.cf',
+    '/etc/postfix/mysql-virtual-alias-maps-userstothemselves.cf',
+    '/etc/postfix/mysql-virtual-mailbox-domains.cf',
+    '/etc/postfix/mysql-virtual-mailbox-maps.cf'
+  ]
+  update_envkeys_in(filepaths, envkeys)
+  system("chgrp postfix /etc/postfix/mysql-*.cf")
+  system("chmod -R o-rwx /etc/postfix/mysql-*.cf")
+
+  # Dovecot
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/10-auth.conf', '/etc/dovecot/conf.d/10-auth.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/10-mail.conf', '/etc/dovecot/conf.d/10-mail.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/10-master.conf', '/etc/dovecot/conf.d/10-master.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/10-ssl.conf', '/etc/dovecot/conf.d/10-ssl.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/15-mailboxes.conf', '/etc/dovecot/conf.d/15-mailboxes.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/20-imap.conf', '/etc/dovecot/conf.d/20-imap.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/20-lmtp.conf', '/etc/dovecot/conf.d/20-lmtp.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/90-quota.conf', '/etc/dovecot/conf.d/90-quota.conf'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/conf.d/90-sieve.conf', '/etc/dovecot/conf.d/90-sieve.conf'
+
+  FileUtils.mkdir '/etc/dovecot/sieve'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/sieve/learn-ham.sh', '/etc/dovecot/sieve/learn-ham.sh'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/sieve/learn-spam.sh', '/etc/dovecot/sieve/learn-spam.sh'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/sieve/learn-ham.sieve', '/etc/dovecot/sieve/learn-ham.sieve'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/sieve/learn-spam.sieve', '/etc/dovecot/sieve/learn-spam.sieve'
+
+  FileUtils.mkdir '/etc/dovecot/sieve-after'
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/sieve-after/spam-to-folder.sieve', '/etc/dovecot/sieve-after/spam-to-folder.sieve'
+
+  FileUtils.cp '/smartmachine/config/emailer/etc/dovecot/dovecot-sql.conf.ext', '/etc/dovecot/dovecot-sql.conf.ext'
+
+  FileUtils.cp '/smartmachine/config/emailer/usr/local/bin/quota-warning.sh', '/usr/local/bin/quota-warning.sh'
+
+  filepaths = [
+    '/etc/dovecot/conf.d/10-ssl.conf',
+    '/etc/dovecot/dovecot-sql.conf.ext',
+    '/usr/local/bin/quota-warning.sh'
+  ]
+  update_envkeys_in(filepaths, envkeys)
+
+  system("groupadd -g 5000 vmail")
+  system("useradd -g vmail -u 5000 vmail -d /var/vmail")
+  system("chown -R vmail:vmail /var/vmail")
+
+  system("sievec /etc/dovecot/sieve/learn-ham.sieve")
+  system("sievec /etc/dovecot/sieve/learn-spam.sieve")
+  system("chmod u=rwx,go= /etc/dovecot/sieve/learn-*.sh")
+  system("chown vmail:vmail /etc/dovecot/sieve/learn-*.sh")
+  system("chmod u=rw,go= /etc/dovecot/sieve/learn-*.sieve")
+  system("chown vmail:vmail /etc/dovecot/sieve/learn-*.sieve")
+  system("chmod u=rw,go= /etc/dovecot/sieve/learn-*.svbin")
+  system("chown vmail:vmail /etc/dovecot/sieve/learn-*.svbin")
+
+  system("sievec /etc/dovecot/sieve-after/spam-to-folder.sieve")
+  system("chmod u=rw,go= /etc/dovecot/sieve-after/spam-to-folder.sieve")
+  system("chown vmail:vmail /etc/dovecot/sieve-after/spam-to-folder.sieve")
+  system("chmod u=rw,go= /etc/dovecot/sieve-after/spam-to-folder.svbin")
+  system("chown vmail:vmail /etc/dovecot/sieve-after/spam-to-folder.svbin")
+
+  system("chown root:root /etc/dovecot/dovecot-sql.conf.ext")
+  system("chmod go= /etc/dovecot/dovecot-sql.conf.ext")
+
+  system("chmod +x /usr/local/bin/quota-warning.sh")
+
+  # Spamassassin
+  FileUtils.cp '/smartmachine/config/emailer/etc/spamassassin/local.cf', '/etc/spamassassin/local.cf'
+  system("adduser --gecos '' --disabled-login spamd", out: File::NULL)
+
+  # OpenDKIM
+  FileUtils.cp '/smartmachine/config/emailer/etc/opendkim.conf', '/etc/opendkim.conf'
+  system("adduser postfix opendkim", out: File::NULL)
+  system("chmod u=rw,go=r /etc/opendkim.conf")
+  unless File.exists? '/etc/opendkim/key.table'
+    FileUtils.mkdir_p '/etc/opendkim/keys'
+    FileUtils.touch '/etc/opendkim/key.table'
+    FileUtils.touch '/etc/opendkim/signing.table'
+    FileUtils.touch '/etc/opendkim/trusted.hosts'
+    key_shortname = envkeys[:mailname].gsub(/[^[:alnum:]]/, "")
+    raise "Could not create key_shortname from mailname to use in opendkim." if key_shortname.match(/\A[a-zA-Z0-9]*\z/).nil?
+    key_selector  = Time.now.getlocal('+05:30').strftime("%Y%m")
+    raise "Could not create key_selector from Local Time to use in opendkim." if key_selector.match(/\A[0-9]*\z/).nil?
+    key_filename = "#{key_shortname}_#{key_selector}"
+    IO.write("/etc/opendkim/key.table",
+             "#{key_shortname}     #{envkeys[:mailname]}:#{key_selector}:/etc/opendkim/keys/#{key_filename}.private\n")
+    IO.write("/etc/opendkim/signing.table",
+             "*@#{envkeys[:mailname]}   #{key_shortname}\n")
+    IO.write("/etc/opendkim/trusted.hosts",
+             "127.0.0.1\n::1\nlocalhost\n#{envkeys[:fqdn]}\n#{envkeys[:mailname]}\n")
+    Dir.chdir("/etc/opendkim/keys") do
+      raise "Could not create DKIM keys." unless system("opendkim-genkey -b 2048 -h rsa-sha256 -r -s #{key_selector} -d #{envkeys[:mailname]} -v")
+      FileUtils.mv("#{key_selector}.private", "#{key_filename}.private")
+      FileUtils.mv("#{key_selector}.txt", "#{key_filename}.txt")
+    end
+  end
+  system("chown -R opendkim:opendkim /etc/opendkim")
+  system("chmod -R go-rw /etc/opendkim/keys")
+  system("mkdir /var/spool/postfix/opendkim")
+  system("chown opendkim:postfix /var/spool/postfix/opendkim")
+
+  # Haproxy
+  FileUtils.mkdir_p '/var/lib/haproxy/dev'
+  FileUtils.mkdir_p '/run/haproxy'
+  FileUtils.cp '/smartmachine/config/emailer/etc/haproxy/haproxy.cfg', '/etc/haproxy/haproxy.cfg'
+  filepaths = [
+    '/etc/haproxy/haproxy.cfg'
+  ]
+  update_envkeys_in(filepaths, envkeys)
+
+  # Monit
+  FileUtils.cp '/smartmachine/config/emailer/etc/monit/monitrc', '/etc/monit/monitrc'
+  FileUtils.cp_r '/smartmachine/config/emailer/etc/monit/conf.d/.', '/etc/monit/conf.d'
+  filepaths = [
+    '/etc/monit/conf.d/services.cfg',
+    '/etc/monit/monitrc'
+  ]
+  update_envkeys_in(filepaths, envkeys)
+
+  # Logtailer
+  FileUtils.cp '/smartmachine/config/emailer/docker/logtailer.rb', '/usr/bin/logtailer.rb'
+  system("chmod +x /usr/bin/logtailer.rb")
+
+  # Command
+  FileUtils.cp '/smartmachine/config/emailer/docker/command.rb', '/usr/bin/command.rb'
+  system("chmod +x /usr/bin/command.rb")
+
+  logger.info "Initial setup completed for #{envkeys[:container_name]}."
+end
+
+ARGV.empty? ? exec("/usr/bin/command.rb") : exec(*ARGV)

data/lib/smart_machine/templates/dotsmartmachine/config/emailer/docker/logtailer.rb

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+STDOUT.sync = true
+
+require 'fileutils'
+
+class Logtailer
+  def initialize
+    @tailers = {
+      "/var/log/monit.log"    => 1,
+      "/var/log/haproxy.log"  => 1,
+      "/var/log/mail.log"     => 1,
+      "/home/spamd/spamd.log" => 1
+    }
+  end
+
+  def start
+    set_start_from_line
+
+    pids = []
+    @tailers.each do |path, start_from_line|
+      pid = Process.spawn("tail", "--lines=+#{start_from_line}", "-q", "-F", "#{path}", [:out, :err] => "/proc/1/fd/1")
+      Process.detach(pid)
+      pids.push(pid)
+    end
+    IO.write("/run/tmpfs/logtailer.pid", "#{pids.join(' ')}\n")
+
+    puts "Started Logtailer with PIDs " + `cat /run/tmpfs/logtailer.pid`.chomp + "."
+  end
+
+  def stop
+    pids = `cat /run/tmpfs/logtailer.pid`.chomp.split(" ")
+    pids.each do |pid|
+      system("/bin/kill --signal SIGTERM #{pid}")
+    end
+    save_start_from_line
+
+    puts "Stopped Logtailer with PIDs " + `cat /run/tmpfs/logtailer.pid`.chomp + "."
+    FileUtils.rm("/run/tmpfs/logtailer.pid")
+  end
+
+  def flush
+    set_start_from_line
+    @tailers.each do |path, start_from_line|
+      system("tail --lines=+#{start_from_line} -q #{path} >> /proc/1/fd/1")
+    end
+    save_start_from_line
+  end
+
+  private
+
+  def set_start_from_line
+    if File.exist?('/run/logtailer.lines')
+      lines = IO.read('/run/logtailer.lines').split("\n")
+      lines.each do |line|
+        previous_line_no, path = line.split(" ")
+        @tailers[path] = previous_line_no.to_i + 1
+      end
+    end
+  end
+
+  def save_start_from_line
+    str = `wc -l #{@tailers.keys.join(' ')} | head --lines=-1`
+    IO.write("/run/logtailer.lines", "#{str}")
+  end
+end
+
+if ARGV[0] == "start"
+  Logtailer.new.start
+elsif ARGV[0] == "stop"
+  Logtailer.new.stop
+elsif ARGV[0] == "flush"
+  Logtailer.new.flush
+end

data/lib/smart_machine/templates/dotsmartmachine/config/emailer/etc/dovecot/conf.d/10-auth.conf

@@ -0,0 +1,132 @@
+##
+## Authentication processes
+##
+
+# Disable LOGIN command and all other plaintext authentications unless
+# SSL/TLS is used (LOGINDISABLED capability). Note that if the remote IP
+# matches the local IP (ie. you're connecting from the same computer), the
+# connection is considered secure and plaintext authentication is allowed.
+# See also ssl=required setting.
+#disable_plaintext_auth = yes
+
+# Authentication cache size (e.g. 10M). 0 means it's disabled. Note that
+# bsdauth and PAM require cache_key to be set for caching to be used.
+#auth_cache_size = 0
+# Time to live for cached data. After TTL expires the cached record is no
+# longer used, *except* if the main database lookup returns internal failure.
+# We also try to handle password changes automatically: If user's previous
+# authentication was successful, but this one wasn't, the cache isn't used.
+# For now this works only with plaintext authentication.
+#auth_cache_ttl = 1 hour
+# TTL for negative hits (user not found, password mismatch).
+# 0 disables caching them completely.
+#auth_cache_negative_ttl = 1 hour
+
+# Space separated list of realms for SASL authentication mechanisms that need
+# them. You can leave it empty if you don't want to support multiple realms.
+# Many clients simply use the first one listed here, so keep the default realm
+# first.
+#auth_realms =
+
+# Default realm/domain to use if none was specified. This is used for both
+# SASL realms and appending @domain to username in plaintext logins.
+#auth_default_realm = 
+
+# List of allowed characters in username. If the user-given username contains
+# a character not listed in here, the login automatically fails. This is just
+# an extra check to make sure user can't exploit any potential quote escaping
+# vulnerabilities with SQL/LDAP databases. If you want to allow all characters,
+# set this value to empty.
+#auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@
+
+# Username character translations before it's looked up from databases. The
+# value contains series of from -> to characters. For example "#@/@" means
+# that '#' and '/' characters are translated to '@'.
+#auth_username_translation =
+
+# Username formatting before it's looked up from databases. You can use
+# the standard variables here, eg. %Lu would lowercase the username, %n would
+# drop away the domain if it was given, or "%n-AT-%d" would change the '@' into
+# "-AT-". This translation is done after auth_username_translation changes.
+#auth_username_format = %Lu
+
+# If you want to allow master users to log in by specifying the master
+# username within the normal username string (ie. not using SASL mechanism's
+# support for it), you can specify the separator character here. The format
+# is then <username><separator><master username>. UW-IMAP uses "*" as the
+# separator, so that could be a good choice.
+#auth_master_user_separator =
+
+# Username to use for users logging in with ANONYMOUS SASL mechanism
+#auth_anonymous_username = anonymous
+
+# Maximum number of dovecot-auth worker processes. They're used to execute
+# blocking passdb and userdb queries (eg. MySQL and PAM). They're
+# automatically created and destroyed as needed.
+#auth_worker_max_count = 30
+
+# Host name to use in GSSAPI principal names. The default is to use the
+# name returned by gethostname(). Use "$ALL" (with quotes) to allow all keytab
+# entries.
+#auth_gssapi_hostname =
+
+# Kerberos keytab to use for the GSSAPI mechanism. Will use the system
+# default (usually /etc/krb5.keytab) if not specified. You may need to change
+# the auth service to run as root to be able to read this file.
+#auth_krb5_keytab = 
+
+# Do NTLM and GSS-SPNEGO authentication using Samba's winbind daemon and
+# ntlm_auth helper. <doc/wiki/Authentication/Mechanisms/Winbind.txt>
+#auth_use_winbind = no
+
+# Path for Samba's ntlm_auth helper binary.
+#auth_winbind_helper_path = /usr/bin/ntlm_auth
+
+# Time to delay before replying to failed authentications.
+#auth_failure_delay = 2 secs
+
+# Require a valid SSL client certificate or the authentication fails.
+#auth_ssl_require_client_cert = no
+
+# Take the username from client's SSL certificate, using 
+# X509_NAME_get_text_by_NID() which returns the subject's DN's
+# CommonName. 
+#auth_ssl_username_from_cert = no
+
+# Space separated list of wanted authentication mechanisms:
+#   plain login digest-md5 cram-md5 ntlm rpa apop anonymous gssapi otp
+#   gss-spnego
+# NOTE: See also disable_plaintext_auth setting.
+##### SmartMachine Begin.
+#auth_mechanisms = plain
+auth_mechanisms = plain login
+##### SmartMachine Close.
+
+##
+## Password and user databases
+##
+
+#
+# Password database is used to verify user's password (and nothing more).
+# You can have multiple passdbs and userdbs. This is useful if you want to
+# allow both system users (/etc/passwd) and virtual users to login without
+# duplicating the system users into virtual database.
+#
+# <doc/wiki/PasswordDatabase.txt>
+#
+# User database specifies where mails are located and what user/group IDs
+# own them. For single-UID configuration use "static" userdb.
+#
+# <doc/wiki/UserDatabase.txt>
+
+#!include auth-deny.conf.ext
+#!include auth-master.conf.ext
+
+##### SmartMachine Begin.
+#!include auth-system.conf.ext
+!include auth-sql.conf.ext
+##### SmartMachine Close.
+#!include auth-ldap.conf.ext
+#!include auth-passwdfile.conf.ext
+#!include auth-checkpassword.conf.ext
+#!include auth-static.conf.ext