scalemail 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7afb9d2cc8a19c8510d3eee98915d87ee123e78d
+  data.tar.gz: ca48eb9cf1a05b3e5c859ba7f0bbc61d437bd307
+SHA512:
+  metadata.gz: 314c9eda18923f751d95619f438e6da41ef2d6f7075290197109769a992e308267aba05baf59153b889f543a1589eae799f15d8c23502a856a9a42dc264f7333
+  data.tar.gz: 46363add0a2fb8c4389a20a1e5e604236da497699cb847dbc759aebf5ab46141a770b40671ab6e45eb210f401fa30a9c08b065d9827e31445b8dbdb4c79b7f74

data/bin/scalemail

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'scalemail'
+Scalemail.new(ARGV)

data/lib/configurator.rb

@@ -0,0 +1,148 @@
+require 'yaml'
+require 'erb'
+require_relative 'hosts'
+
+module Configurator
+  class Config
+    attr_reader :loaded_configuration, :driver, :config_tree
+    @@known_drivers = ['amazonec2']
+    @@known_host_types = %w(generic swarm-master swarm-node)
+
+    def initialize(config_file_path)
+      abort("Docker Machine configuration file not found: #{config_file_path}") unless File.exist?(config_file_path)
+      abort("Can't read Docker Machine configuration file: #{config_file_path}") unless File.readable?(config_file_path)
+
+      template = ERB.new File.new(config_file_path).read
+      @loaded_configuration = YAML.load template.result(binding)
+
+      # Checking 'options' section from Docker Machine config file
+      @driver = get_option('driver')
+      abort(build_wrong_config_format_error('options => driver parameter not found')) unless @driver
+      abort(build_invalid_config_error("unknown driver 'aws'")) unless @@known_drivers.include?(@driver)
+      abort(build_invalid_config_error("driver set as 'amazonec2', but 'aws-credentials-profile' parameter not set")) \
+        unless get_option('aws-credentials-profile')
+
+      # Loading 'hosts' section from Docker Machine config file
+      hosts = @loaded_configuration['hosts']
+      abort(build_invalid_config_error('hosts are not defined')) unless hosts
+
+      hosts.each do |host|
+        host_name = host.first.to_s
+        host_config = get_host_config(host_name)
+        host_type = host_config['host-type']
+
+        # Checking 'host' section from Docker Machine config file
+        abort(build_invalid_config_error("host-type is not defined for host '#{host_name}'")) unless host_type
+        abort(build_invalid_config_error("unknown host-type '#{host_type}' for host '#{host_name}'")) \
+          unless @@known_host_types.include?(host_type)
+        abort(build_invalid_config_error("hosts-amount not defined for host '#{host_name}'")) \
+          unless host_config['hosts-amount']
+
+        is_hosts_amount_int = Integer(host_config['hosts-amount']) rescue false
+        abort(build_invalid_config_error("hosts-amount must be integer. Host: '#{host_name}'")) \
+          unless is_hosts_amount_int
+
+        engine_opts = host_config['engine-opts']
+        abort(build_invalid_config_error("engine-opts must be an array in yml notation. Host: '#{host_name}'")) \
+          unless engine_opts.nil? || engine_opts.kind_of?(Array)
+
+        commands_to_execute = host_config['commands-to-execute']
+        abort(build_invalid_config_error("commands-to-execute must be an array in yml notation. Host: '#{host_name}'")) \
+          unless commands_to_execute.nil? || commands_to_execute.kind_of?(Array)
+
+        engine_install_url = host_config['engine-install-url']
+        abort(build_invalid_config_error("engine-install-url must be a string. Host: '#{host_name}'")) \
+          unless engine_install_url.nil? || engine_install_url.kind_of?(String)
+
+        # Checking 'host' section from Docker Machine config file for 'swarm-node' host type
+        if (host_type == 'swarm-node')
+          abort(build_invalid_config_error("host-type is swarm-node, but swarm-discovery is not set. Host: '#{host_name}'")) \
+          unless host_config['swarm-discovery']
+          abort(build_invalid_config_error("swarm-discovery must be a string. Host: '#{host_name}'")) \
+          unless host_config['swarm-discovery'].kind_of?(String)
+        end
+
+        # Checking 'host' section from Docker Machine config file for 'amazonec2' driver
+        validate_amazonec2_options(host_config, host_name)
+
+        @config_tree ||= {}
+        hosts_amount = host_config['hosts-amount']
+        if hosts_amount > 1
+          for i in 1..hosts_amount
+            create_host_in_config_tree(host_name+"-#{i}", host_config, host_type, @driver)
+          end
+        else
+          create_host_in_config_tree(host_name, host_config, host_type, @driver)
+        end
+      end
+    end
+
+    def validate_amazonec2_options(host_config, host_name)
+      abort(build_invalid_config_error("amazonec2-region not defined for host '#{host_name}'")) unless host_config['amazonec2-region']
+
+      amazonec2_region = host_config['amazonec2-region']
+      abort(build_invalid_config_error("amazonec2-region must be a string. Host: '#{host_name}'")) \
+          unless amazonec2_region.kind_of?(String)
+
+      abort(build_invalid_config_error("amazonec2-zone not defined for host '#{host_name}'")) unless host_config['amazonec2-zone']
+
+      amazonec2_zone = host_config['amazonec2-zone']
+      abort(build_invalid_config_error("amazonec2-zone must be a string. Host: '#{host_name}'")) \
+          unless amazonec2_zone.kind_of?(String)
+
+      abort(build_invalid_config_error("amazonec2-vpc-id not defined for host '#{host_name}'")) unless host_config['amazonec2-vpc-id']
+
+      amazonec2_vpc_id = host_config['amazonec2-vpc-id']
+      abort(build_invalid_config_error("amazonec2-vpc-id must be a string. Host: '#{host_name}'")) \
+          unless amazonec2_vpc_id.kind_of?(String)
+
+      abort(build_invalid_config_error("amazonec2-subnet-id not defined for host '#{host_name}'")) unless host_config['amazonec2-subnet-id']
+
+      amazonec2_subnet_id = host_config['amazonec2-subnet-id']
+      abort(build_invalid_config_error("amazonec2-subnet-id must be a string. Host: '#{host_name}'")) \
+          unless amazonec2_subnet_id.kind_of?(String)
+
+      abort(build_invalid_config_error("amazonec2-security-group not defined for host '#{host_name}'")) unless host_config['amazonec2-security-group']
+
+      amazonec2_security_group = host_config['amazonec2-security-group']
+      abort(build_invalid_config_error("amazonec2-security-group must be a string. Host: '#{host_name}'")) \
+          unless amazonec2_security_group.kind_of?(String)
+
+      abort(build_invalid_config_error("amazonec2-instance-type not defined for host '#{host_name}'")) unless host_config['amazonec2-instance-type']
+
+      amazonec2_instance_type = host_config['amazonec2-instance-type']
+      abort(build_invalid_config_error("amazonec2-instance-type must be a string. Host: '#{host_name}'")) \
+          unless amazonec2_instance_type.kind_of?(String)
+
+      amazonec2_ami = host_config['amazonec2-ami']
+      abort(build_invalid_config_error("amazonec2-ami must be a string. Host: '#{host_name}'")) \
+          unless amazonec2_ami.nil? || amazonec2_ami.kind_of?(String)
+    end
+
+    private
+    def get_option(key)
+      options_hash = @loaded_configuration['options']
+      options_hash ? options_hash[key] : nil
+    end
+
+    def build_wrong_config_format_error(reason)
+      "Wrong Docker Machine config format: #{reason}."
+    end
+
+    def build_invalid_config_error(reason)
+      "Invalid Docker Machine config: #{reason}."
+    end
+
+    def get_host_config(host_node_name)
+      hosts_hash = @loaded_configuration['hosts']
+      hosts_hash ? hosts_hash[host_node_name] : nil
+    end
+
+    def create_host_in_config_tree(host_name, host_config, host_type, driver)
+      case driver
+        when 'amazonec2'
+          @config_tree[host_name] = Hosts::AmazonHost.new(host_config, host_type)
+      end
+    end
+  end
+end

data/lib/entry.rb

@@ -0,0 +1,4 @@
+require_relative './scalemail'
+
+Scalemail.new(ARGV)
+# raise Exception.new("File not found #{ARGV[1]}") unless File.exist?(ARGV[0])

data/lib/host_types.rb

@@ -0,0 +1,7 @@
+class Host
+  attr_reader :engine_install_url, :engine_opts, :engine_insecure_registries, \
+  :engine_registy_mirrors, :engine_labels, :engine_storage_driver, :engine_env_vars
+
+  def initialize(config)
+  end
+end

data/lib/hosts.rb

@@ -0,0 +1,48 @@
+require_relative 'configurator'
+module Hosts
+  class BasicHost
+    attr_reader :creation_string
+    # , :engine_insecure_registries, \
+    # :engine_registy_mirrors, :engine_labels, :engine_storage_driver, :engine_env_vars
+
+    def initialize(config, host_type)
+      @creation_string ||= []
+      engine_install_url = config['engine-install-url']
+      engine_opts = config['engine-opts']
+
+      @creation_string.push("--engine-install-url '#{engine_install_url}'") if engine_install_url
+      if engine_opts
+        engine_opts.each {|opt| @creation_string.push("--engine-opt='"+opt+"'")}
+      end
+
+      add_swarm_node_fields config, host_type if host_type=='swarm-node' || host_type=='swarm-master'
+    end
+
+    def add_swarm_node_fields(config, host_type)
+      swarm_discovery = config['swarm-discovery']
+      @creation_string.push('--swarm') if host_type=='swarm-node'
+      @creation_string.push('--swarm-master') if host_type=='swarm-master'
+      @creation_string.push("--swarm-discovery='#{swarm_discovery}'")
+    end
+
+    def get_creation_string
+      @creation_string.join(' ')
+    end
+  end
+
+  class AmazonHost < BasicHost
+    def initialize(config, host_type)
+      super(config, host_type)
+      @creation_string.unshift('--driver amazonec2')
+      @creation_string.push("--amazonec2-region #{config['amazonec2-region']}")
+      @creation_string.push("--amazonec2-zone #{config['amazonec2-zone']}")
+      @creation_string.push("--amazonec2-vpc-id #{config['amazonec2-vpc-id']}")
+      @creation_string.push("--amazonec2-subnet-id #{config['amazonec2-subnet-id']}")
+      @creation_string.push("--amazonec2-security-group #{config['amazonec2-security-group']}")
+      @creation_string.push("--amazonec2-instance-type #{config['amazonec2-instance-type']}")
+      @creation_string.push("--amazonec2-ami #{config['amazonec2-ami']}")
+    end
+  end
+
+end
+

data/lib/provisioner.rb

@@ -0,0 +1,132 @@
+require 'open4'
+require 'yaml'
+require_relative 'configurator'
+
+class Provisioner
+  attr_reader :config_to_provision, :hosts, :config
+  def initialize(config)
+    @config = config
+    @config_to_provision = @config.config_tree
+    @hosts = {}
+  end
+
+  def provision_hosts
+    threads = []
+    @config_to_provision.each do |host|
+      host_name = host[0]
+      host_config = host[1]
+
+      host_node_name = host_name.sub /\-\d*$/, ''
+      host_config_node = @config.loaded_configuration['hosts'][host_node_name]
+      placeholders = host_config_node['placeholders']
+      if placeholders
+        placeholders.each do |key, value|
+          if key=='host_ip' && @hosts[value] && @hosts[value][:ip]
+            replacement = @hosts[value][:ip]
+          else
+            abort("Cant find IP for host '#{value}' to use in placeholder")
+          end
+          host_config.creation_string.each do |parameter|
+            if parameter.respond_to?(:each)
+              parameter.each do |subparam|
+                if subparam.respond_to?(:sub!)
+                  subparam.sub! "$#{key}$", replacement
+                end
+              end
+            else
+              parameter.sub! "$#{key}$", replacement
+            end
+          end
+        end
+      end
+      print_provisioning_plan host_name
+      synchron = @config.loaded_configuration['hosts'][host_node_name]['synchronous']
+      if synchron == true
+        create_host(host_name, host_config).join()
+        next
+      end
+      threads.push(create_host host_name, host_config)
+    end
+    threads.each {|thread| thread.join}
+    puts @hosts.to_s
+  end
+
+  def create_host(host_name, host_config)
+    host_creation_string = host_config.get_creation_string
+    thread = Thread.new do
+      process = run_os_command(host_name, "docker-machine create #{host_creation_string} #{host_name}")
+      if process[:process].exitstatus != 0
+        abort("#{host_name} creation failed. Check log for details.")
+      end
+      @hosts[host_name] = {:ip => get_host_ip(host_name)}
+      commands = @config.loaded_configuration['hosts'][host_name]['commands-to-execute']
+      if commands
+        commands.each do |command|
+          process = run_os_command(host_name, "docker-machine ssh #{host_name} #{command}")
+          if process[:process].exitstatus != 0
+            abort("#{host_name} provisioning failed. Check log for details.")
+          end
+        end
+      end
+    end
+    thread
+  end
+
+  def formatted_log_to_stdout(prefix, string, timestamp=true)
+    time = Time.new
+    time_string = time.strftime("%H:%M:%S")
+    puts "#{time_string if timestamp} #{prefix} #{string}"
+  end
+
+  def formatted_log_to_stderr(prefix, string, timestamp=true)
+    time = Time.new
+    time_string = time.strftime("%H:%M:%S")
+    STDERR.puts "#{time_string if timestamp} #{prefix} #{string}"
+  end
+
+  def get_host_ip(host_name)
+    while true
+      process = run_os_command host_name, "docker-machine ip #{host_name}"
+      if process[:process].exitstatus == 0
+        ip = process[:stdout][/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/,0]
+        if !ip.nil?
+          formatted_log_to_stdout("OUT [#{host_name}]", "Got IP: #{ip}")
+          return ip
+        end
+      else
+        formatted_log_to_stdout("OUT [#{host_name}]", 'Waiting 10 seconds for IP.')
+        sleep 10
+      end
+    end
+  end
+
+  def print_provisioning_plan(host_name)
+    host_config = @config_to_provision[host_name]
+    formatted_log_to_stdout "PLAN [#{host_name}]", "docker machine create #{host_config.get_creation_string} #{host_name}"
+  end
+
+  def run_os_command(host_name, command, stream_stdout=true, stream_stderr=true)
+    process_hash = {}
+    process = Open4::popen4('sh') do |pid, stdin, stdout, stderr|
+      stdin.puts command
+      stdin.close
+      if stream_stdout
+        stdout.each do |line|
+          formatted_log_to_stdout("OUT [#{host_name}]", line)
+          process_hash[:stdout] ||= ''
+          process_hash[:stdout] += line
+        end
+      end
+      if stream_stderr
+        stderr.each do |line|
+          formatted_log_to_stderr("ERROR [#{host_name}]", line)
+          process_hash[:stderr] ||= ''
+          process_hash[:stderr] += line
+        end
+      end
+      process_hash[:pid] = pid
+    end
+    process_hash[:process] = process
+    process_hash
+  end
+end

data/lib/scalemail.rb

@@ -0,0 +1,53 @@
+require 'optparse'
+require 'ostruct'
+require_relative 'configurator'
+require_relative 'provisioner'
+
+class Scalemail
+  attr_reader :options, :configuration
+
+  def initialize(args)
+    @options = OpenStruct.new
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = 'Usage: scalemail -m DOCKER_MACHINE_CONFIG_PATH [options]'
+      opts.separator ''
+      opts.separator 'Specific options:'
+
+      if args.empty?
+        args << '-h'
+      end
+
+      opts.on('-m', '--docker-machine-config FILE',  'Docker Machine config file to load') { |cfile| @options.docker_machine_config_file = cfile }
+      # opts.on('-c FILE', 'Docker Compose config file to load') { |cfile| @options.docker_compose_config_file = cfile }
+
+      opts.on_tail('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end
+
+    opt_parser.parse!(args)
+    config = get_config
+    provision config
+  end
+
+  def get_config
+    if @configuration.nil?
+      config = configure @options.docker_machine_config_file
+      return config
+    else
+      return @configuration
+    end
+  end
+
+  def configure(config_file)
+    configuration = Configurator::Config.new(config_file)
+    configuration
+  end
+
+  def provision(config_tree)
+    provisioner = Provisioner.new(config_tree)
+    provisioner.provision_hosts
+  end
+
+end

data/lib/test.rb

@@ -0,0 +1,20 @@
+require 'yaml'
+require 'open4'
+include Open4
+
+stdin = '42'
+stdout = ''
+stderr = ''
+
+t = bg 'ruby -e"sleep 4; puts ARGF.read"', 0=>stdin, 1=>stdout, 2=>stderr
+
+waiter = Thread.new{ y t.pid => t.exitstatus } # t.exitstatus is a blocking call!
+
+while((status = t.status))
+  y "status" => status
+  sleep 1
+end
+
+waiter.join
+
+y "stdout" => stdout

data/spec/lib/cli_spec.rb

@@ -0,0 +1,39 @@
+require 'rspec'
+
+describe 'CLI Interaction' do
+  before do
+    $stdout = StringIO.new
+    $stderr = StringIO.new
+    @banner_message = "Usage: scalemail -m DOCKER_MACHINE_CONFIG_PATH [options]\n"\
+                     "\n"\
+                     "Specific options:\n"\
+                     "    -m, --docker-machine-config FILE Docker Machine config file to load\n"\
+                     "    -h, --help                       Show this message\n"
+  end
+  after(:all) do
+    $stdout = STDOUT
+    $stderr = STDERR
+  end
+
+    it 'should show usage banner if -h option used' do
+      expect {Scalemail.new(%w(-h))}.to raise_error(SystemExit)
+      out = $stdout.string
+      expect(out).to eq(@banner_message)
+    end
+    it 'should show usage banner if --help option used' do
+      expect {Scalemail.new(%w(--help))}.to raise_error(SystemExit)
+      out = $stdout.string
+      expect(out).to eq(@banner_message)
+    end
+    it 'should show usage banner if no options provided' do
+      expect {Scalemail.new([])}.to raise_error(SystemExit)
+      out = $stdout.string
+      expect(out).to eq(@banner_message)
+    end
+    it 'should fail with error if DockerMachine config file not specified' do
+      #TODO
+    end
+    it 'should fail with error if DockerCompose file not specified' do
+      #TODO
+    end
+end

data/spec/lib/configuration_loading_spec.rb

@@ -0,0 +1,408 @@
+require 'rspec'
+require 'tempfile'
+
+def write_to_tmp_file(file_content)
+  tmpfile = Tempfile.new('rspec_tmp')
+  tmpfile.write(file_content)
+  tmpfile.close
+  tmpfile
+end
+
+describe 'Configuration Loading' do
+
+  before do
+    $stdout = StringIO.new
+    $stderr = StringIO.new
+    @config_tree_test_file = File.dirname(__FILE__)+ '/../resources/host-tree-generation-test.yml'
+  end
+  after(:all) do
+    $stdout = STDOUT
+    $stderr = STDERR
+  end
+
+  it 'should fail with error if DockerMachine config file not found' do
+    expect { Configurator::Config.new('fakefile') }.to raise_error(SystemExit, /Docker Machine configuration file not found: fakefile/)
+  end
+  it 'should fail with error if DockerMachine config file cannot be read' do
+    tmpfile = Tempfile.new('fakefile')
+    tmpfile.chmod(000)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, /Can't read Docker Machine configuration file: #{tmpfile.path}/)
+  end
+  it 'should fail with error if DockerCompose file not found' do
+    #TODO
+  end
+  it 'should fail with error if DockerCompose file cannot be read' do
+    #TODO
+  end
+  it 'should fail with error if driver is unknown' do
+    file_content = "options:\n  driver: aws"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: unknown driver 'aws'./)
+  end
+  it 'should fail with error if driver is amazonec2 and aws-credentials-profile is nil' do
+    file_content = "options:\n  driver: amazonec2"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: driver set as 'amazonec2', but 'aws-credentials-profile' parameter not set./)
+  end
+  it 'should fail with error if hosts section is not defined' do
+    file_content = "options:\n"\
+                         "  driver: amazonec2\n" \
+                         "  aws-credentials-profile: default\n"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: hosts are not defined./)
+  end
+  it 'should fail with error if host-type is not defined in host node' do
+    file_content = "options:\n"\
+                         "  driver: amazonec2\n" \
+                         "  aws-credentials-profile: default\n"\
+                         "hosts:\n"\
+                         "  fail-host:\n"\
+                         "    aparam: something"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: host-type is not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if host type is unknown' do
+    file_content = "options:\n"\
+                         "  driver: amazonec2\n" \
+                         "  aws-credentials-profile: default\n"\
+                         "hosts:\n"\
+                         "  fail-host:\n"\
+                         "    host-type: something"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: unknown host-type 'something' for host 'fail-host'./)
+  end
+  it 'should fail with error if hosts-amount is not defined in host node' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: hosts-amount not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if hosts-amount is not integer' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: fail"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: hosts-amount must be integer. Host: 'fail-host'./)
+  end
+  it 'should fail with error if AWS region not defined' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-region not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if AWS region is not a string' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: \n"\
+                       "      - test"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-region must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if AWS availability zone not defined' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-zone not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if AWS availability zone is not a string' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: \n"\
+                       "      - test"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-zone must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if AWS VPC ID not defined' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-vpc-id not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if AWS VPC ID is not a string' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id:\n"\
+                       "      - test"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-vpc-id must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if AWS Subnet ID not defined' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id: vpc"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-subnet-id not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if AWS Subnet ID is not a string' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id: vpc\n"\
+                       "    amazonec2-subnet-id:\n"\
+                       "      - test"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-subnet-id must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if AWS Security Group not defined' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id: vpc\n"\
+                       "    amazonec2-subnet-id: subnet"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-security-group not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if AWS Security Group is not a string' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id: vpc\n"\
+                       "    amazonec2-subnet-id: subnet\n"\
+                       "    amazonec2-security-group:\n"\
+                       "      - test"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-security-group must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if AWS Instance Type not defined' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id: vpc\n"\
+                       "    amazonec2-subnet-id: subnet\n"\
+                       "    amazonec2-security-group: sg"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-instance-type not defined for host 'fail-host'./)
+  end
+  it 'should fail with error if AWS Instance Type is not a string' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id: vpc\n"\
+                       "    amazonec2-subnet-id: subnet\n"\
+                       "    amazonec2-security-group: sg\n"\
+                       "    amazonec2-instance-type:\n"\
+                       "      - test"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-instance-type must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if AWS AMI is not a string' do
+    file_content = "options:\n"\
+                       "  driver: amazonec2\n" \
+                       "  aws-credentials-profile: default\n"\
+                       "hosts:\n"\
+                       "  fail-host:\n"\
+                       "    host-type: generic\n"\
+                       "    hosts-amount: 1\n"\
+                       "    amazonec2-region: region\n"\
+                       "    amazonec2-zone: zone\n"\
+                       "    amazonec2-vpc-id: vpc\n"\
+                       "    amazonec2-subnet-id: subnet\n"\
+                       "    amazonec2-security-group: sg\n"\
+                       "    amazonec2-instance-type: t2.micro\n"\
+                       "    amazonec2-ami:\n"\
+                       "      - test"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+            /Invalid Docker Machine config: amazonec2-ami must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if driver is not defined in options section' do
+    file_content = "driver: aws"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Wrong Docker Machine config format: options => driver parameter not found./)
+  end
+  it 'should fail with error if engine-opts is not an array' do
+    file_content = "options:\n"\
+                     "  driver: amazonec2\n" \
+                     "  aws-credentials-profile: default\n"\
+                     "hosts:\n"\
+                     "  fail-host:\n"\
+                     "    host-type: generic\n"\
+                     "    hosts-amount: 2\n"\
+                     "    engine-opts:\n" \
+                     "      op1: sdf"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: engine-opts must be an array in yml notation. Host: 'fail-host'./)
+  end
+  it 'should fail with error if commands-to-execute is not an array' do
+    file_content = "options:\n"\
+                     "  driver: amazonec2\n" \
+                     "  aws-credentials-profile: default\n"\
+                     "hosts:\n"\
+                     "  fail-host:\n"\
+                     "    host-type: generic\n"\
+                     "    hosts-amount: 2\n"\
+                     "    commands-to-execute:\n" \
+                     "      op1: sdf"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: commands-to-execute must be an array in yml notation. Host: 'fail-host'./)
+  end
+  it 'should fail with error if engine-install-url is not a string' do
+    file_content = "options:\n"\
+                     "  driver: amazonec2\n" \
+                     "  aws-credentials-profile: default\n"\
+                     "hosts:\n"\
+                     "  fail-host:\n"\
+                     "    host-type: generic\n"\
+                     "    hosts-amount: 2\n"\
+                     "    engine-install-url:\n" \
+                     "      op1: sdf"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: engine-install-url must be a string. Host: 'fail-host'./)
+  end
+  it 'should fail with error if host-type is "swarm-node" and swarm-discovery is not defined' do
+    file_content = "options:\n"\
+                     "  driver: amazonec2\n" \
+                     "  aws-credentials-profile: default\n"\
+                     "hosts:\n"\
+                     "  fail-host:\n"\
+                     "    host-type: swarm-node\n"\
+                     "    hosts-amount: 2"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: host-type is swarm-node, but swarm-discovery is not set. Host: 'fail-host'./)
+  end
+  it 'should fail with error if host-type is "swarm-node" and swarm-discovery is not a string' do
+    file_content = "options:\n"\
+                     "  driver: amazonec2\n" \
+                     "  aws-credentials-profile: default\n"\
+                     "hosts:\n"\
+                     "  fail-host:\n"\
+                     "    host-type: swarm-node\n"\
+                     "    hosts-amount: 2\n"\
+                     "    swarm-discovery:\n" \
+                     "      op1: sdf"
+    tmpfile = write_to_tmp_file(file_content)
+    expect { Configurator::Config.new(tmpfile.path) }.to raise_error(SystemExit, \
+          /Invalid Docker Machine config: swarm-discovery must be a string. Host: 'fail-host'./)
+  end
+  it 'should return proper creation string for amazonec2 generic host type' do
+    test_creation_string = "--driver amazonec2 "\
+        "--engine-install-url 'http://test.com' --engine-opt='opt1' --engine-opt='opt2' "\
+        "--amazonec2-region region --amazonec2-zone zone --amazonec2-vpc-id vpc "\
+        "--amazonec2-subnet-id subnet --amazonec2-security-group sg --amazonec2-instance-type it"
+    config = Configurator::Config.new(@config_tree_test_file)
+    expect(config.config_tree['generic-host'].get_creation_string).to eq(test_creation_string)
+  end
+  it 'should return proper creation string for swarm-node host type' do
+    test_creation_string = "--driver amazonec2 "\
+        "--engine-install-url 'http://test.com' --engine-opt='opt1' --engine-opt='opt2' "\
+        "--swarm --swarm-discovery='something_somewhere' "\
+        "--amazonec2-region region --amazonec2-zone zone --amazonec2-vpc-id vpc "\
+        "--amazonec2-subnet-id subnet --amazonec2-security-group sg --amazonec2-instance-type it"
+    config = Configurator::Config.new(@config_tree_test_file)
+    expect(config.config_tree['swarm-node'].get_creation_string).to eq(test_creation_string)
+  end
+  it 'should return proper creation string for swarm-master host type' do
+    test_creation_string = "--driver amazonec2 "\
+        "--engine-install-url 'http://test.com' --engine-opt='opt1' --engine-opt='opt2' "\
+        "--swarm-master --swarm-discovery='something_somewhere' "\
+        "--amazonec2-region region --amazonec2-zone zone --amazonec2-vpc-id vpc "\
+        "--amazonec2-subnet-id subnet --amazonec2-security-group sg --amazonec2-instance-type it"
+    config = Configurator::Config.new(@config_tree_test_file)
+    expect(config.config_tree['swarm-master'].get_creation_string).to eq(test_creation_string)
+  end
+end

data/spec/lib/provisioner_spec.rb

@@ -0,0 +1,25 @@
+require 'rspec'
+require_relative '../../lib/provisioner'
+
+describe 'CLI Interaction' do
+  before do
+    $stdout = StringIO.new
+    $stderr = StringIO.new
+  end
+  after(:all) do
+    $stdout = STDOUT
+    $stderr = STDERR
+  end
+
+  it 'should print timestamped message in STDOUT' do
+    # TODO: fix
+    # Provisioner.new(nil).formatted_log_to_stdout('prefix', 'string')
+    # expect($stdout.string).to match(/\d{2}:\d{2} prefix string/)
+  end
+  it 'should print message without timestamp in STDOUT' do
+    #TODO: fix
+    #Provisioner.new(nil).formatted_log_to_stdout('prefix', 'string', false)
+    #expect($stdout.string).to match(/prefix string/)
+  end
+
+end

data/spec/lib/scalemail_spec.rb

@@ -0,0 +1,34 @@
+require 'rspec'
+require_relative '../../lib/scalemail'
+require_relative '../spec_helper'
+
+
+describe Scalemail do
+
+  #
+  # it 'raises error if configuration file cannot be readed' do
+  #   begin
+  #     Scalemail.new(%w(-m fakefile))
+  #   rescue SystemExit
+  #     expect($stderr.string).to match("Can't read Docker Machine configuration file: fakefile")
+  #   end
+  # end
+  # it 'shows usage banner if no options provided' do
+  #
+  #
+  # end
+  # it 'shows usage banner if -h key provided' do
+  #   begin
+  #     Scalemail.new(%w(-h))
+  #   rescue SystemExit
+  #     expect($stdout.string).to match(@banner_message)
+  #   end
+  # end
+  # it 'shows usage banner if --help key provided' do
+  #   begin
+  #     Scalemail.new(%w(--help))
+  #   rescue SystemExit
+  #     expect($stdout.string).to match(@banner_message)
+  #   end
+  # end
+end

data/spec/resources/host-tree-generation-test.yml

@@ -0,0 +1,62 @@
+options:
+  driver: amazonec2
+  aws-credentials-profile: default
+hosts:
+  generic-host:
+    host-type: generic
+    hosts-amount: 1
+    engine-install-url: http://test.com
+    engine-opts:
+      - opt1
+      - opt2
+    amazonec2-region: region
+    amazonec2-zone: zone
+    amazonec2-vpc-id: vpc
+    amazonec2-subnet-id: subnet
+    amazonec2-security-group: sg
+    amazonec2-instance-type: it
+
+  swarm-node:
+    host-type: swarm-node
+    hosts-amount: 1
+    engine-install-url: http://test.com
+    engine-opts:
+      - opt1
+      - opt2
+    swarm-discovery: something_somewhere
+    amazonec2-region: region
+    amazonec2-zone: zone
+    amazonec2-vpc-id: vpc
+    amazonec2-subnet-id: subnet
+    amazonec2-security-group: sg
+    amazonec2-instance-type: it
+
+  swarm-node-multiple:
+    host-type: swarm-node
+    hosts-amount: 3
+    engine-install-url: http://test.com
+    engine-opts:
+      - opt1
+      - opt2
+    swarm-discovery: something_somewhere
+    amazonec2-region: region
+    amazonec2-zone: zone
+    amazonec2-vpc-id: vpc
+    amazonec2-subnet-id: subnet
+    amazonec2-security-group: sg
+    amazonec2-instance-type: it
+
+  swarm-master:
+    host-type: swarm-master
+    hosts-amount: 1
+    engine-install-url: http://test.com
+    engine-opts:
+      - opt1
+      - opt2
+    swarm-discovery: something_somewhere
+    amazonec2-region: region
+    amazonec2-zone: zone
+    amazonec2-vpc-id: vpc
+    amazonec2-subnet-id: subnet
+    amazonec2-security-group: sg
+    amazonec2-instance-type: it

data/spec/spec_helper.rb

@@ -0,0 +1,97 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+=end
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+=begin
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: scalemail
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ievgen Iezhachenko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-13 00:00:00.000000000 Z
+dependencies: []
+description: Framework for microservises scaling using Docker and cloud platforms.
+email: ievgen.iezhachenko@gmail.com
+executables:
+- scalemail
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/scalemail
+- lib/configurator.rb
+- lib/entry.rb
+- lib/host_types.rb
+- lib/hosts.rb
+- lib/provisioner.rb
+- lib/scalemail.rb
+- lib/test.rb
+- spec/lib/cli_spec.rb
+- spec/lib/configuration_loading_spec.rb
+- spec/lib/provisioner_spec.rb
+- spec/lib/scalemail_spec.rb
+- spec/resources/host-tree-generation-test.yml
+- spec/spec_helper.rb
+homepage: https://github.com/iiezhachenko/scalemail
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Application scaling framework
+test_files: []
+has_rdoc: