orchestration 0.3.3 → 0.3.4

data/lib/orchestration/install_generator.rb

@@ -10,14 +10,15 @@ module Orchestration
     def initialize(*_args)
       super
       @env = Environment.new
-      @terminal ||= Terminal.new
       @settings = Settings.new(@env.orchestration_configuration_path)
+      @terminal = Terminal.new(@settings)
+      @docker_compose = DockerCompose::InstallGenerator.new(@env, @terminal)
     end
 
     def orchestration_configuration
       path = @env.orchestration_configuration_path
-      ask_setting('docker.username')
-      ask_setting('docker.repository', @env.default_application_name)
+      @terminal.ask_setting('docker.username')
+      @terminal.ask_setting('docker.repository', @env.default_app_name)
       relpath = relative_path(path)
       return @terminal.write(:create, relpath) unless @settings.exist?
       return @terminal.write(:update, relpath) if @settings.dirty?
@@ -25,15 +26,16 @@ module Orchestration
       @terminal.write(:skip, relpath)
     end
 
-    def makefile
-      environment = { env: @env, wait_commands: wait_commands }
-      content = template('Makefile', environment)
+    def orchestration_makefile
+      content = template('Makefile', makefile_environment)
       path = @env.orchestration_root.join('Makefile')
       path.exist? ? update_file(path, content) : create_file(path, content)
-      inject_if_missing(
-        @env.root.join('Makefile'),
-        'include orchestration/Makefile'
-      )
+    end
+
+    def application_makefile
+      path = @env.root.join('Makefile')
+      simple_copy('application.mk', path) unless File.exist?(path)
+      inject_if_missing(path, 'include orchestration/Makefile')
     end
 
     def dockerfile
@@ -45,7 +47,7 @@ module Orchestration
       )
     end
 
-    def entrypoint
+    def entrypoint_sh
       content = template('entrypoint.sh')
       path = orchestration_dir.join('entrypoint.sh')
       create_file(path, content, overwrite: false)
@@ -54,21 +56,14 @@ module Orchestration
 
     def gitignore
       path = @env.root.join('.gitignore')
-      entries = %w[.build/ Gemfile Gemfile.lock *.gemspec].map do |entry|
+      globs = %w[.build/ .deploy/ Gemfile Gemfile.lock]
+      entries = ['deploy.tar'] + globs.map do |entry|
         "#{@env.orchestration_dir_name}/#{entry}"
       end
 
       ensure_lines_in_file(path, entries)
     end
 
-    def docker_compose
-      path = @env.orchestration_root.join('docker-compose.yml')
-      return if File.exist?(path)
-
-      docker_compose = DockerCompose::Services.new(@env, service_configurations)
-      create_file(path, docker_compose.structure.to_yaml)
-    end
-
     def unicorn
       content = template('unicorn.rb')
       path = @env.root.join('config', 'unicorn.rb')
@@ -79,46 +74,38 @@ module Orchestration
       simple_copy('yaml.bash', @env.orchestration_root.join('yaml.bash'))
     end
 
-    private
-
-    def t(key)
-      I18n.t("orchestration.#{key}")
+    def env
+      simple_copy('env', @env.root.join('.env'), overwrite: false)
     end
 
-    def service_configurations
-      Hash[
-        %i[application database mongo rabbitmq nginx_proxy].map do |key|
-          [key, configuration(key)]
-        end
-      ]
+    def deploy_mk
+      simple_copy('deploy.mk', @env.orchestration_root.join('deploy.mk'))
     end
 
-    def configuration(service)
-      {
-        application: Services::Application::Configuration,
-        database: Services::Database::Configuration,
-        mongo: Services::Mongo::Configuration,
-        rabbitmq: Services::RabbitMQ::Configuration,
-        nginx_proxy: Services::NginxProxy::Configuration
-      }.fetch(service).new(@env)
+    def docker_compose
+      @docker_compose.docker_compose_yml
+      @docker_compose.docker_compose_test_yml
+      @docker_compose.docker_compose_development_yml
+      @docker_compose.docker_compose_production_yml
+      @docker_compose.docker_compose_override_yml
     end
 
-    def wait_commands
-      [
-        configuration(:database).settings.nil? ? nil : 'wait-database',
-        configuration(:mongo).settings.nil? ? nil : 'wait-mongo',
-        configuration(:rabbitmq).settings.nil? ? nil : 'wait-rabbitmq',
-        'wait-nginx-proxy',
-        'wait-application'
-      ].compact.join(' ')
+    private
+
+    def t(key)
+      I18n.t("orchestration.#{key}")
     end
 
-    def ask_setting(setting, default = nil)
-      return unless @settings.get(setting).nil?
+    def makefile_environment
+      { env: @env, wait_commands: wait_commands }
+    end
 
-      @terminal.write(:setup, t("settings.#{setting}.description"))
-      prompt = t("settings.#{setting}.prompt")
-      @settings.set(setting, @terminal.read(prompt, default))
+    def wait_commands
+      %i[test development production].map do |environment|
+        @docker_compose.enabled_services(environment).map do |service|
+          "wait-#{service}"
+        end
+      end.flatten.uniq
     end
   end
 end

data/lib/orchestration/services/{application → app}/configuration.rb

@@ -2,27 +2,27 @@
 
 module Orchestration
   module Services
-    module Application
+    module App
       class Configuration
         include ConfigurationBase
 
-        self.service_name = 'application'
+        self.service_name = 'app'
 
         def initialize(env, service_name = nil)
           super
-          @settings = {} # Included for interface consistency; currently unused.
+          @settings = {}
         end
 
         def docker_username
           @env.settings.get('docker.username')
         end
 
-        def application_name
+        def app_name
           @env.settings.get('docker.repository')
         end
 
         def friendly_config
-          "[#{application_name}] #{host}:#{local_port}"
+          "[#{app_name}] #{host}:#{local_port}"
         end
 
         def database_settings

data/lib/orchestration/services/{application → app}/healthcheck.rb

@@ -2,7 +2,7 @@
 
 module Orchestration
   module Services
-    module Application
+    module App
       class Healthcheck
         include HealthcheckBase
 
@@ -14,14 +14,14 @@ module Orchestration
         end
 
         def connection_errors
-          [Errno::ECONNREFUSED, ApplicationConnectionError]
+          [Errno::ECONNREFUSED, AppConnectionError]
         end
 
         private
 
         def connection_error(code)
-          raise ApplicationConnectionError,
-                I18n.t('orchestration.application.connection_error', code: code)
+          raise AppConnectionError,
+                I18n.t('orchestration.app.connection_error', code: code)
         end
 
         def connection_error?(code)

data/lib/orchestration/services/app.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Orchestration
+  module Services
+    module App
+    end
+  end
+end
+
+require 'net/http'
+
+require 'orchestration/services/app/configuration'
+require 'orchestration/services/app/healthcheck'

data/lib/orchestration/services/configuration_base.rb

@@ -32,7 +32,9 @@ module Orchestration
       end
 
       def local_port
-        key = @service_name == 'application' ? 'nginx-proxy' : @service_name
+        key = @service_name == 'app' ? 'nginx_proxy' : @service_name
+
+        return ENV.fetch('LISTEN_PORT', '3000').to_i if key == 'nginx_proxy'
 
         @env.docker_compose_config
             .fetch('services')

data/lib/orchestration/services/database/adapters/mysql2.rb

@@ -5,6 +5,14 @@ module Orchestration
     module Database
       module Adapters
         class Mysql2
+          def name
+            'mysql2'
+          end
+
+          def image
+            'library/mysql'
+          end
+
           def credentials
             {
               'username' => 'root',
@@ -20,6 +28,17 @@ module Orchestration
           def default_port
             3306
           end
+
+          def environment
+            {
+              'MYSQL_ROOT_PASSWORD' => 'password',
+              'MYSQL_TCP_PORT' => DockerCompose::DatabaseService::PORT.to_s
+            }
+          end
+
+          def data_dir
+            '/var/lib/mysql'
+          end
         end
       end
     end

data/lib/orchestration/services/database/adapters/postgresql.rb

@@ -5,6 +5,14 @@ module Orchestration
     module Database
       module Adapters
         class Postgresql
+          def name
+            'postgresql'
+          end
+
+          def image
+            'library/postgres'
+          end
+
           def credentials
             {
               'username' => 'postgres',
@@ -20,6 +28,18 @@ module Orchestration
           def default_port
             5432
           end
+
+          def environment
+            {
+              'PGPORT' => DockerCompose::DatabaseService::PORT.to_s,
+              'POSTGRES_PASSWORD' => 'password',
+              'PGDATA' => data_dir
+            }
+          end
+
+          def data_dir
+            '/var/pgdata'
+          end
         end
       end
     end

data/lib/orchestration/services/database/adapters/sqlite3.rb

@@ -5,6 +5,10 @@ module Orchestration
     module Database
       module Adapters
         class Sqlite3
+          def name
+            'sqlite3'
+          end
+
           def credentials
             {
               'username' => '',

data/lib/orchestration/services/database/configuration.rb

@@ -22,24 +22,28 @@ module Orchestration
         end
 
         def friendly_config
-          adapter = @settings.fetch('adapter')
-          return "[#{adapter}]" if adapter == 'sqlite3'
+          return "[#{@adapter.name}]" if @adapter.name == 'sqlite3'
 
-          "[#{adapter}] #{host}:#{local_port}"
+          "[#{@adapter.name}] #{host}:#{local_port}"
         end
 
         private
 
         def setup
-          @adapter = adapter_object(base['adapter'])
-          @settings = base.merge(@adapter.credentials)
-                          .merge(
-                            'scheme' => base['adapter'],
-                            'port' => DockerCompose::DatabaseService::PORT
-                          )
+          @adapter = adapter_for(base['adapter'])
+          @settings = merged_settings
+          return if @adapter.name == 'sqlite3'
+          return unless %w[test development].include?(@env.environment)
+
           @settings.merge!('port' => local_port) if @env.docker_compose_config?
         end
 
+        def merged_settings
+          base.merge(@adapter.credentials)
+              .merge('scheme' => base['adapter'],
+                     'port' => DockerCompose::DatabaseService::PORT)
+        end
+
         def parse(content)
           yaml(erb(content))
         end
@@ -52,7 +56,7 @@ module Orchestration
           YAML.safe_load(content, [], [], true) # true: Allow aliases
         end
 
-        def adapter_object(name)
+        def adapter_for(name)
           {
             'mysql2' => adapters::Mysql2,
             'postgresql' => adapters::Postgresql,
@@ -61,7 +65,13 @@ module Orchestration
         end
 
         def environment
-          @environments[@env.environment]
+          @environments.fetch(@env.environment)
+        rescue KeyError
+          raise UnknownEnvironmentError,
+                I18n.t(
+                  'orchestration.database.unknown_environment',
+                  environment: @env.environment
+                )
         end
 
         def base
@@ -69,7 +79,7 @@ module Orchestration
         end
 
         def host
-          return nil if @adapter.is_a?(adapters::Sqlite3)
+          return nil if @adapter&.name == 'sqlite3'
 
           super
         end
@@ -79,7 +89,7 @@ module Orchestration
         end
 
         def default_port
-          return {} if @adapter.is_a?(adapters::Sqlite3)
+          return {} if @adapter.name == 'sqlite3'
 
           { 'port' => @adapter.default_port }
         end

data/lib/orchestration/services/healthcheck_base.rb

@@ -15,7 +15,7 @@ module Orchestration
           exit_on_error = options.fetch(:exit_on_error, true)
           options.delete(:exit_on_error)
           env ||= Environment.new
-          terminal ||= Terminal.new
+          terminal ||= Terminal.new(env.settings)
           name = options.delete(:service_name)
           check = ServiceCheck.new(new(env, name), terminal, options)
 

data/lib/orchestration/services/nginx_proxy/configuration.rb

@@ -6,10 +6,15 @@ module Orchestration
       class Configuration
         include ConfigurationBase
 
-        self.service_name = 'nginx-proxy'
+        self.service_name = 'nginx_proxy'
+
+        def initialize(env, service_name = nil)
+          super
+          @settings = {}
+        end
 
         def friendly_config
-          "[nginx-proxy] #{host}:#{local_port}"
+          "[nginx_proxy] #{host}:#{local_port}"
         end
       end
     end

data/lib/orchestration/services.rb

@@ -8,7 +8,7 @@ end
 require 'orchestration/services/configuration_base'
 require 'orchestration/services/healthcheck_base'
 
-require 'orchestration/services/application'
+require 'orchestration/services/app'
 require 'orchestration/services/database'
 require 'orchestration/services/listener'
 require 'orchestration/services/mongo'

data/lib/orchestration/templates/Makefile.erb

@@ -1,86 +1,145 @@
-.PHONY: start stop migrate docker build push start logs compose config <%= wait_commands %>
+.PHONY: start stop migrate bundle docker build push start logs compose config test-setup <%= wait_commands.join(' ') %>
 
-### Container management commands ###
+### Environment setup ###
+
+docker_username:=$(shell bash ./<%= env.orchestration_dir_name %>/yaml.bash docker_username)
+docker_repository:=$(shell bash ./<%= env.orchestration_dir_name %>/yaml.bash docker_repository)
+
+compose_base:=HOST_UID=$(shell id -u) \
+              DOCKER_USERNAME=${docker_username} \
+              DOCKER_REPOSITORY=${docker_repository} \
+              docker-compose \
+              -p ${docker_repository} \
+              -f <%= env.orchestration_dir_name %>/docker-compose.yml
+
+git_branch:=$(if $(branch),$(branch),$(shell git rev-parse --abbrev-ref HEAD))
+git_version:=$(shell git rev-parse --short --verify ${git_branch})
 
-DOCKER_USERNAME:=$(shell bash ./<%= env.orchestration_dir_name %>/yaml.bash docker_username)
-DOCKER_REPOSITORY:=$(shell bash ./<%= env.orchestration_dir_name %>/yaml.bash docker_repository)
-COMPOSE:=HOST_UID=$(shell id -u) DOCKER_USERNAME=${DOCKER_USERNAME} DOCKER_REPOSITORY=${DOCKER_REPOSITORY} docker-compose -p $(shell bash ./<%= env.orchestration_dir_name %>/yaml.bash docker_repository) -f orchestration/docker-compose.yml
+-include .env
+export
+
+ifneq (,$(RAILS_ENV))
+  env:=$(RAILS_ENV)
+else ifneq (,$(RACK_ENV))
+  env:=$(RACK_ENV)
+else ifeq (,$(env))
+  env:=development
+endif
+
+compose:=${compose_base} -f <%= env.orchestration_dir_name %>/docker-compose.${env}.yml -f <%= env.orchestration_dir_name %>/docker-compose.override.yml
+rake:=RACK_ENV=${env} RAILS_ENV=${env} bin/rake
+
+### Container management commands ###
 
 start:
 	@echo "Starting containers..."
-	@${COMPOSE} up -d --scale application=$${INSTANCES:-1}
-	@make wait
+ifeq (${env},$(filter ${env},test development))
+	@${compose} up -d
+else
+	@${compose} up -d --scale app=$${INSTANCES:-1}
+endif
+	@$(MAKE) wait
 
 stop:
 	@echo "Stopping containers..."
-	@${COMPOSE} down
+	@${compose} down
 	@echo "All containers stopped."
 
-start-application:
-	@echo "Starting containers..."
-	@${COMPOSE} up -d --scale application=$${INSTANCES:-1} nginx-proxy application
-	@make wait-nginx-proxy wait-application
-	@echo "Application started."
-
 logs:
-	@${COMPOSE} logs
+	@${compose} logs -f
 
 config:
-	@${COMPOSE} config
+	@${compose} config
 
 compose:
-	@${COMPOSE} $$cmd
+	@${compose} $$cmd
+
+test-setup:
+	@$(MAKE) start wait migrate env=test
+
+### Deployment utility commands ###
+
+bundle:
+	@echo 'Building deployment bundle...'
+	@rm -rf <%= env.orchestration_dir_name %>/.deploy/
+	@mkdir -p <%= env.orchestration_dir_name %>/.deploy/${docker_repository}/
+	@sed -e "s/%%VERSION%%/${git_version}/g" \
+             -e "s/%%REPOSITORY%%/${docker_repository}/g" \
+             -e "s/%%USERNAME%%/${docker_username}/g" \
+             <%= env.orchestration_dir_name %>/deploy.mk > \
+             <%= env.orchestration_dir_name %>/.deploy/${docker_repository}/Makefile
+	@cp <%= env.orchestration_dir_name %>/docker-compose.yml \
+            <%= env.orchestration_dir_name %>/docker-compose.production.yml \
+            <%= env.orchestration_dir_name %>/docker-compose.override.yml \
+            <%= env.orchestration_dir_name %>/.deploy/${docker_repository}/
+	@tar -C <%= env.orchestration_dir_name %>/.deploy -cf ./deploy.tar ./${docker_repository}
+	@echo 'Deployment bundle written to ./deploy.tar'
 
 ### Database utility commands ###
 
-migrate: wait-database
+migrate:
 	@echo "Running migrations..."
-	@${COMPOSE} run application bundle exec rake db:migrate
+ifeq (${env},$(filter ${env},test development))
+	@${rake} db:migrate
+else
+	@${compose} run --rm app bin/rake db:migrate RAILS_ENV=${env}
+endif
 	@echo "Migrations complete."
 
 ### Service healthcheck commands ###
 
-wait: <%= wait_commands %>
+wait: <%= wait_commands.join(' ') %>
 	@echo "All Containers ready."
 
-wait-application:
-	@bin/rake orchestration:application:wait
+## Test/development wait commands
 
 wait-database:
-	@bin/rake orchestration:database:wait
+ifeq (${env},$(filter ${env},test development))
+	@${rake} orchestration:database:wait
+endif
 
 wait-mongo:
-	@bin/rake orchestration:mongo:wait
-
-wait-nginx-proxy:
-	@bin/rake orchestration:nginx_proxy:wait
+ifeq (${env},$(filter ${env},test development))
+	@${rake} orchestration:mongo:wait
+endif
 
 wait-rabbitmq:
-	@bin/rake orchestration:rabbitmq:wait
+ifeq (${env},$(filter ${env},test development))
+	@${rake} orchestration:rabbitmq:wait
+endif
+
+## Production wait commands
+
+wait-nginx_proxy:
+ifneq (${env},$(filter ${env},test development))
+	@${rake} orchestration:nginx_proxy:wait LISTEN_PORT=${LISTEN_PORT}
+endif
+
+wait-app:
+ifneq (${env},$(filter ${env},test development))
+	@${rake} orchestration:app:wait LISTEN_PORT=${LISTEN_PORT}
+endif
 
 ### Docker build commands ###
 
 docker: build push
 
-GIT_BRANCH:=$(if $(BRANCH),$(BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
-
 build:
-	@echo "Preparing build from ${GIT_BRANCH}"
+	@echo "Preparing build from ${git_branch}"
 	@mkdir -p ./<%= env.orchestration_dir_name %>/.build
-	@git show ${GIT_BRANCH}:./Gemfile > ./<%= env.orchestration_dir_name %>/.build/Gemfile
-	@git show ${GIT_BRANCH}:./Gemfile.lock > ./<%= env.orchestration_dir_name %>/.build/Gemfile.lock
-	<% if defined?(Webpacker) %>@git show ${GIT_BRANCH}:./package.json > ./<%= env.orchestration_dir_name %>/.build/package.json<% end %>
-	<% if defined?(Webpacker) %>@git show ${GIT_BRANCH}:./yarn.lock > ./<%= env.orchestration_dir_name %>/.build/yarn.lock<% end %>
+	@git show ${git_branch}:./Gemfile > ./<%= env.orchestration_dir_name %>/.build/Gemfile
+	@git show ${git_branch}:./Gemfile.lock > ./<%= env.orchestration_dir_name %>/.build/Gemfile.lock
+	<% if defined?(Webpacker) %>@git show ${git_branch}:./package.json > ./<%= env.orchestration_dir_name %>/.build/package.json<% end %>
+	<% if defined?(Webpacker) %>@git show ${git_branch}:./yarn.lock > ./<%= env.orchestration_dir_name %>/.build/yarn.lock<% end %>
 	@echo "Building..."
-	@git archive --format tar.gz -o ./<%= env.orchestration_dir_name %>/.build/context.tar.gz ${GIT_BRANCH}
+	@git archive --format tar.gz -o ./<%= env.orchestration_dir_name %>/.build/context.tar.gz ${git_branch}
 	@docker build \
 	             --build-arg BUNDLE_GITHUB__COM \
 	             --build-arg BUNDLE_BITBUCKET__ORG \
-		     -t ${DOCKER_USERNAME}/${DOCKER_REPOSITORY} \
-		     -t ${DOCKER_USERNAME}/${DOCKER_REPOSITORY}:$(shell git rev-parse --short --verify ${GIT_BRANCH}) \
+		     -t ${docker_username}/${docker_repository} \
+		     -t ${docker_username}/${docker_repository}:${git_version} \
 		     ./<%= env.orchestration_dir_name %>/
 	@echo "Build complete."
 
-push: VERSION := $(shell git rev-parse --short --verify ${GIT_BRANCH})
 push:
-	docker push ${DOCKER_USERNAME}/${DOCKER_REPOSITORY}:${VERSION}
+	docker push ${docker_username}/${docker_repository}:${git_version}

data/lib/orchestration/templates/application.mk.erb

@@ -0,0 +1,14 @@
+#
+# Example test command
+#
+# This command will call `test-setup` before running your usual test pipeline.
+#
+# `test-setup` starts all containers specified in `docker-compose.test.yml`,
+# waits for them to be ready, and runs DB migrations.
+#
+# In your CI environment, simply run `make test`.
+#
+.PHONY: test
+test: test-setup
+	bundle exec rspec
+        bundle exec rubocop

data/lib/orchestration/templates/deploy.mk.erb

@@ -0,0 +1,34 @@
+.PHONY: deploy stop start config pull logs migrate
+
+COMPOSE_BASE:=HOST_UID=$(shell id -u) \
+              DOCKER_USERNAME=%%USERNAME%% \
+              DOCKER_REPOSITORY=%%REPOSITORY%%:%%VERSION%% \
+              docker-compose \
+              -p %%REPOSITORY%% \
+              -f docker-compose.yml
+
+COMPOSE:=${COMPOSE_BASE} -f docker-compose.production.yml -f docker-compose.override.yml
+
+deploy:
+	@echo "Deploying application to Docker swarm..."
+	@${COMPOSE} config | docker stack deploy -c - %%REPOSITORY%%
+
+stop:
+	@echo "Stopping containers..."
+	@${COMPOSE} down
+
+start:
+	@echo "Launching application..."
+	@${COMPOSE} up -d
+
+config:
+	@${COMPOSE} config
+
+pull:
+	@${COMPOSE} pull
+
+logs:
+	@${COMPOSE} logs -f
+
+migrate:
+	@${COMPOSE} run --rm app bundle exec rake db:migrate

data/lib/orchestration/templates/docker-compose.override.yml.erb

@@ -0,0 +1 @@
+version: <%= env.docker_api_version.to_json %>