orchestration 0.4.20 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc36f50df9f55e1edf2cefa9bde6fb76bc9a9682cf4f4692f9bcd822a49abe60
-  data.tar.gz: 155fa0c6af9f37ca0c5c7e259b641ec023a3f5eb58d4cf42f217e7dc2a181b3b
+  metadata.gz: 90d0d9e479a7b4797751a7a6b457c613813b2c8d2df9a6b17bb11c8b26c379ff
+  data.tar.gz: 7812364e32a9f3289fea6e86591b46d055cfe35ab13244932bf23b8597b2f0c8
 SHA512:
-  metadata.gz: d851ca72e1521f2042a79ebae4486de7cb8bbafff38a5f33be243efe2a9a152e69cfbf0c0ce5fb4178fcc562e3c2a025ff2b3ef6d732d27fc21011d65ed23b7b
-  data.tar.gz: 784ec5045ad62e360258f93c7ae8b4a9adb9277a63a4b6f0c9126bf8a08869e551a459318f44de50c3a5459d689e96516f6093d2ad867667ac3cc498bc938644
+  metadata.gz: ee3ebbeb61d6be44d819067fb0dfcc4db27ee3a4c5c14510ca5b7313838ce853568365aebe4b7593550738834fadf5f34564c6db5bc9404669aa5f9b2e3dc4b7
+  data.tar.gz: ec9cfbd34b8909e7ff72887301423739c02ec96d86e2e6bece4f25cc27738fe4de6d8486d28e02e72103fadfe408b7b1cc4e61fc8b2b45060352e1ac1a45137f

data/Makefile

@@ -3,12 +3,3 @@ test:
 	bundle exec rspec
 	bundle exec rubocop
 	bundle exec strong_versions
-
-.PHONY: manifest
-manifest:
-	 git ls-files | GREP_OPTIONS='' grep -v '^spec' > MANIFEST
-	 git diff-index --quiet HEAD || (git add MANIFEST && git commit -m "Update manifest" || :)
-
-.PHONY: release
-release: manifest
-	gem build orchestration.gemspec

data/README.md

@@ -1,13 +1,5 @@
 # Orchestration
 
-```
-I've got two tickets to the game
-It'd be great if I could take you to it this Sunday
-                                       --Nickelback
-```
-
-## Overview
-
 _Orchestration_ aims to provide a convenient and consistent process for working with _Rails_ and _Docker_ without obscuring underlying components.
 
 At its core _Orchestration_ is simply a `Makefile` and a set of `docker-compose.yml` files with sensible, general-purpose default settings. Users are encouraged to tailor the generated build-out to suit their application; once the build-out has been generated it belongs to the application.
@@ -35,7 +27,7 @@ The below screenshot demonstrates _Orchestration_ being installed in a brand new
 Add _Orchestration_ to your Gemfile:
 
 ```ruby
-gem 'orchestration', '~> 0.4.20'
+gem 'orchestration', '~> 0.5.3'
 ```
 
 Install:
@@ -56,6 +48,11 @@ rake orchestration:install server=unicorn # (or 'puma' [default], etc.)
 
 To rebuild all build-out at any time, pass `force=yes` to the above install command.
 
+To rebuild just `orchestration/Makefile` (useful after upgrading the _Orchestration_ gem):
+```bash
+rake orchestration:install:makefile
+```
+
 You will be prompted to enter values for your _Docker_ organisation and repository name. For example, the _organisation_ and _repository_ for https://hub.docker.com/r/rubyorchestration/sampleapp are `rubyorchestration` and `sampleapp` respectively. If you are unsure of these values, they can be modified later by editing `.orchestration.yml` in the root of your project directory.
 
 #### Configuration files
@@ -138,7 +135,20 @@ Note that `git archive` is used to generate the build context. Any uncommitted c
 make build
 ```
 
-See [build environment](#build-environment) for more details.
+The `include` option can also be passed to provide a manifest file. Any files listed in this file will also be built into the _Docker_ image. Files **must** be located within the project directory.
+
+```bash
+make build include=manifest.txt
+```
+
+```bash
+# manifest.txt
+doc/api/swagger.json
+doc/api/soap.xml
+doc/api/doc.html
+```
+
+See also [build environment](#build-environment) if you use gems hosted on private _GitHub_/_Bitbucket_ repositories.
 
 #### Push latest image
 
@@ -276,10 +286,10 @@ The output from most underlying components is hidden in an effort to make contin
 tail -f log/orchestration*.log
 ```
 
-A convenience `Makefile` target `dump` is provided which will output all consumed _stdout_ and _stderr_:
+A convenience `Makefile` target `dump` is provided. The following command will output all consumed _stdout_, _stderr_, and _Docker Compose_ container logs for the test environment:
 
 ```bash
-make dump
+make dump env=test
 ```
 
 All commands also support the `verbose` flag which will output all logs immediately to the console:
@@ -395,44 +405,10 @@ This is a convention of the _Orchestration_ gem intended to make _RabbitMQ_ conf
 
 ## Alternate Database Configuration Files
 
-If you have multiple databases configured in various (e.g.) `config/database.*.yml` files then the `make wait-database` command can be used directly in continuous integration environments to verify that your database services are available.
-
-Note that all services from the relevant `docker-compose.yml` configuration will be loaded when using the `make start` or `make test-setup` (called by default `make test` command).
+If you have multiple databases configured in various `config/database.*.yml` files then the `make wait` command will automatically detect database configurations.
 
-Assuming the following configurations:
-```
-# orchestration/docker-compose.test.yml
-version: '3.7'
-services:
-  customdb:
-    image: postgres
-    ports:
-      - "55667:5432"
-  # ...
-```
+If a service `database-example` is included in the relevant _Docker Compose_ configuration then `config/database.example.yml` will be used to load the connection configuration. Note that the service name _must_ begin with `database-`.
 
-```
-# config/database.custom.yml
-test:
-  adapter: postgresql
-  host: 127.0.0.1
-  port: 55667
-  username: postgres
-  password: password
-  database: postgres
-```
-
-The following command can be used to ensure that the `customdb` service is available:
-```
-make wait-database service=custom config=config/database.custom.yml env=test
-```
-
-You may wish to extend the example `Makefile` to include something like this:
-```
-test: test-setup
-	$(MAKE) wait-database service=custom config=config/database.custom.yml env=test
-	# ...
-```
 ## License
 
 [MIT License](LICENSE)

data/config/locales/en.yml

@@ -2,6 +2,7 @@ en:
   orchestration:
     attempt_limit: "Unable to reconnect after %{limit} attempts. Aborting."
     default: "default"
+    auto_update: "Orchestration Makefile was automatically updated to the latest version."
 
     app:
       waiting: "Waiting for app: %{config}"
@@ -19,10 +20,6 @@ en:
       ready: "Mongo is ready."
       bad_config: "Unable to parse Mongo config: %{path}. Expected section for one of: %{expected}"
 
-    haproxy:
-      waiting: "Waiting for HAProxy: %{config}"
-      ready: "HAProxy is ready."
-
     rabbitmq:
       waiting: "Waiting for RabbitMQ: %{config}"
       ready: "RabbitMQ is ready."
@@ -42,24 +39,8 @@ en:
           prompt: "project name"
 
     rake:
-      app:
-        wait: "Wait for app to become available"
-        unknown_web_server: "Unrecognised web server '%{server}'. Expected: %{expected}"
-
-      database:
-        wait: "Wait for database to become available"
-
-      listener:
-        wait: "Wait for a locally-bound service to accept connections (pass `service=NAME`)"
-
-      mongo:
-        wait: "Wait for Mongo to become available"
-
-      haproxy:
-        wait: "Wait for HAProxy to become available"
-
-      rabbitmq:
-        wait: "Wait for RabbitMQ to become available"
-
+      config: "Parse and output Orchestration config (internal use)"
+      healthcheck: "Execute healthcheck; used for HEALTHCHECK command in Docker image"
       install: "Install Orchestration tools"
       install_makefile: "(Re)create orchestration/Makefile"
+      wait: "Wait for development/test dependencies to be available"

data/lib/orchestration.rb

@@ -22,6 +22,7 @@ require 'orchestration/file_helpers'
 require 'orchestration/docker_compose'
 require 'orchestration/environment'
 require 'orchestration/errors'
+require 'orchestration/docker_healthcheck'
 require 'orchestration/install_generator'
 require 'orchestration/railtie' if defined?(Rails)
 require 'orchestration/service_check'

data/lib/orchestration/docker_compose/app_service.rb

@@ -27,7 +27,7 @@ module Orchestration
 
         def healthcheck
           {
-            'test' => ['ruby', "/app/#{orchestration}/healthcheck.rb"],
+            'test' => ['bundle', 'exec', 'rake', 'orchestration:healthcheck'],
             # Defaults according to
             # https://docs.docker.com/engine/reference/builder/#healthcheck
             # Except start_period which cannot be set to 0s

data/lib/orchestration/docker_compose/install_generator.rb

@@ -10,10 +10,6 @@ module Orchestration
         @terminal = terminal
       end
 
-      def docker_compose_yml
-        create_compose_file
-      end
-
       def docker_compose_test_yml
         create_compose_file(:test)
       end
@@ -22,22 +18,10 @@ module Orchestration
         create_compose_file(:development)
       end
 
-      def docker_compose_local_yml
-        create_compose_file(:local)
-      end
-
       def docker_compose_production_yml
         create_compose_file(:production)
       end
 
-      def docker_compose_override_yml
-        simple_copy(
-          'docker-compose.override.yml',
-          @env.docker_compose_path(:override),
-          overwrite: false
-        )
-      end
-
       def enabled_services(environment)
         service_names(environment).select { |name| service_enabled?(name) }
       end

data/lib/orchestration/docker_healthcheck.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'net/http'
+module Orchestration
+  class DockerHealthcheck
+    def self.execute
+      new.execute
+    end
+
+    def execute
+      return_code = 1
+
+      # rubocop:disable Lint/RescueException
+      begin
+        response = run
+        return_code = 0 if success?(response.code)
+        puts message(response.code)
+      rescue Exception => e
+        puts "[#{__FILE__}] ERROR: #{e.inspect}"
+      ensure
+        exit return_code
+      end
+      # rubocop:enable Lint/RescueException
+    end
+
+    private
+
+    def run
+      client = Net::HTTP.new(
+        ENV.fetch('WEB_HOST', 'localhost'),
+        ENV.fetch('WEB_PORT', '8080').to_i
+      )
+
+      client.read_timeout = ENV.fetch('WEB_HEALTHCHECK_READ_TIMEOUT', '10').to_i
+      client.open_timeout = ENV.fetch('WEB_HEALTHCHECK_OPEN_TIMEOUT', '10').to_i
+
+      client.start do |request|
+        request.get(ENV.fetch('WEB_HEALTHCHECK_PATH') { '/' })
+      end
+    end
+
+    def success_codes
+      ENV.fetch('WEB_HEALTHCHECK_SUCCESS_CODES', '200,202,204').split(',')
+    end
+
+    def success?(code)
+      success_codes.include?(code.to_s)
+    end
+
+    def message(code)
+      if success?(code)
+        outcome = 'SUCCESS ✓ '
+        in_or_not = 'IN'
+      else
+        outcome = 'FAILURE ✘ '
+        in_or_not = 'NOT IN'
+      end
+
+      accepted = success_codes.join(', ')
+      message = "#{in_or_not} [#{accepted}] : #{outcome} [#{__FILE__}]"
+
+      "# HTTP_STATUS(#{code}) #{message}"
+    end
+  end
+end

data/lib/orchestration/environment.rb

@@ -53,7 +53,7 @@ module Orchestration
     end
 
     def docker_compose_path(env = nil)
-      return orchestration_root.join('docker-compose.yml') if env.nil?
+      env ||= 'development'
 
       orchestration_root.join("docker-compose.#{env}.yml")
     end

data/lib/orchestration/file_helpers.rb

@@ -66,9 +66,8 @@ module Orchestration
     end
 
     def skip?(present, content, previous_content, options)
-      overwrite = options.fetch(:overwrite, true)
       return false unless present
-      return true unless overwrite || force?
+      return true unless options.fetch(:overwrite, true) || force?
 
       previous_content == content
     end

data/lib/orchestration/install_generator.rb

@@ -27,10 +27,18 @@ module Orchestration
       @terminal.write(:skip, relpath)
     end
 
-    def orchestration_makefile
+    def verify_makefile(skip = true)
+      # Only run when called explicitly [from Rake tasks].
+      # (I know this is hacky).
+      return if skip
+
       content = template('orchestration.mk', makefile_environment)
       path = @env.orchestration_root.join('Makefile')
-      path.exist? ? update_file(path, content) : create_file(path, content)
+      return if path.exist? && content == File.read(path)
+
+      write_file(path, content)
+      @terminal.write(:update, 'Makefile')
+      @terminal.write(:status, t(:auto_update))
     end
 
     def application_makefile
@@ -39,6 +47,12 @@ module Orchestration
       inject_if_missing(path, 'include orchestration/Makefile')
     end
 
+    def orchestration_makefile
+      content = template('orchestration.mk', makefile_environment)
+      path = @env.orchestration_root.join('Makefile')
+      path.exist? ? update_file(path, content) : create_file(path, content)
+    end
+
     def dockerfile
       create_file(
         orchestration_dir.join('Dockerfile'),
@@ -54,23 +68,10 @@ module Orchestration
       FileUtils.chmod('a+x', path)
     end
 
-    def gitignore
-      path = @env.root.join('.gitignore')
-      globs = %w[.build/ .deploy/ Gemfile Gemfile.lock docker-compose.local.yml]
-      lines = %w[orchestration/.sidecar .env deploy.tar] + globs.map do |line|
-        "#{@env.orchestration_dir_name}/#{line}"
-      end
-
-      ensure_lines_in_file(path, lines)
-    end
-
     def docker_compose
-      @docker_compose.docker_compose_yml
       @docker_compose.docker_compose_test_yml
       @docker_compose.docker_compose_development_yml
-      @docker_compose.docker_compose_local_yml
       @docker_compose.docker_compose_production_yml
-      @docker_compose.docker_compose_override_yml
     end
 
     def puma
@@ -112,16 +113,18 @@ module Orchestration
       service_config('rabbitmq.yml', Services::RabbitMQ::Configuration)
     end
 
-    def healthcheck
-      simple_copy('healthcheck.rb')
+    def env
+      simple_copy('env', @env.root.join('.env'), overwrite: false)
     end
 
-    def yaml_bash
-      simple_copy('yaml.bash')
-    end
+    def gitignore
+      path = @env.root.join('.gitignore')
+      globs = %w[.build/ .deploy/ Gemfile Gemfile.lock docker-compose.local.yml]
+      lines = %w[orchestration/.sidecar .env deploy.tar] + globs.map do |line|
+        "#{@env.orchestration_dir_name}/#{line}"
+      end
 
-    def env
-      simple_copy('env', @env.root.join('.env'), overwrite: false)
+      ensure_lines_in_file(path, lines)
     end
 
     def deploy_mk

data/lib/orchestration/service_check.rb

@@ -16,7 +16,7 @@ module Orchestration
     end
 
     def run
-      return echo_missing unless @service.configuration.configured?
+      return unless @service.configuration.configured?
 
       echo_start
       success = attempt_connection
@@ -39,14 +39,6 @@ module Orchestration
       false
     end
 
-    def echo_missing
-      @terminal.write(
-        @service_name.to_sym,
-        "#{@service.configuration.error} (skipping)",
-        :error
-      )
-    end
-
     def echo_start
       @terminal.write(@service_name.to_sym, '', :status)
     end

data/lib/orchestration/services/database/configuration.rb

@@ -18,14 +18,14 @@ module Orchestration
           "[#{adapter.name}] #{host}:#{port}"
         end
 
-        def settings
+        def settings(healthcheck: false)
           {
             adapter: adapter.name,
             host: host,
             port: port,
             username: username,
             password: password,
-            database: database
+            database: healthcheck ? adapter.credentials['database'] : database
           }.transform_keys(&:to_s)
         end
 

data/lib/orchestration/services/database/healthcheck.rb

@@ -26,7 +26,7 @@ module Orchestration
         end
 
         def settings
-          @configuration.settings
+          @configuration.settings(healthcheck: true)
         end
       end
     end

data/lib/orchestration/services/listener/configuration.rb

@@ -11,7 +11,7 @@ module Orchestration
         end
 
         def friendly_config
-          "[#{@service_name}] #{host}:#{port}"
+          "[#{image || @service_name}] #{host}:#{port}"
         end
       end
     end

data/lib/orchestration/services/mixins/configuration_base.rb

@@ -29,6 +29,7 @@ module Orchestration
       end
 
       def host
+        return @service_name if @env.environment == 'test' && @options[:sidecar]
         return '127.0.0.1' if %w[test development].include?(@env.environment)
 
         @service_name
@@ -42,15 +43,23 @@ module Orchestration
         false
       end
 
+      def image
+        service['image']
+      end
+
       def port
         return @env.app_port if @service_name == 'app'
 
-        service = @env.docker_compose_config
-                      .fetch('services')
-                      .fetch(@service_name)
+        local, remote = parse_port(service).map(&:to_i)
+        return remote if @env.environment == 'test' && @options[:sidecar]
+
+        (@env.environment == 'production' ? remote : local)
+      end
 
-        local, remote = parse_port(service)
-        (@env.environment == 'production' ? remote : local).to_i
+      def service
+        @service ||= @env.docker_compose_config
+                         .fetch('services')
+                         .fetch(@service_name)
       end
 
       def parse_port(service)

data/lib/orchestration/services/mixins/healthcheck_base.rb

@@ -37,7 +37,8 @@ module Orchestration
 
       def initialize(env, service_name = nil, options = {})
         @options = options
-        @configuration = configuration_class.new(env, service_name, options)
+        @options[:sidecar] = options[:sidecar] && !options[:sidecar].empty?
+        @configuration = configuration_class.new(env, service_name, @options)
       end
 
       def service_name

data/lib/orchestration/templates/Dockerfile.erb

@@ -20,10 +20,10 @@ COPY .build/Gemfile .build/Gemfile.lock ./
 RUN bundle install --without development test --deployment
 <% if defined?(Webpacker) %>
 COPY .build/package.json .build/yarn.lock ./
-RUN . /root/.bashrc && yarn install
+RUN . /root/.bashrc ; yarn install
 <% end %>
 ADD .build/context.tar .
-<% if defined?(Webpacker) %>RUN . /root/.bashrc && NODE_ENV=production RAILS_ENV=production yarn install && NODE_ENV=production RAILS_ENV=production SECRET_KEY_BASE=abc123 bundle exec rake assets:precompile<% elsif Rake::Task.tasks.map(&:name).include?('assets:precompile') %>RUN NODE_ENV=production RAILS_ENV=production SECRET_KEY_BASE=abc123 bundle exec rake assets:precompile<% end %>
+<% if defined?(Webpacker) %>RUN . /root/.bashrc ; NODE_ENV=production RAILS_ENV=production yarn install && NODE_ENV=production RAILS_ENV=production SECRET_KEY_BASE=abc123 bundle exec rake assets:precompile<% elsif Rake::Task.tasks.map(&:name).include?('assets:precompile') %>RUN NODE_ENV=production RAILS_ENV=production SECRET_KEY_BASE=abc123 bundle exec rake assets:precompile<% end %>
 RUN echo "${GIT_COMMIT}" > /app/GIT_COMMIT
 HEALTHCHECK --interval=<%= healthcheck['interval'] %> \
             --timeout=<%= healthcheck['timeout'] %> \

data/lib/orchestration/templates/application.mk.erb

@@ -12,3 +12,11 @@
 test: test-setup
 	bundle exec rspec
 	bundle exec rubocop
+
+# Start development containers and create/migrate/seed database
+.PHONY: develop
+develop: start
+	bundle install
+	bundle exec rake db:create
+	bundle exec rake db:migrate
+	bundle exec rake db:seed

data/lib/orchestration/templates/database.yml.erb

@@ -8,7 +8,7 @@ development:
   port: <%= compose.call('development').local_port('database') %>
   username: <%= compose.call(nil).database_adapter.credentials['username'] %>
   password: <%= compose.call(nil).database_adapter.credentials['password'] %>
-  database: <%= compose.call(nil).database_adapter.credentials['database'] %>
+  database: development
   <%% if ENV.key?('DEVELOPMENT_DATABASE_URL') %>
   url: <%%= ENV['DEVELOPMENT_DATABASE_URL'] %>
   <%% end %>
@@ -21,7 +21,7 @@ test:
   port: <%= compose.call('test').local_port('database') %>
   username: <%= compose.call(nil).database_adapter.credentials['username'] %>
   password: <%= compose.call(nil).database_adapter.credentials['password'] %>
-  database: <%= compose.call(nil).database_adapter.credentials['database'] %>
+  database: test
   # Useful for certain continuous integration environments (e.g. Jenkins in
   # Docker) where the DB hostname may be a service name rather than `127.0.0.1`:
   <%% if ENV.key?('TEST_DATABASE_URL') %>

data/lib/orchestration/templates/deploy.mk.erb

@@ -22,9 +22,7 @@ compose_base:=env HOST_UID=$(shell id -u) \
               DOCKER_REPOSITORY=%%REPOSITORY%%:%%VERSION%% \
               docker-compose \
               -p ${project_name} \
-              -f docker-compose.yml
-
-compose:=${compose_base} -f docker-compose.production.yml -f docker-compose.override.yml
+              -f docker-compose.production.yml
 
 .PHONY: deploy
 deploy:
@@ -36,7 +34,7 @@ endif
 
 .PHONY: deploy-stack
 deploy-stack:
-	@${compose} config | docker stack deploy --prune --with-registry-auth -c - ${project_name}
+	${compose} config | docker stack deploy --prune --with-registry-auth -c - ${project_name}
 
 .PHONY: console
 service := app

data/lib/orchestration/templates/entrypoint.sh.erb

@@ -2,7 +2,7 @@
 HOST_UID=${HOST_UID:-$(id -u)}
 set -e
 id owner >/dev/null 2>&1 || useradd -u ${HOST_UID} -m -o owner
-mkdir -p /app/tmp/pids
+mkdir -p /app/tmp/pids /app/log /app/db
 chown -Rf owner:owner /app/tmp /app/log /app/db
 rm -f /app/tmp/pids/server.pid
 

data/lib/orchestration/templates/makefile_macros.mk.erb

@@ -87,17 +87,19 @@ restore_env:=( \
 
 key_chars:=[a-zA-Z0-9_]
 censored:=**********
-censor=sed 's/\(^${key_chars}*$(1)${key_chars}*\)=\(.*\)$$/\1=${censored}/'
-censor_urls:=sed 's|\([a-zA-Z0-9_+]\+://.*:\).*\(@.*\)$$|\1${censored}\2|'
-format_env:=$(call censor,SECRET) | \
-            $(call censor,PASSWORD) | \
-            $(call censor,TOKEN) | \
-            $(call censor,PRIVATE) | \
-            $(call censor,KEY) | \
-            ${censor_urls} | \
-            sed 's/\(^[a-zA-Z0-9_]\+\)=/${blue}\1${reset}=/' | \
-            sed 's/^/  /' | \
-            sed 's/=\(.*\)$$/=${yellow}\1${reset}/'
+censor=s/\(^${key_chars}*$(1)${key_chars}*\)=\(.*\)$$/\1=${censored}/
+censor_urls:=s|\([a-zA-Z0-9_+]\+://.*:\).*\(@.*\)$$|\1${censored}\2|
+format_env:=sed '$(call censor,SECRET); \
+                 $(call censor,TOKEN); \
+                 $(call censor,PRIVATE); \
+                 $(call censor,KEY); \
+                 $(censor_urls); \
+                 /^\s*$$/d; \
+                 /^\s*\#/d; \
+                 s/\(^[a-zA-Z0-9_]\+\)=/${blue}\1${reset}=/; \
+                 s/^/  /; \
+                 s/=\(.*\)$$/=${yellow}\1${reset}/' | \
+            sort
 
 fail=( \
        $(call printraw,' ${cross}') ; \

data/lib/orchestration/templates/orchestration.mk.erb

@@ -38,8 +38,9 @@ ifneq (,$(wildcard ${env_file}))
   rake=. ${env_file} && ${rake_cmd}
 endif
 
-docker_organization=$(shell bash ${orchestration_dir}/yaml.bash docker_organization)
-docker_repository=$(shell bash ${orchestration_dir}/yaml.bash docker_repository)
+docker_config:=$(shell ${rake} orchestration:config)
+docker_organization=$(word 1,$(docker_config))
+docker_repository=$(word 2,$(docker_config))
 
 ifeq (,$(project_name))
   project_base = ${docker_repository}_${env}
@@ -73,19 +74,26 @@ else
   compose_project_name = ${project_base}
 endif
 
-compose_base=env HOST_UID=$(shell id -u) \
+compose_base=env -i \
+             PATH=$(PATH) \
+             HOST_UID=$(shell id -u) \
              DOCKER_ORGANIZATION="${docker_organization}" \
              DOCKER_REPOSITORY="${docker_repository}" \
              COMPOSE_PROJECT_NAME="${compose_project_name}" \
 	     ${sidecar_compose} \
              docker-compose \
-             -f "${orchestration_dir}/docker-compose.yml"
+             -f ${orchestration_dir}/docker-compose.${env}.yml
 
 git_branch ?= $(if $(branch),$(branch),$(shell git rev-parse --abbrev-ref HEAD))
-git_version ?= $(shell git rev-parse --short --verify ${git_branch})
+ifndef dev
+  git_version ?= $(shell git rev-parse --short --verify ${git_branch})
+else
+  git_version = dev
+endif
+
 docker_image=${docker_organization}/${docker_repository}:${git_version}
 
-compose=${compose_base} -f ${orchestration_dir}/docker-compose.${env}.yml -f ${orchestration_dir}/docker-compose.override.yml
+compose=${compose_base}
 random_str=cat /dev/urandom | LC_ALL=C tr -dc 'a-z' | head -c $1
 
 ifneq (,$(wildcard ${orchestration_dir}/docker-compose.local.yml))
@@ -101,9 +109,9 @@ ifndef network
 start: network := ${compose_project_name}_default
 endif
 start: _clean-logs
-	@$(call print,'${yellow}Starting containers${reset} ...')
+	@$(call print,'${yellow}Starting ${cyan}${env}${yellow} containers${reset} ...')
 ifeq (${env},$(filter ${env},test development))
-	@${compose} up --detach --force-recreate --renew-anon-volumes ${services} ${log} || ${fail}
+	@${compose} up --detach --force-recreate --renew-anon-volumes --remove-orphans ${services} ${log} || ${fail}
 	@[ -n '${sidecar}' ] && \
          ( \
            $(call printraw,' ${yellow}(joining dependency network ${green}${network}${yellow})${reset} ... ') ; \
@@ -116,15 +124,11 @@ ifeq (${env},$(filter ${env},test development))
          ) \
          || ( [ -z '${sidecar}' ] || ${fail} )
 else
-	@${compose} up --detach --scale app=$${instances:-1} ${services} ${log} || ${fail}
+	@${compose} up --detach --scale app=$${instances:-1} ${log} || ${fail}
 endif
 	@$(call printrawln,' ${green}started${reset} ${tick}')
 	@$(call println,'${yellow}Waiting for services to become available${reset} ...')
-ifdef services
-	@$(call make,wait services='${services}') 2>${stderr} || ${fail}
-else
 	@$(call make,wait) 2>${stderr} || ${fail}
-endif
 
 <% services.each do |service| %>
 .PHONY: start-<%= service %>
@@ -136,7 +140,7 @@ start-<%= service %>:
 .PHONY: stop
 stop: network := ${compose_project_name}_default
 stop:
-	@$(call print,'${yellow}Stopping containers${reset} ...')
+	@$(call print,'${yellow}Stopping ${cyan}${env}${yellow} containers${reset} ...')
 	@if docker ps --format "{{.ID}}" | grep -q $(shell hostname) ; \
           then \
             ( docker network disconnect ${network} $(shell hostname) ${log} || : ) \
@@ -169,7 +173,7 @@ serve:
          then ( \
                 $(call println,'${yellow}Environment${reset}: ${green}${env_file}${reset}') && \
                 cat '${env_file}' | ${format_env} && \
-                $(call println,'') \
+                $(call printrawln,'') \
             ) ; \
         fi
 	${rails}
@@ -182,7 +186,7 @@ console:
          then ( \
                 $(call println,'${yellow}Environment${reset}: ${green}${env_file}${reset}') && \
                 cat '${env_file}' | ${format_env} && \
-                $(call println,'') \
+                $(call printrawln,'') \
             ) ; \
         fi
 	${rails} console
@@ -190,23 +194,25 @@ console:
 .PHONY: test-setup
 test-setup: env := test
 test-setup:
+ifndef light
 	@$(call make,start env=test)
-ifneq (,$(wildcard config/database.yml))
+  ifneq (,$(wildcard config/database.yml))
 	${rake} db:create || :
-  ifneq (,$(wildcard db/structure.sql))
+    ifneq (,$(wildcard db/structure.sql))
 	${rake} db:structure:load
-  else ifneq (,$(wildcard db/schema.rb))
+    else ifneq (,$(wildcard db/schema.rb))
 	${rake} db:schema:load
-  endif
+    endif
 
 	${rake} db:migrate
+  endif
 endif
 
 .PHONY: dump
 dump:
 ifndef verbose
 	@$(call println)
-	@$(call println,'${yellow}Captured${reset} ${green}stdout${reset} ${yellow}and${reset} ${red}stderr${reset} ${yellow}log data${reset}:')
+	@$(call println,'${yellow}Captured${reset} ${green}stdout${reset} ${yellow}and${reset} ${red}stderr${reset} ${yellow}log data [${cyan}${env}${yellow}]${reset}:')
 	@$(call println)
 	@echo
 	@test -f '${stdout}' && ( \
@@ -226,6 +232,14 @@ ifndef verbose
           $(call hr,${red}) ; \
         )
 endif
+	@echo ; \
+        $(call hr,${yellow}) ; \
+        $(call println,'${gray}docker-compose logs${reset}') ; \
+        $(call hr,${yellow}) ; \
+        echo
+	@${compose} logs
+	@echo ; \
+        $(call hr,${yellow})
 
 .PHONY: image
 image:
@@ -245,10 +259,6 @@ endif
              -e "s/%%ORGANIZATION%%/${docker_organization}/g" \
              ${orchestration_dir}/deploy.mk > \
              ${orchestration_dir}/.deploy/${docker_repository}/Makefile
-	@cp ${orchestration_dir}/docker-compose.yml \
-            ${orchestration_dir}/docker-compose.production.yml \
-            ${orchestration_dir}/docker-compose.override.yml \
-            ${orchestration_dir}/.deploy/${docker_repository}/
 	@bundle_path="${path}" ; tar -C '${orchestration_dir}/.deploy' -cf "$${bundle_path:-./bundle.tar}" ./${docker_repository}
 
 .PHONY: deploy
@@ -296,55 +306,43 @@ endif
 ### Service healthcheck commands ###
 
 .PHONY: wait
-<% services.each do |service| %>
-ifneq (,$(findstring <%= service %>,${services}))
-wait: wait-<%= service %>
-endif
-<% end %>
-ifndef services
-wait: <%= services.map { |command| "wait-#{command}" }.join(' ') %>
-endif
+wait:
+	@${rake} orchestration:wait
 	@$(call println,'${yellow}All services${reset} ${green}ready${reset}. ${tick}')
 
 ## Generic Listener healthcheck for TCP services ##
 
 wait-listener:
-	@${rake} orchestration:listener:wait service=${service}
-
-## Test/development wait commands
-
-<% services.each do |service| %>
-<% next if service.to_sym == :app %>
-.PHONY: wait-<%= service %>
-ifdef config
-wait-<%= service %>: configvar := config=$(config)
-endif
-ifdef service
-wait-<%= service %>: servicevar := service=$(service)
-endif
-wait-<%= service %>:
-	@${rake} orchestration:<%= service %>:wait $(configvar) $(servicevar)
-
-<% end %>
-
-.PHONY: wait-app
-wait-app:
-	@# no-op
+	@${rake} orchestration:listener:wait service=${service} sidecar=${sidecar}
 
 ### Docker build commands ###
 
 .PHONY: build
-build: context = ${orchestration_dir}/.build/context.tar
-build:
+build: build_dir = ${orchestration_dir}/.build
+build: context = ${build_dir}/context.tar
+build: check-local-changes
 	@$(call print,'${yellow}Preparing build context from${reset} ${cyan}${git_branch}:${git_version}${reset} ... ')
 	@mkdir -p ${orchestration_dir}/.build ${log} || ${fail}
+ifndef dev
 	@git show ${git_branch}:./Gemfile > ${orchestration_dir}/.build/Gemfile 2>${stderr} || ${fail}
 	@git show ${git_branch}:./Gemfile.lock > ${orchestration_dir}/.build/Gemfile.lock 2>${stderr} || ${fail}
 <% if defined?(Webpacker) %>	@git show ${git_branch}:./package.json > ${orchestration_dir}/.build/package.json 2>${stderr} || ${fail}<% end %>
 <% if defined?(Webpacker) %>	@git show ${git_branch}:./yarn.lock > ${orchestration_dir}/.build/yarn.lock 2>${stderr} || ${fail}<% end %>
 	@git archive --format 'tar' -o '${context}' '${git_branch}' ${log} || ${fail}
-	@temp=$$(mktemp -d) ; ( cd "$${temp}" && tar -uvf '${context}' . ) ${log} || ${fail}
+else
+	@tar -cvf '${context}' . ${log} || ${fail}
+endif
 	@$(call printrawln,'${green}complete.${reset} ${tick}')
+ifdef include
+	@$(call print,'${yellow}Including files from:${reset} ${cyan}${include}${reset} ...')
+	@(while read line; do \
+            export line; \
+            include_dir="${build_dir}/$$(dirname "$${line}")/" && \
+            mkdir -p "$${include_dir}" && cp "$${line}" "$${include_dir}" \
+            && (cd '${orchestration_dir}/.build/' && tar rf 'context.tar' "$${line}"); \
+          done < '${include}') ${log} || ${fail}
+	@$(call printrawln,' ${green}complete.${reset} ${tick}')
+endif
 ifdef sidecar
         # Assume we are in a line-buffered environment (e.g. Jenkins)
 	@$(call println,'${yellow}Building image${reset} ...')
@@ -368,6 +366,17 @@ push:
 	@docker push ${docker_image} ${log_progress} || ${fail}
 	@$(call printrawln,' ${green}complete${reset}. ${tick}')
 
+.PHONY: check-local-changes
+check-local-changes:
+ifndef dev
+	@if [[ ! -z "$$(git status --porcelain)" ]] ; \
+         then \
+           $(call println,'${red}You have uncommitted changes which will not be included in your build:${reset}') ; \
+           git status --porcelain ; \
+           $(call println,'${yellow}Use ${cyan}make build dev=1${reset} ${yellow}to include these files.${reset}\n') ; \
+         fi
+endif
+
 ### Internal Commands ###
 #
 .PHONY: _clean-logs

data/lib/orchestration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Orchestration
-  VERSION = '0.4.20'
+  VERSION = '0.5.3'
 end

data/lib/tasks/orchestration.rake

@@ -15,47 +15,40 @@ namespace :orchestration do
     end
   end
 
-  namespace :app do
-    desc I18n.t('orchestration.rake.app.wait')
-    task :wait do
-      Orchestration::Services::App::Healthcheck.start(
-        nil, nil, config_path: ENV['config'], service_name: ENV['service']
-      )
-    end
-  end
-
-  namespace :database do
-    desc I18n.t('orchestration.rake.database.wait')
-    task :wait do
-      Orchestration::Services::Database::Healthcheck.start(
-        nil, nil, config_path: ENV['config'], service_name: ENV['service']
-      )
-    end
+  desc I18n.t('orchestration.rake.config')
+  task :config do
+    config = YAML.safe_load(File.read('.orchestration.yml'))
+    puts "#{config['docker']['organization']} #{config['docker']['repository']}"
   end
 
-  namespace :mongo do
-    desc I18n.t('orchestration.rake.mongo.wait')
-    task :wait do
-      Orchestration::Services::Mongo::Healthcheck.start(
-        nil, nil, config_path: ENV['config'], service_name: ENV['service']
-      )
-    end
-  end
-
-  namespace :rabbitmq do
-    desc I18n.t('orchestration.rake.rabbitmq.wait')
-    task :wait do
-      Orchestration::Services::RabbitMQ::Healthcheck.start(
-        nil, nil, config_path: ENV['config'], service_name: ENV['service']
-      )
-    end
+  desc I18n.t('orchestration.rake.healthcheck')
+  task :healthcheck do
+    Orchestration::DockerHealthcheck.execute
   end
 
-  namespace :listener do
-    desc I18n.t('orchestration.rake.listener.wait')
-    task :wait do
-      Orchestration::Services::Listener::Healthcheck.start(
-        nil, nil, service_name: ENV.fetch('service')
+  desc I18n.t('orchestration.rake.wait')
+  task :wait do
+    Orchestration::InstallGenerator.new.verify_makefile(false)
+    env = Orchestration::Environment.new
+    services = Orchestration::Services
+    env.docker_compose_config['services'].each do |name, _service|
+      path = nil
+
+      adapter = if name == 'database'
+                  services::Database
+                elsif name.include?('database')
+                  path = "config/database.#{name.sub('database-', '')}.yml"
+                  services::Database
+                elsif name == 'mongo'
+                  services::Mongo
+                elsif name == 'rabbitmq'
+                  services::RabbitMQ
+                else
+                  services::Listener
+                end
+
+      adapter::Healthcheck.start(
+        nil, nil, config_path: path, service_name: name, sidecar: ENV['sidecar']
       )
     end
   end

data/orchestration.gemspec

@@ -16,8 +16,11 @@ Gem::Specification.new do |spec|
   spec.homepage = url
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    File.readlines('MANIFEST').map(&:chomp)
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
   end
+
   spec.bindir = 'bin'
   spec.executables = []
   spec.require_paths = ['lib']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: orchestration
 version: !ruby/object:Gem::Version
-  version: 0.4.20
+  version: 0.5.3
 platform: ruby
 authors:
 - Bob Farrell
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-07 00:00:00.000000000 Z
+date: 2020-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: database_url
@@ -318,7 +318,6 @@ files:
 - ".travis.yml"
 - Gemfile
 - LICENSE
-- MANIFEST
 - Makefile
 - README.md
 - Rakefile
@@ -340,6 +339,7 @@ files:
 - lib/orchestration/docker_compose/install_generator.rb
 - lib/orchestration/docker_compose/mongo_service.rb
 - lib/orchestration/docker_compose/rabbitmq_service.rb
+- lib/orchestration/docker_healthcheck.rb
 - lib/orchestration/environment.rb
 - lib/orchestration/errors.rb
 - lib/orchestration/file_helpers.rb
@@ -374,17 +374,14 @@ files:
 - lib/orchestration/templates/application.mk.erb
 - lib/orchestration/templates/database.yml.erb
 - lib/orchestration/templates/deploy.mk.erb
-- lib/orchestration/templates/docker-compose.override.yml.erb
 - lib/orchestration/templates/entrypoint.sh.erb
 - lib/orchestration/templates/env.erb
-- lib/orchestration/templates/healthcheck.rb.erb
 - lib/orchestration/templates/makefile_macros.mk.erb
 - lib/orchestration/templates/mongoid.yml.erb
 - lib/orchestration/templates/orchestration.mk.erb
 - lib/orchestration/templates/puma.rb.erb
 - lib/orchestration/templates/rabbitmq.yml.erb
 - lib/orchestration/templates/unicorn.rb.erb
-- lib/orchestration/templates/yaml.bash.erb
 - lib/orchestration/terminal.rb
 - lib/orchestration/version.rb
 - lib/tasks/orchestration.rake

data/MANIFEST

@@ -1,78 +0,0 @@
-.gitignore
-.rspec
-.rubocop.yml
-.strong_versions.yml
-.travis.yml
-Gemfile
-LICENSE
-MANIFEST
-Makefile
-README.md
-Rakefile
-bin/console
-bin/rspec
-bin/rubocop
-bin/setup
-bin/strong_versions
-config/locales/en.yml
-doc/images/example.png
-lib/Rakefile
-lib/orchestration.rb
-lib/orchestration/docker_compose.rb
-lib/orchestration/docker_compose/app_service.rb
-lib/orchestration/docker_compose/compose_configuration.rb
-lib/orchestration/docker_compose/compose_helpers.rb
-lib/orchestration/docker_compose/configuration.rb
-lib/orchestration/docker_compose/database_service.rb
-lib/orchestration/docker_compose/install_generator.rb
-lib/orchestration/docker_compose/mongo_service.rb
-lib/orchestration/docker_compose/rabbitmq_service.rb
-lib/orchestration/environment.rb
-lib/orchestration/errors.rb
-lib/orchestration/file_helpers.rb
-lib/orchestration/install_generator.rb
-lib/orchestration/railtie.rb
-lib/orchestration/service_check.rb
-lib/orchestration/services.rb
-lib/orchestration/services/app.rb
-lib/orchestration/services/app/configuration.rb
-lib/orchestration/services/app/healthcheck.rb
-lib/orchestration/services/database.rb
-lib/orchestration/services/database/adapters.rb
-lib/orchestration/services/database/adapters/mysql2.rb
-lib/orchestration/services/database/adapters/postgresql.rb
-lib/orchestration/services/database/adapters/sqlite3.rb
-lib/orchestration/services/database/configuration.rb
-lib/orchestration/services/database/healthcheck.rb
-lib/orchestration/services/listener.rb
-lib/orchestration/services/listener/configuration.rb
-lib/orchestration/services/listener/healthcheck.rb
-lib/orchestration/services/mixins/configuration_base.rb
-lib/orchestration/services/mixins/healthcheck_base.rb
-lib/orchestration/services/mixins/http_healthcheck.rb
-lib/orchestration/services/mongo.rb
-lib/orchestration/services/mongo/configuration.rb
-lib/orchestration/services/mongo/healthcheck.rb
-lib/orchestration/services/rabbitmq.rb
-lib/orchestration/services/rabbitmq/configuration.rb
-lib/orchestration/services/rabbitmq/healthcheck.rb
-lib/orchestration/settings.rb
-lib/orchestration/templates/Dockerfile.erb
-lib/orchestration/templates/application.mk.erb
-lib/orchestration/templates/database.yml.erb
-lib/orchestration/templates/deploy.mk.erb
-lib/orchestration/templates/docker-compose.override.yml.erb
-lib/orchestration/templates/entrypoint.sh.erb
-lib/orchestration/templates/env.erb
-lib/orchestration/templates/healthcheck.rb.erb
-lib/orchestration/templates/makefile_macros.mk.erb
-lib/orchestration/templates/mongoid.yml.erb
-lib/orchestration/templates/orchestration.mk.erb
-lib/orchestration/templates/puma.rb.erb
-lib/orchestration/templates/rabbitmq.yml.erb
-lib/orchestration/templates/unicorn.rb.erb
-lib/orchestration/templates/yaml.bash.erb
-lib/orchestration/terminal.rb
-lib/orchestration/version.rb
-lib/tasks/orchestration.rake
-orchestration.gemspec

data/lib/orchestration/templates/docker-compose.override.yml.erb

@@ -1 +0,0 @@
-version: <%= env.docker_api_version.to_json %>

data/lib/orchestration/templates/healthcheck.rb.erb

@@ -1,56 +0,0 @@
-#
-# Orchestration Healthcheck Utility
-#
-#
-# https://github.com/bobf/orchestration
-#
-
-require 'net/http'
-
-def run
-  client = Net::HTTP.new(
-    ENV.fetch('WEB_HOST', 'localhost'),
-    ENV.fetch('WEB_PORT', '8080').to_i
-  )
-
-  client.read_timeout = ENV.fetch('WEB_HEALTHCHECK_READ_TIMEOUT', '10').to_i
-  client.open_timeout = ENV.fetch('WEB_HEALTHCHECK_OPEN_TIMEOUT', '10').to_i
-
-  client.start do |request|
-    request.get(ENV.fetch('WEB_HEALTHCHECK_PATH') { '/' })
-  end
-end
-
-def success_codes
-  ENV.fetch('WEB_HEALTHCHECK_SUCCESS_CODES', '200,202,204').split(',')
-end
-
-def success?(code)
-  success_codes.include?(code.to_s)
-end
-
-def message(code)
-  if success?(code)
-    outcome = 'SUCCESS ✓ '
-    in_or_not = 'IN'
-  else
-    outcome = 'FAILURE ✘ '
-    in_or_not = 'NOT IN'
-  end
-
-  accepted = success_codes.join(', ')
-
-  "# HTTP_STATUS(#{code}) #{in_or_not} [#{accepted}] : #{outcome} [#{__FILE__}]"
-end
-
-return_code = 1
-
-begin
-  response = run
-  return_code = 0 if success?(response.code)
-  puts message(response.code)
-rescue Exception => e
-  puts "[#{__FILE__}] ERROR: #{e.inspect}"
-ensure
-  exit return_code
-end

data/lib/orchestration/templates/yaml.bash.erb

@@ -1,22 +0,0 @@
-# https://stackoverflow.com/a/21189044 - thanks.
-function parse_yaml {
-   local prefix=$2
-   local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
-   sed -ne "s|^\($s\):|\1|" \
-        -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \
-        -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p"  $1 |
-   awk -F$fs '{
-      indent = length($1)/2;
-      vname[indent] = $2;
-      for (i in vname) {if (i > indent) {delete vname[i]}}
-      if (length($3) > 0) {
-         vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
-         printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
-      }
-   }'
-}
-
-eval $(parse_yaml ./.orchestration.yml)
-
-set -u
-echo -n "$(eval echo \$$1)"