orchestration 0.3.17 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a45477bd5cd5d3aaf948247ed294c24509c59b55c3244d2bdb34d7adb50bf7d8
-  data.tar.gz: cc1be97aec363dabbe212d4bc5160651ec78b2f46ed3fe5461950c8392522adc
+  metadata.gz: 2186d476519af97c595c422ec2e8e85244de7a21ba38ac57a9dac6078508147e
+  data.tar.gz: d62c7ae58f1c20eb34e94cf0120d1ccd2dafbfa5820a4f9cdde0a36e51a4942d
 SHA512:
-  metadata.gz: e151308a54ac3048249820773ee5c2657fd1e6f1cfa2d8429ccfae1782bb77bc6413dcdd55a010004fc6b869eae9e057f89b52101eb9af5e9c236d34c08ccb49
-  data.tar.gz: 472e8a63dbd0277c2bc6a5e6cfc5c24ab69b27a132109f08f097288c8958c6ad0f7c1565c60db5b6075f4ee7dc1eaa49b1c7d46908995cb7b7e4c55cccb4c2ca
+  metadata.gz: 3c04057ada42ee20e573629778b56c4b35fc160325e1314e5055e30931b4e21c6948b1cdde40b6d0aff56e5076e708fd07af8ef55322d812f9b555f6ee7dbb03
+  data.tar.gz: 60a0c33ee614f8fb71a5feaae10e5788bd9f5cd8b45fbeb2a39b7cdcad0aa870bb5d61464a06c5478229a7ebf1f96a1a226ed0ff4ef0dc69280e0a617731310c

data/.gitignore

@@ -21,9 +21,13 @@ spec/dummy/.gitignore
 spec/dummy/Makefile
 spec/dummy/orchestration/*
 spec/dummy/config/unicorn.rb
+spec/dummy/config/*.bak
 
 orchestration-*.gem
 
 .byebug_history
 
 TODO
+
+docker/.build/
+docker/.context.tar

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2018-2019 Robert Farrell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/MANIFEST

@@ -0,0 +1,76 @@
+.gitignore
+.rspec
+.rubocop.yml
+.strong_versions.yml
+.travis.yml
+Gemfile
+LICENSE
+MANIFEST
+Makefile
+README.md
+Rakefile
+bin/console
+bin/rspec
+bin/rubocop
+bin/setup
+bin/strong_versions
+config/locales/en.yml
+lib/Rakefile
+lib/orchestration.rb
+lib/orchestration/docker_compose.rb
+lib/orchestration/docker_compose/app_service.rb
+lib/orchestration/docker_compose/compose_configuration.rb
+lib/orchestration/docker_compose/configuration.rb
+lib/orchestration/docker_compose/database_service.rb
+lib/orchestration/docker_compose/install_generator.rb
+lib/orchestration/docker_compose/mongo_service.rb
+lib/orchestration/docker_compose/rabbitmq_service.rb
+lib/orchestration/environment.rb
+lib/orchestration/errors.rb
+lib/orchestration/file_helpers.rb
+lib/orchestration/install_generator.rb
+lib/orchestration/railtie.rb
+lib/orchestration/service_check.rb
+lib/orchestration/services.rb
+lib/orchestration/services/app.rb
+lib/orchestration/services/app/configuration.rb
+lib/orchestration/services/app/healthcheck.rb
+lib/orchestration/services/database.rb
+lib/orchestration/services/database/adapters.rb
+lib/orchestration/services/database/adapters/mysql2.rb
+lib/orchestration/services/database/adapters/postgresql.rb
+lib/orchestration/services/database/adapters/sqlite3.rb
+lib/orchestration/services/database/configuration.rb
+lib/orchestration/services/database/healthcheck.rb
+lib/orchestration/services/listener.rb
+lib/orchestration/services/listener/configuration.rb
+lib/orchestration/services/listener/healthcheck.rb
+lib/orchestration/services/mixins/configuration_base.rb
+lib/orchestration/services/mixins/healthcheck_base.rb
+lib/orchestration/services/mixins/http_healthcheck.rb
+lib/orchestration/services/mongo.rb
+lib/orchestration/services/mongo/configuration.rb
+lib/orchestration/services/mongo/healthcheck.rb
+lib/orchestration/services/rabbitmq.rb
+lib/orchestration/services/rabbitmq/configuration.rb
+lib/orchestration/services/rabbitmq/healthcheck.rb
+lib/orchestration/settings.rb
+lib/orchestration/templates/Dockerfile.erb
+lib/orchestration/templates/application.mk.erb
+lib/orchestration/templates/database.yml.erb
+lib/orchestration/templates/deploy.mk.erb
+lib/orchestration/templates/docker-compose.override.yml.erb
+lib/orchestration/templates/entrypoint.sh.erb
+lib/orchestration/templates/env.erb
+lib/orchestration/templates/healthcheck.rb.erb
+lib/orchestration/templates/makefile_macros.mk.erb
+lib/orchestration/templates/mongoid.yml.erb
+lib/orchestration/templates/orchestration.mk.erb
+lib/orchestration/templates/puma.rb.erb
+lib/orchestration/templates/rabbitmq.yml.erb
+lib/orchestration/templates/unicorn.rb.erb
+lib/orchestration/templates/yaml.bash.erb
+lib/orchestration/terminal.rb
+lib/orchestration/version.rb
+lib/tasks/orchestration.rake
+orchestration.gemspec

data/Makefile

@@ -1,9 +1,9 @@
 .PHONY: test
-
 test:
 	./bin/rspec
 	./bin/rubocop
 	./bin/strong_versions
 
-readme:
-	markdown-toc -i README.md
+.PHONY: manifest
+manifest:
+	 git ls-files | GREP_OPTIONS='' grep -v '^spec' > MANIFEST

data/README.md

@@ -1,6 +1,17 @@
 # Orchestration
 
-_Orchestration_ is a toolkit for testing, building, and deploying _Ruby_ (including _Rails_) applications in _Docker_.
+## Overview
+
+_Orchestration_ aims to provide a convenient and consistent process for working with _Rails_ and _Docker_ without obscuring underlying components.
+
+At its core _Orchestration_ is just a `Makefile` and a set of `docker-compose.yml` files with sensible, general-purpose default settings. Users are encouraged to tailor the generated build-out to suit their application; once the build-out has been generated it belongs to the application.
+
+A typical _Rails_ application can be tested, built, pushed to _Docker Hub_, and deployed to _Docker Swarm_ with the following commands:
+
+```bash
+make test build push
+make deploy manager=user@swarm.example.com env_file=/var/configs/myapp.env
+```
 
 ## Getting Started
 
@@ -11,7 +22,7 @@ _Orchestration_ is a toolkit for testing, building, and deploying _Ruby_ (includ
 Add _Orchestration_ to your Gemfile:
 
 ```ruby
-gem 'orchestration', '~> 0.3.17'
+gem 'orchestration', '~> 0.4.0'
 ```
 
 Install:
@@ -22,13 +33,33 @@ bundle install
 
 ### Setup
 
-Generate configuration files:
+Generate configuration files and select your deployment server:
+
+#### Generate build-out
 
 ```bash
-bin/rake orchestration:install
+rake orchestration:install server=unicorn # (or 'puma' [default], etc.)
 ```
 
-Commit changes:
+To rebuild all build-out at any time, pass `force=yes` to the above install command.
+
+You will be prompted to enter values for your _Docker_ organisation and repository name. For example, the _organisation_ and _repository_ for https://hub.docker.com/r/redislabs/redis/ are `redislabs` and `redis` respectively. If you are unsure of these values, they can be modified later by editing `.orchestration.yml` in the root of your project directory.
+
+#### Configuration files
+
+_Orchestration_ generates the following files where appropriate. Backups are created if a file is replaced.
+
+* `config/database.yml`
+* `config/mongoid.yml`
+* `config/rabbitmq.yml` (see [RabbitMQ Configuration](#markdown-header-rabbitmq-configuration) for more details)
+* `config/unicorn.rb`
+* `config/puma.rb`
+
+You may need to merge your previous configurations with the generated files. You will only need to do this once.
+
+Test and development dependency containers bind to a randomly-generated [at install time] local port to avoid collisions. You may compare e.g. `orchestration/docker-compose.test.yml` with the `test` section of the generated `config/database.yml` to see how things fit together.
+
+When setup is complete, add the generated build-out to _Git_:
 
 ```bash
 git add .
@@ -37,201 +68,195 @@ git commit -m "Add Orchestration gem"
 
 ## Usage
 
-Start your dependencies:
+All `make` commands provided by _Orchestration_ (with the exception of `test` and `deploy`) recognise the `env` parameter. This is equivalent to setting the `RAILS_ENV` environment variable.
 
-```bash
-make start
+e.g.:
 ```
-
-Log in to your _Docker_ registry, then build and push your image:
-```bash
-docker login
-make build push
+# Stop all test containers
+make stop env=test
 ```
 
-Make a compact, portable, production-ready tarball:
+The default value for `env` is `development`.
+
+As with any `Makefile` targets can be chained together, e.g.:
 ```
-make bundle
+# Run tests, build, and push image
+make test build push
 ```
 
-Copy tarball to your server and unpack:
+### Containers
+
+All auto-detected services will be added to the relevant `docker-compose.<environment>.yml` files at installation time.
+
+#### Start services
+
 ```bash
-tar xf deploy.tar
-cd <your-app-name>/
+make start
 ```
 
-Add required config to `.env` file:
+#### Stop services
+
+```bash
+make stop
 ```
-# .env
-SECRET_KEY_BASE=<your-secure-token>
-LISTEN_PORT=8080
-VIRTUAL_HOST=yourdomain.com
+
+#### Interface directly with `docker-compose`
+
+```bash
+$(make compose env=test) logs -f database
 ```
 
-Load three instances of your container, load-balanced by _Nginx_, with dependecies:
+### Images
+
+Image tags are generated using the following convention:
+
 ```
-make start instances=3
+# See .orchestration.yml for `organization` and `repository` values.
+<organization>/<repository>:<git-commit-hash>
+
+# e.g.
+acme/anvil:abcd1234
 ```
 
-Or deploy to _Docker Swarm_:
+#### Build an application image
+
+Note that `git archive` is used to generate the build context. Any uncommitted changes will _not_ be included in the image.
 ```
-make deploy
+make build
 ```
 
-## Table of Contents
-
-<!-- toc -->
+See [build environment](#markdown-header-build-environment) for more details.
 
-- [Configuration Files](#configuration-files)
-  * [Makefile](#makefile)
-  * [.orchestration.yml](#orchestrationyml)
-  * [.env](#env)
-  * [orchestration/Dockerfile](#orchestrationdockerfile)
-  * [orchestration/entrypoint.sh](#orchestrationentrypointsh)
-  * [orchestration/docker-compose.yml](#orchestrationdocker-composeyml)
-  * [config/unicorn.rb](#configunicornrb)
-- [Building](#building)
-- [Build Environment](#build-environment)
-- [Commands](#commands)
-- [Dependencies](#dependencies)
+#### Push latest image
 
-<!-- tocstop -->
+You must be logged in to a _Docker_ registry. Use the `docker login` command (see [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/) for further reference).
 
-## Configuration Files
+```
+make push
+```
 
-_Orchestration_ autogenerates boilerplate configuration based on your application's requirements and configuration.
+### Development
 
-When supported dependencies are detected they will be created as services in your _Docker Compose_ configurations ready for use in testing, development, and production.
+An [`.env`](https://docs.docker.com/compose/env-file/) is created automatically in your project root. This file is _not_ stored in version control. Set all application environment variables in this file.
 
-The following files are created on setup:
+#### Launching a development server
 
-### Makefile
+To load all variables from `.env` and launch a development server, run the following command:
 
-Contains an `include` for the main _Orchestration_ `Makefile`. If this file already exists then the `include` will be added to the top of the file.
+```bash
+make serve
+```
 
-### .orchestration.yml
+The application environment will be output on launch for convenience.
 
-_Orchestration_-specific configuration such as your _Docker_ registry and username.
+To pass extra commands to the _Rails_ server:
+```bash
+# Custom server, custom port
+make serve server='webrick -p 3001'
 
-### .env
+# Default server, custom port, custom bind address
+make serve server='-p 3001 -b 192.168.0.1'
+```
 
-Specify any environment variables (e.g. `SECRET_KEY_BASE`) your application will need to run in production mode.
+### Testing
 
-The following two variables _must_ be defined:
+A default `test` target is provided in your application's main `Makefile`. You are encouraged to modify this target to suit your application's requirements.
 
-```bash
-VIRTUAL_HOST=localhost
-LISTEN_PORT=3000
+To launch all dependency containers, run database migrations, and run tests:
+```
+make test
 ```
 
-When running in production mode your application will be load-balanced by _Nginx_ proxy and available at http://localhost:3000/
+Note that _Orchestration_ will wait for all services to become fully available (i.e. running and providing valid responses) before attempting to run tests. This is specifically intended to facilitate testing in continuous integration environments.
 
-Take a look at `orchestration/docker-compose.production.yml` to see what variables will be exposed to various containers.
+### Deployment to Docker Swarm
 
-### orchestration/Dockerfile
+To deploy your application to _Docker Swarm_:
+```
+make deploy manager=user@manager.swarm.example.com
+```
 
-The basic requirements of a typical _Rails_ application. It is optimised for build speed and will automatically build assets (with our without `Webpacker`).
+To use a custom `.env` file:
+```
+make deploy env_file=/path/to/.env manager=user@manager.swarm.example.com
+```
 
-### orchestration/entrypoint.sh
+Note that the following two variables _must_ be set in the relevant `.env` file (will look in the current working directory if no path provided):
 
-Entrypoint script to handle user switching, permissions, stale pidfiles, etc.
+```
+# Published port for your application service:
+CONTAINER_PORT=3000
 
-### orchestration/docker-compose.yml
+# Number of replicas of your application service:
+REPLICAS=5
+```
 
-Along with the base `docker-compose.yml` a separate configuration is created for each environment. An override file is also generated.
+It is also recommended to set `SECRET_KEY_BASE` etc. in this file.
 
-See related documentation:
+## Build Environment
 
-https://docs.docker.com/compose/extends/
+The following environment variables will be passed as `ARG` variables when building images:
 
-* `orchestration/docker-compose.yml`
-* `orchestration/docker-compose.test.yml`
-* `orchestration/docker-compose.development.yml`
-* `orchestration/docker-compose.production.yml`
-* `orchestration/docker-compose.override.yml`
+```
+BUNDLE_BITBUCKET__ORG
+BUNDLE_GITHUB__COM
+```
 
-You can modify these files to suit your requirements.
+Set these variables in your shell if your `Gemfile` references privately-hosted gems on either _Bitbucket_ or _GitHub_.
 
-The famous [`jwilder/nginx-proxy`](https://github.com/jwilder/nginx-proxy) is used to load-balance replicas of your application when running in production.
+See related documentation:
 
-### config/unicorn.rb
+* https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
+* https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html
 
-If not already present, a [Unicorn](https://bogomips.org/unicorn/) configuration will be created. This is the default server when running in production.
+## Healthchecks
 
-## Building
+[Healthchecks](https://docs.docker.com/engine/reference/builder/#healthcheck) are automatically configured for your application. A healthcheck utility is provided in `orchestration/healthcheck.rb`. The following environment variables can be configured (in the `app` service of `orchestration/docker-compose.production.yml`):
 
-_Orchestration_ provides tools for building your application as a _Docker_ image.
+| Variable | Meaning | Default Value |
+|-|-|-|
+| `WEB_HOST` | Host to reach application (relevant to application container) | `localhost` |
+| `WEB_PORT` | Port to reach application (relevant to application container) | `8080` |
+| `WEB_HEALTHCHECK_PATH` | Path of expected successful response | `/` |
+| `WEB_HEALTHCHECK_READ_TIMEOUT` | Number of seconds to wait for data before failing healthcheck | `10` |
+| `WEB_HEALTHCHECK_OPEN_TIMEOUT` | Number of seconds to wait for connection before failing healthcheck | `10` |
+| `WEB_HEALTHCHECK_SUCCESS_CODES` | Comma-separated list of HTTP status codes that will be deemed a success | `200,202,204` |
 
-```bash
-make build
+If your application does not have a suitable always-available route to use as a healthcheck, the following one-liner may be useful:
+
+```ruby
+# config/routes.rb
+get '/healthcheck', to: proc { [200, { 'Content-Type' => 'text/html' }, ['']] }
 ```
 
-Running `make build` does the following:
+In this case, `WEB_HEALTHCHECK_PATH` must be set to `/healthcheck`.
 
-* Takes a snapshot of your application from current _Git_ `HEAD`. Only committed files are included.
-* Copies your `Gemfile` and installs your bundle (optimised for _Docker_ image caching).
-* Tags your image with your configured username/organisation, repository, and the current commit hash (abbreviated) of `HEAD`, e.g. `myorg/myapp:abc123`
+## Entrypoint
 
-Your image can then be pushed to your configured registry (use `docker login` before running):
+An [entrypoint](https://docs.docker.com/engine/reference/builder/#entrypoint) script for your application is provided which does the following:
 
-```
-make push
-```
+* Runs the `CMD` process as the same system user that launched the container (rather than the default `root` user);
+* Creates various required temporary directories and removes stale `pid` files;
+* Adds a route `host.docker.internal` to the host machine running the container (mimicking the same route provided by _Docker_ itself on _Windows_ and _OS
+  X_).
 
-## Build Environment
+## RabbitMQ Configuration
 
-The following environment variables will be passed as `ARG` variables when building your image:
+The [Bunny](https://github.com/ruby-amqp/bunny) _RabbitMQ_ gem does not recognise `config/rabbitmq.yml`. If your application uses _RabbitMQ_ then you must manually update your code to reference this file, e.g.:
 
-```
-BUNDLE_BITBUCKET__ORG
-BUNDLE_GITHUB__COM
+```ruby
+connection = Bunny.new(config_for(:rabbit_mq)['url'])
+connection.start
 ```
 
-See related documentation:
+The environment variable `RABBITMQ_URL` can be used to configure _Bunny_ in production (similar to `DATABASE_URL` and `MONGO_URL`).
 
-* https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
-* https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html
+This is a convention of the _Orchestration_ gem intended to make _RabbitMQ_ configuration consistent with other services.
 
-## Commands
+## License
 
-_Orchestration_ provides a number of `make` commands to help you work with your application in _Docker_.
+[MIT License](LICENSE)
 
-All commands respect `RAILS_ENV` or `RACK_ENV`. Alternatively you can pass `env` to any command:
-```bash
-make config env=production
-```
-
-The following commands are implemented:
-
-| Command | Description |
-|---|---|
-| `start` | Start all containers and wait for their services to become available. In production mode, pass `instances=N` to start `N` replicas of your app. |
-| `stop` | Stop all containers. |
-| `logs` | Tail logs for all containers. |
-| `config` | Output the full configuration for the current environment with all variables substituted. |
-| `compose` | Output full `docker-compose` command. Run arbitrary commands for your environment, e.g. `$(make compose env=test) ps --services` |
-| `test-setup` | Launch test dependency containers, wait for them to become ready, run database migrations. Call before running tests in a CI environment. |
-| `wait` | Wait for all dependencies to be ready (i.e. verify that database is up and accepting connections, etc.). |
-| `wait-database` | Wait for database container (supported: _PostgreSQL_ and _MySQL_) to become available. |
-| `wait-mongo` | Wait for _Mongo_ container to become available.
-| `wait-rabbitmq` | Wait for _RabbitMQ_ container to become available. |
-| `wait-nginx_proxy` | Wait for _Nginx_ container to become available (`production` only). |
-| `wait-app` | Wait for main application container to become available (`production` only). |
-| `build` | Build your application as a _Docker_ image. |
-| `push` | Push the current version of your application image to a _Docker_ registry. |
-| `bundle` | Create `deploy.tar` which contains pre-cooked production configurations and `Makefile` ready to deploy your application on any machine with _Docker_ and _Docker Compose_ installed.
-
-## Dependencies
-
-Dependencies are automatically detected. The following services are currently supported:
-
-| Service | Configuration File |
-|---|---|
-| _PostrgeSQL_ | `config/database.yml` |
-| _MySQL_ | `config/database.yml` |
-| _RabbitMQ_ | `config/rabbitmq.yml` |
-| _Mongo_ | `config/mongoid.yml` |
-
-Running `bin/rake orchestration:install` will automatically add services to your _Compose_ configurations that reflect your configuration files.
-
-For _RabbitMQ_, `config/rabbitmq.yml` should contain `host` and `port` fields for each environment.
+## Contributing
+
+Feel free to make a pull request. Use `make test` to ensure that all tests, _Rubocop_ checks, and dependency validations pass correctly.

data/Rakefile

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require 'bundler/gem_tasks'
-require 'rspec/core/rake_task'
+require 'rspec/core/rake_task' unless ENV.key?('ORCHESTRATION_TOOLKIT_ONLY')
 
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:spec) if defined?(RSpec)
 
 task default: :spec

data/config/locales/en.yml

@@ -12,6 +12,7 @@ en:
       waiting: "Waiting for database: %{config}"
       ready: "Database is ready."
       unknown_environment: "Environment not defined in database configuration: %{environment}"
+      unknown_adapter: "Unable to determine adapter from database.yml: %{adapter}"
 
     mongo:
       waiting: "Waiting for Mongo: %{config}"
@@ -33,7 +34,7 @@ en:
     settings:
       docker:
         organization:
-          description: "Registry organization/username"
+          description: "Docker registry organization/username"
           prompt: "organization"
 
         repository:
@@ -43,6 +44,7 @@ en:
     rake:
       app:
         wait: "Wait for app to become available"
+        unknown_web_server: "Unrecognised web server '%{server}'. Expected: %{expected}"
 
       database:
         wait: "Wait for database to become available"