chatops_deployer 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2d2860bee470087ff7e2dca10b45174989c32e31
4
+ data.tar.gz: ea456212c652d3579008e9cf2848f9454d6b5eae
5
+ SHA512:
6
+ metadata.gz: 1f2dbe5436e077a30b0d0128c3f988c44736764190efb882ba4bbad828bd7294e7ac5748e4268a340f96843011a7c061cd37c688e1b43eaae9f2abb9e4b2819d
7
+ data.tar.gz: 31976f8bde5739bbc503bc5040b4690e31e078bd6c7ebb2954e9c69fba7a5550f9d684d27a9a34bb1762ed2e660a8b6f904dffa314e94ee29773dff9fb83eb6a
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --require spec_helper
@@ -0,0 +1,14 @@
1
+ sudo: false # Use a container for faster boot time
2
+ language: ruby
3
+ bundler_args: "--without exclude_in_travis"
4
+ rvm:
5
+ - 2.1.3
6
+ - 2.2.2
7
+ script: bundle exec rake
8
+ notifications:
9
+ hipchat:
10
+ rooms:
11
+ secure: KdhzB9VueLEJp9p/ZUFaXAMO0u1CEbT0sjdRt10jqseAJCld9CAK1KAb1f/mNO/+cbVd+7W9UdHA0cEGUZ9dLP9Mcpmy31QxOJSITUCrRiLOF1ti13Lq4WHpMM46Tz6p7vVYd4zpPj9Mup+lCTGOeCovReX+neIoZw0UjDg9oGs=
12
+ template:
13
+ - '%{result}: %{repository_slug}#%{branch} - <a href="%{compare_url}">%{commit}</a>(%{commit_message}) by %{author} (<a href="%{build_url}">Details</a>)'
14
+ format: html
data/Gemfile ADDED
@@ -0,0 +1,12 @@
1
+ source 'https://rubygems.org'
2
+
3
+ group :exclude_in_travis do
4
+ gem 'pry-byebug'
5
+ end
6
+
7
+ gem 'rake'
8
+ gem 'rspec'
9
+ gem 'memfs'
10
+ gem 'webmock'
11
+
12
+ gemspec
@@ -0,0 +1,85 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ chatops_deployer (0.1.0)
5
+ haikunator (~> 1.1)
6
+ httparty (~> 0.13)
7
+ sinatra (~> 1.4)
8
+ sucker_punch (~> 1.5)
9
+ vault (~> 0.1)
10
+
11
+ GEM
12
+ remote: https://rubygems.org/
13
+ specs:
14
+ addressable (2.3.8)
15
+ byebug (4.0.5)
16
+ columnize (= 0.9.0)
17
+ celluloid (0.16.0)
18
+ timers (~> 4.0.0)
19
+ coderay (1.1.0)
20
+ columnize (0.9.0)
21
+ crack (0.4.2)
22
+ safe_yaml (~> 1.0.0)
23
+ diff-lcs (1.2.5)
24
+ haikunator (1.1.0)
25
+ hitimes (1.2.2)
26
+ httparty (0.13.5)
27
+ json (~> 1.8)
28
+ multi_xml (>= 0.5.2)
29
+ json (1.8.3)
30
+ memfs (0.4.3)
31
+ method_source (0.8.2)
32
+ multi_xml (0.5.5)
33
+ pry (0.10.1)
34
+ coderay (~> 1.1.0)
35
+ method_source (~> 0.8.1)
36
+ slop (~> 3.4)
37
+ pry-byebug (3.1.0)
38
+ byebug (~> 4.0)
39
+ pry (~> 0.10)
40
+ rack (1.6.4)
41
+ rack-protection (1.5.3)
42
+ rack
43
+ rake (10.4.2)
44
+ rspec (3.3.0)
45
+ rspec-core (~> 3.3.0)
46
+ rspec-expectations (~> 3.3.0)
47
+ rspec-mocks (~> 3.3.0)
48
+ rspec-core (3.3.2)
49
+ rspec-support (~> 3.3.0)
50
+ rspec-expectations (3.3.1)
51
+ diff-lcs (>= 1.2.0, < 2.0)
52
+ rspec-support (~> 3.3.0)
53
+ rspec-mocks (3.3.2)
54
+ diff-lcs (>= 1.2.0, < 2.0)
55
+ rspec-support (~> 3.3.0)
56
+ rspec-support (3.3.0)
57
+ safe_yaml (1.0.4)
58
+ sinatra (1.4.6)
59
+ rack (~> 1.4)
60
+ rack-protection (~> 1.4)
61
+ tilt (>= 1.3, < 3)
62
+ slop (3.6.0)
63
+ sucker_punch (1.5.1)
64
+ celluloid (= 0.16.0)
65
+ tilt (2.0.1)
66
+ timers (4.0.4)
67
+ hitimes
68
+ vault (0.1.5)
69
+ webmock (1.21.0)
70
+ addressable (>= 2.3.6)
71
+ crack (>= 0.3.2)
72
+
73
+ PLATFORMS
74
+ ruby
75
+
76
+ DEPENDENCIES
77
+ chatops_deployer!
78
+ memfs
79
+ pry-byebug
80
+ rake
81
+ rspec
82
+ webmock
83
+
84
+ BUNDLED WITH
85
+ 1.10.6
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 TODO: Write your name
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,160 @@
1
+ # ChatopsDeployer
2
+
3
+ A lightweight Sinatra app that deploys staging apps of git branches
4
+ in docker containers. Meant to be used with hubot.
5
+
6
+ ## Requirements
7
+
8
+ **All commands need to be run as the root user**
9
+ So it's best if you can run this on a dedicated disposable server.
10
+
11
+ 1. virtualbox - For creating isolated VMs for each project
12
+ 3. docker-machine - For starting docker daemons on VMs
13
+ 2. docker-compose - For running multi-container apps using docker daemons on VMs
14
+ 4. nginx - For setting up a subdomain for each deployment
15
+
16
+ TODO: setup script to install requirements on Ubuntu 14.04
17
+
18
+ ## Installation
19
+
20
+ $ gem install chatops_deployer
21
+
22
+ ## Usage
23
+
24
+ Set the following ENV vars:
25
+
26
+ ```bash
27
+ export DEPLOYER_HOST=<hostname where nginx listens>
28
+ export WORKSPACE=<path where you want your projects to be git-cloned> # default: '/var/www'
29
+ export NGINX_SITES_ENABLED_DIR=<path to sites-enabled directory in nginx conf> # default: '/etc/nginx/sites-enabled'
30
+ export COPY_SOURCE_DIR = <path to directory containing source files to be copied over to projects> # default: '/etc/chatops_deployer/copy'
31
+ export DEPLOYER_REGISTRY_MIRROR = <URL of docker registry mirror if you want to make use of faster docker image pulls> # default: nil
32
+
33
+ # Optional to use Vault for managing and distributing secrets
34
+ export VAULT_ADDR= <address where vault server is listening>
35
+ export VAULT_TOKEN= <token which can read keys stored under path secret/*>
36
+ export VAULT_CACERT= <CA certificate file to verify vault server SSL certificate>
37
+ ```
38
+ And run the server as the root user:
39
+
40
+ $ chatops_deployer
41
+
42
+ ### Configuration
43
+
44
+ To configure an app for deployment using chatops_deployer API, you need to follow the following steps:
45
+
46
+ #### 1. Dockerize the app
47
+
48
+ Add a `docker-compose.yml` file inside the root of the app that can run the app
49
+ and the dependent services as docker containers using the command `docker-compose up`.
50
+ Refer [the docker compose docs](https://docs.docker.com/compose/) to learn how
51
+ to prepare this file for your app.
52
+
53
+ #### 2. Add chatops_deployer.yml
54
+
55
+ Add a `chatops_deployer.yml` file inside the root of the app.
56
+ This file will tell `chatops_deployer` about ports to expose as haikunated
57
+ subdomains, commands to run after cloning the repository and also if any files
58
+ need to be copied into the project after cloning it for any runtime configuration.
59
+
60
+ Here's an example `chatops_deployer.yml` :
61
+
62
+ ```yaml
63
+ # `expose` is a hash in the format <service>:<array of ports>
64
+ # <service> : Service name as specified in docker-compose.yml
65
+ # <array of ports> : Ports on the container which should be exposed as subdomains
66
+ expose:
67
+ web: [3000]
68
+
69
+ # `commands` is a list of commands that should be run inside a service container
70
+ # before all systems are go.
71
+ # Commands are run in the same order as they appear in the list.
72
+ commands:
73
+ - [db, "./setup_script_in_container"]
74
+ - [web, "bundle exec rake db:create"]
75
+ - [web, "bundle exec rake db:schema:load"]
76
+
77
+ # `copy` is an array of strings in the format "<source>:<destination>"
78
+ # If source begins with './' , the source file is searched from the root of the cloned
79
+ # repo, else it is assumed to be a path to a file relative to
80
+ # /etc/chatops_deployer/copy in the deployer server.
81
+ # destination is the path relative to chatops_deployer.yml to which the source file
82
+ # should be copied. Copying of files happen soon after the repository is cloned
83
+ # and before any docker containers are created.
84
+ # If the source file ends with .erb, it's treated as an ERB template and gets
85
+ # processed. You have access to the following objects inside the ERB templates:
86
+ # "env", "vault"
87
+ #
88
+ # "env" holds the exposed urls. For example:
89
+ # "<%= env['urls']['web']['3000'] %>" will be replaced with "http://crimson-cloud-12.example.com"
90
+ #
91
+ # "vault" can be used to access secrets managed using Vault if you have set it up
92
+ # "<%= vault.read('secret/app-name/AWS_SECRET_KEY', 'value') %>" will be replaced with the secret key fetched from Vault
93
+ # using the command `vault read -field=value secret/app-name/AWS_SECRET_KEY`
94
+ copy:
95
+ - "./config.dev.env.erb:config.env"
96
+
97
+ # `cache` is a hash in the format <directory_in_code>: {<service>: <directory_in_service>}
98
+ # <directory_in_code> is a directory under the root of the cloned repo
99
+ # where a cached directory is created.
100
+ # <service> is the name of a service which will have the cached directory in its container.
101
+ # <directory_in_service> is the absolute path of the cached directory inside the running service.
102
+ # The `cache` option allows you to share data among deployments (for faster deployments).
103
+ # Before every deployment, each cache directory is mounted under the cloned repo.
104
+ # These directories can then be used during docker build. Once the app is deployed,
105
+ # the cache directories are updated with their latest content from the running
106
+ # containers, which will be used for subsequent deployments.
107
+ cache:
108
+ - tmp/bundler
109
+ - node_modules
110
+ ```
111
+
112
+ ### Deployment
113
+
114
+ To deploy an app using `chatops_deployer`, send a POST request to `chatops_deployer`
115
+ like so :
116
+
117
+ ```
118
+ curl -XPOST -d '{"repository":"https://github.com/user/app.git","branch":"master","callback_url":"example.com/deployment_status"}' -H "Content-Type: application/json" localhost:8000/deploy
119
+ ```
120
+
121
+ You can see that the request accepts a `callback_url`. chatops_deployer will
122
+ POST to this callback_url with the following data:
123
+
124
+ 1. Success callback
125
+
126
+ Example:
127
+ ```json
128
+ {
129
+ status: 'deployment_success',
130
+ branch: 'master',
131
+ urls: { web: ['misty-meadows-123.deployer-host.com'] }
132
+ }
133
+ ```
134
+
135
+ 2. Failure callback
136
+
137
+ Example:
138
+ ```json
139
+ {
140
+ status: 'deployment_failure',
141
+ branch: 'master',
142
+ reason: 'f052f10148bd290321b84f44: Nginx error: Config directory /etc/nginx/sites-enabled does not exist'
143
+ }
144
+ ```
145
+
146
+ ## Development
147
+
148
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
149
+
150
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
151
+
152
+ ## Contributing
153
+
154
+ Bug reports and pull requests are welcome on GitHub at https://github.com/code-mancers/chatops_deployer.
155
+
156
+
157
+ ## License
158
+
159
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
160
+
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new
5
+
6
+ task default: [:spec]
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,27 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'chatops_deployer/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "chatops_deployer"
8
+ spec.version = ChatopsDeployer::VERSION
9
+ spec.authors = ["Emil Soman"]
10
+ spec.email = ["emil@codemancers.com"]
11
+
12
+ spec.summary = %q{An opinionated Chatops backend}
13
+ spec.description = %q{ChatopsDeployer deploys containerized services in isolated VMs and exposes public facing URLs}
14
+ spec.homepage = "https://github.com/code-mancers/chatops-deployer"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ spec.bindir = "exe"
19
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
20
+ spec.require_paths = ["lib"]
21
+
22
+ spec.add_dependency "sinatra", "~> 1.4"
23
+ spec.add_dependency "sucker_punch", "~> 1.5"
24
+ spec.add_dependency "httparty", "~> 0.13"
25
+ spec.add_dependency "haikunator", "~> 1.1"
26
+ spec.add_dependency "vault", "~> 0.1"
27
+ end
@@ -0,0 +1,106 @@
1
+ Using Vault for secrets management
2
+ ==================================
3
+
4
+ [Vault](https://vaultproject.io/) is an awesome tool to securely store and
5
+ distribute secrets among your apps. The flow that we'll use for our deployer
6
+ is as follows:
7
+
8
+ ## 1. Set up Vault server
9
+
10
+ The admin who sets up chatops_deployer will need to set up Vault and run the
11
+ Vault server.
12
+ Follow the [Vault deployment guide](https://vaultproject.io/intro/getting-started/deploy.html).
13
+
14
+ You'll have to generate a TLS key and cert first.
15
+
16
+ ```
17
+ openssl genrsa 1024 > vault.key
18
+ chmod 400 vault.key
19
+ openssl req -new -x509 -nodes -sha1 -days 365 -key vault.key > vault.crt
20
+ ```
21
+ Pass the `vault.crt` file to the person who'll be writing secrets.
22
+
23
+ Then use the following config file when starting the server :
24
+
25
+ ```
26
+ backend "file" {
27
+ path = "secret"
28
+ }
29
+
30
+ listener "tcp" {
31
+ address = "0.0.0.0:8200"
32
+ tls_cert_file = "vault.crt"
33
+ tls_key_file = "vault.key"
34
+ }
35
+ ```
36
+
37
+ Follow rest of the guide to complete the deployment. Please make sure you
38
+ note down the "Initial Root Token" which you get after `vault init` step.
39
+
40
+ Export the following ENV vars and start chatops_deployer
41
+ ```
42
+ export VAULT_ADDR= <address where vault server is listening>
43
+ export VAULT_TOKEN= <token which can read keys stored under path secret/*>
44
+ export VAULT_CACERT= <CA certificate file to verify vault server SSL certificate>
45
+ ```
46
+ ## 2. Setup for each environment
47
+
48
+ ### 1. Generate an access policy
49
+
50
+ Use the following template to create an ACL
51
+
52
+ Contents of `myapp-staging.hcl`
53
+ ```
54
+ path "secret/myapp-staging/*" {
55
+ policy = "write"
56
+ }
57
+ ```
58
+
59
+ Use the above policy file to create a policy :
60
+
61
+ ```
62
+ vault policy-write myapp-staging myapp-staging.hcl
63
+ ```
64
+
65
+ ### 2. Generate a token using policy
66
+
67
+ ```
68
+ vault token-create -display-name="myapp-staging" -policy="myapp-staging"
69
+ ```
70
+
71
+ Pass this token to the person who'll be setting the secrets.
72
+
73
+ ## 3. Write secrets
74
+
75
+ The developer or the person who's to set the secrets will need to install
76
+ Vault first. Then set the following ENV vars :
77
+
78
+ export VAULT_ADDR=https://<vault server url>
79
+ export VAULT_CACERT=vault.crt
80
+ export VAULT_TOKEN=<token generated in step 2>
81
+
82
+ Write secrets with the following command :
83
+
84
+ ```
85
+ vault write secret/myapp-staging/SECRET_KEY value=SECRET_VALUE
86
+ ```
87
+
88
+ ## 4. Use secrets in `copy` files
89
+
90
+ In `chatops_deployer.yml`, use `copy` option to write a config file using the
91
+ secret. For example:
92
+
93
+ ```
94
+ # chatops_deployer.yml
95
+ copy:
96
+ - "./config/secrets.staging.yml.erb:config/secrets.yml"
97
+ ```
98
+
99
+ ```
100
+ # config/secrets.staging.yml.erb
101
+ staging:
102
+ SECRET_KEY: <%= vault.read('secret/myapp-staging/SECRET_KEY', 'value') %>
103
+ ```
104
+
105
+ chatops_deployer will expand the ERB tag by reading the secret from Vault and
106
+ write the file to the specified destination, ie, `config/secrets.yml`
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ begin
4
+ require 'chatops_deployer/app.rb'
5
+ rescue LoadError => e
6
+ require 'rubygems'
7
+ path = File.expand_path '../../lib', __FILE__
8
+ $:.unshift(path) if File.directory?(path) && !$:.include?(path)
9
+ require 'chatops_deployer/app.rb'
10
+ end
@@ -0,0 +1,35 @@
1
+ require 'sinatra'
2
+ require 'net/http'
3
+ require 'json'
4
+ require 'chatops_deployer/deploy_job'
5
+ require 'fileutils'
6
+
7
+ module ChatopsDeployer
8
+ class App < Sinatra::Base
9
+ set :port, 8000
10
+ set :bind, '0.0.0.0'
11
+
12
+ configure do
13
+ [WORKSPACE, COPY_SOURCE_DIR].each do |dir|
14
+ FileUtils.mkdir_p dir unless Dir.exists?(dir)
15
+ end
16
+ end
17
+
18
+ post '/deploy' do
19
+ content_type :json
20
+ json = JSON.parse(request.body.read)
21
+
22
+ DeployJob.new.async.perform(repository: json['repository'], branch: json['branch'], callback_url: json['callback_url'])
23
+ { log_url: LOG_URL }.to_json
24
+ end
25
+
26
+ post '/destroy' do
27
+ content_type :json
28
+ json = JSON.parse(request.body.read)
29
+
30
+ DestroyJob.new.async.perform(repository: json['repository'], branch: json['branch'], callback_url: json['callback_url'])
31
+ end
32
+ end
33
+ end
34
+
35
+ ChatopsDeployer::App.run!
@@ -0,0 +1,33 @@
1
+ require 'logger'
2
+ require 'open3'
3
+
4
+ module ChatopsDeployer
5
+ class Command
6
+ attr_reader :output
7
+
8
+ def self.run(command: "", logger: ::Logger.new(STDOUT))
9
+ new.run(command, logger)
10
+ end
11
+
12
+ def initialize
13
+ @output = ""
14
+ @status = nil
15
+ end
16
+
17
+ def run(command, logger)
18
+ logger.info "Running command: #{command.inspect}"
19
+ Open3.popen2e(*(Array(command))) do |_, out_err, thread|
20
+ out_err.each_line do |line|
21
+ logger.info line
22
+ @output << line
23
+ end
24
+ @status = thread.value
25
+ end
26
+ self
27
+ end
28
+
29
+ def success?
30
+ @status && @status.success?
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,107 @@
1
+ require 'chatops_deployer/error'
2
+ require 'chatops_deployer/command'
3
+ require 'chatops_deployer/globals'
4
+ require 'chatops_deployer/logger'
5
+ require 'yaml'
6
+
7
+ module ChatopsDeployer
8
+ class Container
9
+ include Logger
10
+ class Error < ChatopsDeployer::Error; end
11
+
12
+ attr_reader :urls
13
+ def initialize(project)
14
+ @sha1 = project.sha1
15
+ @urls = {}
16
+ @project = project
17
+ end
18
+
19
+ def build
20
+ @config = @project.config
21
+ create_docker_machine
22
+ setup_docker_environment
23
+ docker_compose_run_commands
24
+ docker_compose_up
25
+ end
26
+
27
+ def destroy
28
+ raise_error("Cannot destroy VM because it doesn't exist") unless vm_exists?
29
+ destroy_vm
30
+ end
31
+
32
+ private
33
+
34
+ def create_docker_machine
35
+ if vm_exists?
36
+ logger.info "VM for the branch already exists. Destroying it."
37
+ Command.run(command: "docker-machine rm #{@sha1}", logger: logger)
38
+ end
39
+ logger.info "Creating VM #{@sha1}"
40
+ mirror_config = REGISTRY_MIRROR ? " --engine-registry-mirror=#{REGISTRY_MIRROR}" : ""
41
+ Command.run(command: "docker-machine create --driver virtualbox #{@sha1}#{mirror_config}", logger: logger)
42
+ get_ip = Command.run(command: "docker-machine ip #{@sha1}", logger: logger)
43
+ unless get_ip.success?
44
+ raise_error('Cannot create VM for running docker containers')
45
+ end
46
+ @ip = get_ip.output.chomp
47
+ end
48
+
49
+ def setup_docker_environment
50
+ logger.info "Setting up docker environment for #{@sha1}"
51
+ docker_env = Command.run(command: "docker-machine env #{@sha1}", logger: logger)
52
+ raise_error('Cannot set docker environment variables') unless docker_env.success?
53
+
54
+ matches = []
55
+ docker_env.output.scan(/export (?<env_key>.*)="(?<env_value>.*)"\n/){ matches << $~ }
56
+ matches.each do |match|
57
+ ENV[match[:env_key]] = match[:env_value]
58
+ end
59
+ end
60
+
61
+ def docker_compose_run_commands
62
+ logger.info "Running commands on containers"
63
+ commands = @config['commands']
64
+ commands.each do |service_commands|
65
+ service = service_commands[0]
66
+ command = service_commands[1]
67
+ docker_compose_run = Command.run(command: "docker-compose run #{service} #{command}", logger: logger)
68
+ raise_error("docker-compose run #{service} #{command} failed") unless docker_compose_run.success?
69
+ end
70
+ end
71
+
72
+ def docker_compose_up
73
+ logger.info "Running docker-compose up"
74
+ docker_compose = Command.run(command: 'docker-compose up -d', logger: logger)
75
+ raise_error('docker-compose up failed') unless docker_compose.success?
76
+
77
+ if expose = @config['expose']
78
+ expose.each do |service, ports|
79
+ @urls[service] = ports.collect do |port|
80
+ get_url_on_vm(service, port)
81
+ end
82
+ end
83
+ end
84
+ end
85
+
86
+ def vm_exists?
87
+ Command.run(command: "docker-machine url #{@sha1}", logger: logger).success?
88
+ end
89
+
90
+ def destroy_vm
91
+ logger.info "Destroying VM #{@sha1}"
92
+ system("docker-machine stop #{@sha1}") &&
93
+ system("docker-machine rm #{@sha1}")
94
+ end
95
+
96
+ def get_url_on_vm(service, port)
97
+ docker_port = Command.run(command: "docker-compose port #{service} #{port}", logger: logger)
98
+ raise_error("Cannot find exposed port for #{port} in service #{service}") unless docker_port.success?
99
+ port = docker_port.output.chomp.split(':').last
100
+ [@ip, port]
101
+ end
102
+
103
+ def raise_error(message)
104
+ raise Error, "Container error: #{message}"
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,58 @@
1
+ require 'sucker_punch'
2
+ require 'fileutils'
3
+ require 'httparty'
4
+ require 'chatops_deployer/globals'
5
+ require 'chatops_deployer/project'
6
+ require 'chatops_deployer/nginx_config'
7
+ require 'chatops_deployer/container'
8
+ require 'chatops_deployer/logger'
9
+
10
+ module ChatopsDeployer
11
+ class DeployJob
12
+ include SuckerPunch::Job
13
+
14
+ def perform(repository:, branch: 'master', config_file: 'chatops_deployer.yml', callback_url:)
15
+ @branch = branch
16
+ @project = Project.new(repository, branch, config_file)
17
+ log_file = File.open(LOG_FILE, 'a')
18
+ @logger = ::Logger.new(MultiIO.new($stdout, log_file)).tap do |l|
19
+ l.progname = @project.sha1
20
+ end
21
+
22
+ @nginx_config = NginxConfig.new(@project)
23
+ @container = Container.new(@project)
24
+ [@project, @nginx_config, @container].each do |obj|
25
+ obj.logger = @logger
26
+ end
27
+
28
+ Dir.chdir(@project.branch_directory) do
29
+ @project.fetch_repo
30
+ @project.read_config
31
+ @nginx_config.prepare_urls
32
+ @project.copy_files_from_deployer
33
+ @project.setup_cache_directories
34
+ @container.build
35
+ @project.update_cache
36
+ end
37
+ @nginx_config.add_urls(@container.urls)
38
+ callback(callback_url, :deployment_success)
39
+ rescue ChatopsDeployer::Error => e
40
+ @logger.error(e.message)
41
+ callback(callback_url, :deployment_failure, e.message)
42
+ end
43
+
44
+ private
45
+
46
+ def callback(callback_url, status, reason=nil)
47
+ body = {status: status, branch: @branch}
48
+ if status == :deployment_success
49
+ body[:urls] = @nginx_config.readable_urls
50
+ @logger.info "Succesfully deployed #{@branch}"
51
+ else
52
+ body[:reason] = reason
53
+ @logger.info "Failed deploying #{@branch}. Reason: #{reason}"
54
+ end
55
+ HTTParty.post(callback_url, body: body.to_json, headers: {'Content-Type' => 'application/json'})
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,53 @@
1
+ require 'sucker_punch'
2
+ require 'fileutils'
3
+ require 'httparty'
4
+
5
+ module ChatopsDeployer
6
+ class DestroyJob
7
+ include SuckerPunch::Job
8
+
9
+ def perform(repository:, branch:, callback_url:)
10
+ git_basename = repository.split('/').last
11
+ project = File.basename(git_basename,File.extname(git_basename))
12
+ @branch = branch
13
+ @deployment_alias = "#{project}-#{branch}"
14
+ project_dir = "#{WORKSPACE}/#{project}/#{branch}"
15
+ puts "Removing #{project_dir}"
16
+ FileUtils.rm_rf project_dir
17
+
18
+ #TODO: No error conditions are handled in the following methods.
19
+ if remove_nginx_config && dockerdown
20
+ callback(callback_url, :destroy_success)
21
+ else
22
+ callback(callback_url, :destroy_failure)
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ def dockerdown
29
+ puts "Removing docker container #{@deployment_alias}"
30
+ system('docker', 'stop', @deployment_alias) &&
31
+ system('docker', 'rm', @deployment_alias)
32
+ end
33
+
34
+ def remove_nginx_config
35
+ nginx_config = File.join(NGINX_SITES_ENABLED_DIR, @deployment_alias)
36
+ return false if !File.exists?(nginx_config)
37
+
38
+ puts "Removing nginx config at #{nginx_config}"
39
+ File.delete(nginx_config)
40
+ system('service nginx reload')
41
+ end
42
+
43
+ def callback(callback_url, status)
44
+ body = {status: status, branch: @branch}
45
+ if status == :destroy_success
46
+ puts "Succesfully destroyed staging env of #{@branch}"
47
+ else
48
+ puts "Failed destroying staging env of #{@branch}"
49
+ end
50
+ HTTParty.post(callback_url, body: body.to_json, headers: {'Content-Type' => 'application/json'})
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,3 @@
1
+ module ChatopsDeployer
2
+ class Error < StandardError; end
3
+ end
@@ -0,0 +1,10 @@
1
+ module ChatopsDeployer
2
+ WORKSPACE = ENV['DEPLOYER_WORKSPACE'] || '/var/www'
3
+ DEPLOYER_HOST = ENV['DEPLOYER_HOST'] || '127.0.0.1.xip.io'
4
+ NGINX_SITES_ENABLED_DIR = ENV['NGINX_SITES_ENABLED_DIR'] || '/etc/nginx/sites-enabled'
5
+ LOG_FILE = ENV['DEPLOYER_LOG_FILE'] || '/var/log/chatops_deployer.log'
6
+ COPY_SOURCE_DIR = ENV['DEPLOYER_COPY_SOURCE_DIR'] || '/etc/chatops_deployer/copy'
7
+ LOG_URL = ENV['DEPLOYER_LOG_URL']
8
+ REGISTRY_MIRROR = ENV['DEPLOYER_REGISTRY_MIRROR']
9
+ end
10
+
@@ -0,0 +1,39 @@
1
+ require 'logger'
2
+
3
+ module ChatopsDeployer
4
+ class MultiIO
5
+ def initialize(*targets)
6
+ @targets = targets
7
+ end
8
+
9
+ def write(*args)
10
+ @targets.each{|t| t.write(*args); t.flush }
11
+ end
12
+
13
+ def close
14
+ @targets.each(&:close)
15
+ end
16
+ end
17
+
18
+ module Logger
19
+ def self.included(base)
20
+ class << base
21
+ def logger
22
+ @logger ||= ::Logger.new($stdout)
23
+ end
24
+
25
+ def logger=(logger)
26
+ @logger = logger
27
+ end
28
+ end
29
+ end
30
+
31
+ def logger
32
+ self.class.logger
33
+ end
34
+
35
+ def logger=(logger)
36
+ self.class.logger = logger
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,125 @@
1
+ require 'chatops_deployer/globals'
2
+ require 'chatops_deployer/error'
3
+ require 'chatops_deployer/command'
4
+ require 'haikunator'
5
+ require 'fileutils'
6
+ require 'chatops_deployer/logger'
7
+
8
+ module ChatopsDeployer
9
+ class NginxConfig
10
+ include Logger
11
+ attr_reader :urls
12
+
13
+ class Error < ChatopsDeployer::Error; end
14
+
15
+ def initialize(project)
16
+ @sha1 = project.sha1
17
+ @project = project
18
+ check_sites_enabled_dir_exists!
19
+ @config_path = File.join NGINX_SITES_ENABLED_DIR, @sha1
20
+ @urls = {}
21
+ end
22
+
23
+ def exists?
24
+ File.exists? @config_path
25
+ end
26
+
27
+ # service_urls is an array in the format:
28
+ # {"web" => [["10.1.1.2", "3000"],["10.1.1.2", "4000"]] }
29
+ def add_urls(service_urls)
30
+ return if service_urls.nil?
31
+ remove if exists?
32
+
33
+ service_urls.each do |service, internal_urls|
34
+ Array(internal_urls).each do |internal_url|
35
+ expose(service, internal_url)
36
+ end
37
+ end
38
+ logger.info "Reloading nginx"
39
+ nginx_reload = Command.run(command: 'service nginx reload', logger: logger)
40
+ unless nginx_reload.success?
41
+ raise_error("Cannot reload nginx after adding config. Check #{NGINX_SITES_ENABLED_DIR}/#{@sha1} for errors")
42
+ end
43
+ end
44
+
45
+ def remove
46
+ logger.info "Removing nginx config"
47
+ FileUtils.rm @config_path
48
+ system('service nginx reload')
49
+ end
50
+
51
+ def readable_urls
52
+ urls = {}
53
+ @urls.each do |service, port_exposed_urls|
54
+ urls[service] = port_exposed_urls.collect do |port, exposed_url|
55
+ "http://#{exposed_url}"
56
+ end
57
+ end
58
+ urls.to_json
59
+ end
60
+
61
+ def prepare_urls
62
+ @project.env['urls'] = {}
63
+ service_ports_from_config.each do |service, ports|
64
+ @urls[service] = {}
65
+ @project.env['urls'][service] = {}
66
+ ports.each do |port|
67
+ @urls[service][port.to_s] = generate_haikunated_url
68
+ @project.env['urls'][service][port.to_s] = "http://#{@urls[service][port.to_s]}"
69
+ end
70
+ end
71
+ end
72
+
73
+ private
74
+
75
+ def check_sites_enabled_dir_exists!
76
+ unless Dir.exist? NGINX_SITES_ENABLED_DIR
77
+ raise_error("Config directory #{NGINX_SITES_ENABLED_DIR} does not exist")
78
+ end
79
+ end
80
+
81
+ def service_ports_from_config
82
+ @project.config['expose'] || {}
83
+ end
84
+
85
+ def generate_haikunated_url
86
+ haiku = Haikunator.haikunate
87
+ "#{haiku}.#{DEPLOYER_HOST}"
88
+ end
89
+
90
+ # service => name of service , example: "web"
91
+ # internal_url => a pair of ip and port, example: ["10.1.1.2", "3000"]
92
+ def expose(service, internal_url)
93
+ raise_error("Cannot add nginx config because host is nil") if internal_url.nil? || internal_url.empty?
94
+ ip = internal_url[0]
95
+ port = internal_url[1]
96
+ begin
97
+ exposed_url = @urls[service][port]
98
+ rescue
99
+ raise_error("Cannot add nginx config because exposed ports could not be read from chatops_deployer.yml")
100
+ end
101
+ contents = <<-EOM
102
+ server{
103
+ listen 80;
104
+ server_name #{exposed_url};
105
+
106
+ # host error and access log
107
+ access_log /var/log/nginx/#{exposed_url}.access.log;
108
+ error_log /var/log/nginx/#{exposed_url}.error.log;
109
+
110
+ location / {
111
+ proxy_pass http://#{ip}:#{port};
112
+ }
113
+ }
114
+ EOM
115
+ logger.info "Adding nginx config at #{NGINX_SITES_ENABLED_DIR}/#{@sha1}"
116
+ File.open(@config_path, 'a') do |file|
117
+ file << contents
118
+ end
119
+ end
120
+
121
+ def raise_error(message)
122
+ raise Error, "Nginx error: #{message}"
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,118 @@
1
+ require 'chatops_deployer/globals'
2
+ require 'chatops_deployer/error'
3
+ require 'chatops_deployer/command'
4
+ require 'chatops_deployer/template'
5
+ require 'chatops_deployer/logger'
6
+ require 'digest/sha1'
7
+ require 'fileutils'
8
+ require 'yaml'
9
+
10
+ module ChatopsDeployer
11
+ class Project
12
+ include Logger
13
+ class Error < ChatopsDeployer::Error; end
14
+
15
+ attr_reader :sha1, :branch_directory, :config
16
+ attr_accessor :env
17
+ def initialize(repository, branch, config_file="chatops_deployer.yml")
18
+ @sha1 = Digest::SHA1.hexdigest(repository + branch)
19
+ @repository = repository
20
+ @branch = branch
21
+ @config_file = config_file
22
+ @env = {}
23
+ setup_project_directory
24
+ end
25
+
26
+ def fetch_repo
27
+ logger.info "Fetching #{@repository}:#{@branch}"
28
+ unless Dir.entries('.').size == 2
29
+ logger.info "Branch already cloned. Deleting everything before cloning again"
30
+ FileUtils.rm_rf '.'
31
+ end
32
+ logger.info "Cloning branch #{@repository}:#{@branch}"
33
+ git_clone = Command.run(command: ['git', 'clone', "--branch=#{@branch}", '--depth=1', @repository, '.'], logger: logger)
34
+ unless git_clone.success?
35
+ raise_error("Cannot clone git repository: #{@repository}, branch: #{@branch}")
36
+ end
37
+ end
38
+
39
+ def read_config
40
+ @config = if File.exists?(@config_file)
41
+ begin
42
+ YAML.load_file(@config_file) || {}
43
+ rescue StandardError => e
44
+ raise_error("Cannot parse YAML content in #{@config_file}")
45
+ end
46
+ else
47
+ {}
48
+ end
49
+ end
50
+
51
+ def copy_files_from_deployer
52
+ copy_list = @config['copy'].to_a
53
+ return if copy_list.empty?
54
+ logger.info "Copying files from deployer to project"
55
+ copy_list.each do |copy_string|
56
+ source, destination = copy_string.split(':')
57
+ # source is from COPY_SOURCE_DIR if source doesn't start with ./
58
+ source = File.join(COPY_SOURCE_DIR, source) unless source.match(/^\.\//)
59
+ if File.extname(source) == '.erb'
60
+ destination ||= File.basename(source, '.erb')
61
+ logger.info "Processing ERB template #{source} into #{destination}"
62
+ Template.new(source).inject(@env).write(destination)
63
+ else
64
+ destination ||= File.basename source
65
+ logger.info "Copying #{source} into #{destination}"
66
+ FileUtils.cp source, destination
67
+ end
68
+ end
69
+ end
70
+
71
+ def setup_cache_directories
72
+ cache_directory_list = @config['cache'].to_h
73
+ cache_directory_list.each do |directory, _|
74
+ cache_dir = File.join(@common_cache_dir, directory)
75
+ target_cache_dir = File.join(@branch_directory, directory)
76
+ FileUtils.mkdir_p cache_dir
77
+ FileUtils.mkdir_p target_cache_dir
78
+ FileUtils.rm_rf target_cache_dir
79
+ FileUtils.cp_r(cache_dir, target_cache_dir)
80
+ end
81
+ end
82
+
83
+ def update_cache
84
+ cache_directory_list = @config['cache'].to_h
85
+ cache_directory_list.each do |directory, service_paths|
86
+ service_paths.each do |service, path|
87
+ ps = Command.run(command: ["docker-compose", "ps", "-q", service], logger: logger)
88
+ container = ps.output.chomp
89
+ next if container.empty?
90
+ cache_dir = File.join(@common_cache_dir, directory)
91
+ tmp_dir = File.join(@project_directory, 'tmp_cache')
92
+ copy_from_container = Command.run(command: ["docker", "cp", "#{container}:#{path}", tmp_dir], logger: logger)
93
+ raise_error("Cannot copy '#{path}' from container '#{container}' of service '#{service}'") unless copy_from_container.success?
94
+ FileUtils.rm_rf(cache_dir)
95
+ FileUtils.mv(tmp_dir, cache_dir)
96
+ FileUtils.rm_rf(tmp_dir)
97
+ end
98
+ end
99
+ end
100
+
101
+ private
102
+
103
+ def setup_project_directory
104
+ matchdata = @repository.match(/.*github.com\/(.*)\/(.*).git/)
105
+ raise_error("Bad github repository: #{@repository}") if matchdata.nil?
106
+ org, repo = matchdata.captures
107
+ @branch_directory = File.join(WORKSPACE, org, repo, 'repositories', @branch)
108
+ @project_directory = File.join(WORKSPACE, org, repo)
109
+ @common_cache_dir = File.join(@project_directory, 'cache')
110
+ FileUtils.mkdir_p @branch_directory
111
+ FileUtils.mkdir_p @common_cache_dir
112
+ end
113
+
114
+ def raise_error(message)
115
+ raise Error, "Project error: #{message}"
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,24 @@
1
+ require 'chatops_deployer/vault'
2
+
3
+ module ChatopsDeployer
4
+ class Template
5
+ attr_reader :env, :vault
6
+
7
+ def initialize(input_file_path)
8
+ @erb = ERB.new(File.read(input_file_path))
9
+ @env = {}
10
+ @vault = ChatopsDeployer::Vault.new
11
+ end
12
+
13
+ def inject(env)
14
+ @env = env
15
+ self
16
+ end
17
+
18
+ def write(output_file_path)
19
+ File.open(output_file_path, 'w') do |f|
20
+ f.write(@erb.result(binding))
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,10 @@
1
+ require 'vault'
2
+
3
+ module ChatopsDeployer
4
+ class Vault
5
+ def read(secret, field)
6
+ secret = ::Vault.logical.read(secret)
7
+ secret ? secret.data[field.to_sym] : nil
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,3 @@
1
+ module ChatopsDeployer
2
+ VERSION = "0.1.0"
3
+ end
metadata ADDED
@@ -0,0 +1,140 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: chatops_deployer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Emil Soman
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2015-10-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: sinatra
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.4'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.4'
27
+ - !ruby/object:Gem::Dependency
28
+ name: sucker_punch
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.5'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.5'
41
+ - !ruby/object:Gem::Dependency
42
+ name: httparty
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '0.13'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '0.13'
55
+ - !ruby/object:Gem::Dependency
56
+ name: haikunator
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.1'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.1'
69
+ - !ruby/object:Gem::Dependency
70
+ name: vault
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0.1'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0.1'
83
+ description: ChatopsDeployer deploys containerized services in isolated VMs and exposes
84
+ public facing URLs
85
+ email:
86
+ - emil@codemancers.com
87
+ executables:
88
+ - chatops_deployer
89
+ extensions: []
90
+ extra_rdoc_files: []
91
+ files:
92
+ - ".rspec"
93
+ - ".travis.yml"
94
+ - Gemfile
95
+ - Gemfile.lock
96
+ - LICENSE.txt
97
+ - README.md
98
+ - Rakefile
99
+ - bin/setup
100
+ - chatops_deployer.gemspec
101
+ - doc/using_vault.md
102
+ - exe/chatops_deployer
103
+ - lib/chatops_deployer/app.rb
104
+ - lib/chatops_deployer/command.rb
105
+ - lib/chatops_deployer/container.rb
106
+ - lib/chatops_deployer/deploy_job.rb
107
+ - lib/chatops_deployer/destroy_job.rb
108
+ - lib/chatops_deployer/error.rb
109
+ - lib/chatops_deployer/globals.rb
110
+ - lib/chatops_deployer/logger.rb
111
+ - lib/chatops_deployer/nginx_config.rb
112
+ - lib/chatops_deployer/project.rb
113
+ - lib/chatops_deployer/template.rb
114
+ - lib/chatops_deployer/vault.rb
115
+ - lib/chatops_deployer/version.rb
116
+ homepage: https://github.com/code-mancers/chatops-deployer
117
+ licenses:
118
+ - MIT
119
+ metadata: {}
120
+ post_install_message:
121
+ rdoc_options: []
122
+ require_paths:
123
+ - lib
124
+ required_ruby_version: !ruby/object:Gem::Requirement
125
+ requirements:
126
+ - - ">="
127
+ - !ruby/object:Gem::Version
128
+ version: '0'
129
+ required_rubygems_version: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ requirements: []
135
+ rubyforge_project:
136
+ rubygems_version: 2.4.7
137
+ signing_key:
138
+ specification_version: 4
139
+ summary: An opinionated Chatops backend
140
+ test_files: []