chatops_deployer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2d2860bee470087ff7e2dca10b45174989c32e31
+  data.tar.gz: ea456212c652d3579008e9cf2848f9454d6b5eae
+SHA512:
+  metadata.gz: 1f2dbe5436e077a30b0d0128c3f988c44736764190efb882ba4bbad828bd7294e7ac5748e4268a340f96843011a7c061cd37c688e1b43eaae9f2abb9e4b2819d
+  data.tar.gz: 31976f8bde5739bbc503bc5040b4690e31e078bd6c7ebb2954e9c69fba7a5550f9d684d27a9a34bb1762ed2e660a8b6f904dffa314e94ee29773dff9fb83eb6a

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,14 @@
+sudo: false # Use a container for faster boot time
+language: ruby
+bundler_args: "--without exclude_in_travis"
+rvm:
+- 2.1.3
+- 2.2.2
+script: bundle exec rake
+notifications:
+  hipchat:
+    rooms:
+      secure: KdhzB9VueLEJp9p/ZUFaXAMO0u1CEbT0sjdRt10jqseAJCld9CAK1KAb1f/mNO/+cbVd+7W9UdHA0cEGUZ9dLP9Mcpmy31QxOJSITUCrRiLOF1ti13Lq4WHpMM46Tz6p7vVYd4zpPj9Mup+lCTGOeCovReX+neIoZw0UjDg9oGs=
+    template:
+      - '%{result}: %{repository_slug}#%{branch} - <a href="%{compare_url}">%{commit}</a>(%{commit_message}) by %{author} (<a href="%{build_url}">Details</a>)'
+    format: html

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+group :exclude_in_travis do
+  gem 'pry-byebug'
+end
+
+gem 'rake'
+gem 'rspec'
+gem 'memfs'
+gem 'webmock'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,85 @@
+PATH
+  remote: .
+  specs:
+    chatops_deployer (0.1.0)
+      haikunator (~> 1.1)
+      httparty (~> 0.13)
+      sinatra (~> 1.4)
+      sucker_punch (~> 1.5)
+      vault (~> 0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    byebug (4.0.5)
+      columnize (= 0.9.0)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    coderay (1.1.0)
+    columnize (0.9.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.2.5)
+    haikunator (1.1.0)
+    hitimes (1.2.2)
+    httparty (0.13.5)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    json (1.8.3)
+    memfs (0.4.3)
+    method_source (0.8.2)
+    multi_xml (0.5.5)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (3.1.0)
+      byebug (~> 4.0)
+      pry (~> 0.10)
+    rack (1.6.4)
+    rack-protection (1.5.3)
+      rack
+    rake (10.4.2)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    safe_yaml (1.0.4)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    slop (3.6.0)
+    sucker_punch (1.5.1)
+      celluloid (= 0.16.0)
+    tilt (2.0.1)
+    timers (4.0.4)
+      hitimes
+    vault (0.1.5)
+    webmock (1.21.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  chatops_deployer!
+  memfs
+  pry-byebug
+  rake
+  rspec
+  webmock
+
+BUNDLED WITH
+   1.10.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,160 @@
+# ChatopsDeployer
+
+A lightweight Sinatra app that deploys staging apps of git branches
+in docker containers. Meant to be used with hubot.
+
+## Requirements
+
+**All commands need to be run as the root user**
+So it's best if you can run this on a dedicated disposable server.
+
+1. virtualbox - For creating isolated VMs for each project
+3. docker-machine - For starting docker daemons on VMs
+2. docker-compose - For running multi-container apps using docker daemons on VMs
+4. nginx - For setting up a subdomain for each deployment
+
+TODO: setup script to install requirements on Ubuntu 14.04
+
+## Installation
+
+    $ gem install chatops_deployer
+
+## Usage
+
+Set the following ENV vars:
+
+```bash
+export DEPLOYER_HOST=<hostname where nginx listens>
+export WORKSPACE=<path where you want your projects to be git-cloned> # default: '/var/www'
+export NGINX_SITES_ENABLED_DIR=<path to sites-enabled directory in nginx conf> # default: '/etc/nginx/sites-enabled'
+export COPY_SOURCE_DIR = <path to directory containing source files to be copied over to projects> # default: '/etc/chatops_deployer/copy'
+export DEPLOYER_REGISTRY_MIRROR = <URL of docker registry mirror if you want to make use of faster docker image pulls> # default: nil
+
+# Optional to use Vault for managing and distributing secrets
+export VAULT_ADDR= <address where vault server is listening>
+export VAULT_TOKEN= <token which can read keys stored under path secret/*>
+export VAULT_CACERT= <CA certificate file to verify vault server SSL certificate>
+```
+And run the server as the root user:
+
+    $ chatops_deployer
+
+### Configuration
+
+To configure an app for deployment using chatops_deployer API, you need to follow the following steps:
+
+#### 1. Dockerize the app
+
+Add a `docker-compose.yml` file inside the root of the app that can run the app
+and the dependent services as docker containers using the command `docker-compose up`.
+Refer [the docker compose docs](https://docs.docker.com/compose/) to learn how
+to prepare this file for your app.
+
+#### 2. Add chatops_deployer.yml
+
+Add a `chatops_deployer.yml` file inside the root of the app.
+This file will tell `chatops_deployer` about ports to expose as haikunated
+subdomains, commands to run after cloning the repository and also if any files
+need to be copied into the project after cloning it for any runtime configuration.
+
+Here's an example `chatops_deployer.yml` :
+
+```yaml
+# `expose` is a hash in the format <service>:<array of ports>
+# <service> : Service name as specified in docker-compose.yml
+# <array of ports> : Ports on the container which should be exposed as subdomains
+expose:
+  web: [3000]
+
+# `commands` is a list of commands that should be run inside a service container
+# before all systems are go.
+# Commands are run in the same order as they appear in the list.
+commands:
+  - [db, "./setup_script_in_container"]
+  - [web, "bundle exec rake db:create"]
+  - [web, "bundle exec rake db:schema:load"]
+
+# `copy` is an array of strings in the format "<source>:<destination>"
+# If source begins with './' , the source file is searched from the root of the cloned
+# repo, else it is assumed to be a path to a file relative to
+# /etc/chatops_deployer/copy in the deployer server.
+# destination is the path relative to chatops_deployer.yml to which the source file
+# should be copied. Copying of files happen soon after the repository is cloned
+# and before any docker containers are created.
+# If the source file ends with .erb, it's treated as an ERB template and gets
+# processed. You have access to the following objects inside the ERB templates:
+# "env", "vault"
+#
+# "env" holds the exposed urls. For example:
+# "<%= env['urls']['web']['3000'] %>" will be replaced with "http://crimson-cloud-12.example.com"
+#
+# "vault" can be used to access secrets managed using Vault if you have set it up
+# "<%= vault.read('secret/app-name/AWS_SECRET_KEY', 'value') %>" will be replaced with the secret key fetched from Vault
+# using the command `vault read -field=value secret/app-name/AWS_SECRET_KEY`
+copy:
+  - "./config.dev.env.erb:config.env"
+
+# `cache` is a hash in the format <directory_in_code>: {<service>: <directory_in_service>}
+# <directory_in_code> is a directory under the root of the cloned repo
+# where a cached directory is created.
+# <service> is the name of a service which will have the cached directory in its container.
+# <directory_in_service> is the absolute path of the cached directory inside the running service.
+# The `cache` option allows you to share data among deployments (for faster deployments).
+# Before every deployment, each cache directory is mounted under the cloned repo.
+# These directories can then be used during docker build. Once the app is deployed,
+# the cache directories are updated with their latest content from the running
+# containers, which will be used for subsequent deployments.
+cache:
+  - tmp/bundler
+  - node_modules
+```
+
+### Deployment
+
+To deploy an app using `chatops_deployer`, send a POST request to `chatops_deployer`
+like so :
+
+```
+curl -XPOST  -d '{"repository":"https://github.com/user/app.git","branch":"master","callback_url":"example.com/deployment_status"}' -H "Content-Type: application/json" localhost:8000/deploy
+```
+
+You can see that the request accepts a `callback_url`. chatops_deployer will
+POST to this callback_url with the following data:
+
+1. Success callback
+
+Example:
+```json
+{
+  status: 'deployment_success',
+  branch: 'master',
+  urls: { web: ['misty-meadows-123.deployer-host.com'] }
+}
+```
+
+2. Failure callback
+
+Example:
+```json
+{
+  status: 'deployment_failure',
+  branch: 'master',
+  reason: 'f052f10148bd290321b84f44: Nginx error: Config directory /etc/nginx/sites-enabled does not exist'
+}
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/code-mancers/chatops_deployer.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task default: [:spec]

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/chatops_deployer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'chatops_deployer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "chatops_deployer"
+  spec.version       = ChatopsDeployer::VERSION
+  spec.authors       = ["Emil Soman"]
+  spec.email         = ["emil@codemancers.com"]
+
+  spec.summary       = %q{An opinionated Chatops backend}
+  spec.description   = %q{ChatopsDeployer deploys containerized services in isolated VMs and exposes public facing URLs}
+  spec.homepage      = "https://github.com/code-mancers/chatops-deployer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sinatra", "~> 1.4"
+  spec.add_dependency "sucker_punch", "~> 1.5"
+  spec.add_dependency "httparty", "~> 0.13"
+  spec.add_dependency "haikunator", "~> 1.1"
+  spec.add_dependency "vault", "~> 0.1"
+end

data/doc/using_vault.md

@@ -0,0 +1,106 @@
+Using Vault for secrets management
+==================================
+
+[Vault](https://vaultproject.io/) is an awesome tool to securely store and
+distribute secrets among your apps. The flow that we'll use for our deployer
+is as follows:
+
+## 1. Set up Vault server
+
+The admin who sets up chatops_deployer will need to set up Vault and run the
+Vault server.
+Follow the [Vault deployment guide](https://vaultproject.io/intro/getting-started/deploy.html).
+
+You'll have to generate a TLS key and cert first.
+
+```
+openssl genrsa 1024 > vault.key
+chmod 400 vault.key
+openssl req -new -x509 -nodes -sha1 -days 365 -key vault.key > vault.crt
+```
+Pass the `vault.crt` file to the person who'll be writing secrets.
+
+Then use the following config file when starting the server :
+
+```
+backend "file" {
+  path = "secret"
+}
+
+listener "tcp" {
+ address = "0.0.0.0:8200"
+ tls_cert_file = "vault.crt"
+ tls_key_file = "vault.key"
+}
+```
+
+Follow rest of the guide to complete the deployment. Please make sure you
+note down the "Initial Root Token" which you get after `vault init` step.
+
+Export the following ENV vars and start chatops_deployer
+```
+export VAULT_ADDR= <address where vault server is listening>
+export VAULT_TOKEN= <token which can read keys stored under path secret/*>
+export VAULT_CACERT= <CA certificate file to verify vault server SSL certificate>
+```
+## 2. Setup for each environment
+
+### 1. Generate an access policy
+
+Use the following template to create an ACL
+
+Contents of `myapp-staging.hcl`
+```
+path "secret/myapp-staging/*" {
+  policy = "write"
+}
+```
+
+Use the above policy file to create a policy :
+
+```
+vault policy-write myapp-staging myapp-staging.hcl
+```
+
+### 2. Generate a token using policy
+
+```
+vault token-create -display-name="myapp-staging" -policy="myapp-staging"
+```
+
+Pass this token to the person who'll be setting the secrets.
+
+## 3. Write secrets
+
+The developer or the person who's to set the secrets will need to install
+Vault first. Then set the following ENV vars :
+
+export VAULT_ADDR=https://<vault server url>
+export VAULT_CACERT=vault.crt
+export VAULT_TOKEN=<token generated in step 2>
+
+Write secrets with the following command :
+
+```
+vault write secret/myapp-staging/SECRET_KEY value=SECRET_VALUE
+```
+
+## 4. Use secrets in `copy` files
+
+In `chatops_deployer.yml`, use `copy` option to write a config file using the
+secret. For example:
+
+```
+# chatops_deployer.yml
+copy:
+  - "./config/secrets.staging.yml.erb:config/secrets.yml"
+```
+
+```
+# config/secrets.staging.yml.erb
+staging:
+  SECRET_KEY: <%= vault.read('secret/myapp-staging/SECRET_KEY', 'value') %>
+```
+
+chatops_deployer will expand the ERB tag by reading the secret from Vault and
+write the file to the specified destination, ie, `config/secrets.yml`

data/exe/chatops_deployer

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'chatops_deployer/app.rb'
+rescue LoadError => e
+  require 'rubygems'
+  path = File.expand_path '../../lib', __FILE__
+  $:.unshift(path) if File.directory?(path) && !$:.include?(path)
+  require 'chatops_deployer/app.rb'
+end

data/lib/chatops_deployer/app.rb

@@ -0,0 +1,35 @@
+require 'sinatra'
+require 'net/http'
+require 'json'
+require 'chatops_deployer/deploy_job'
+require 'fileutils'
+
+module ChatopsDeployer
+  class App < Sinatra::Base
+    set :port, 8000
+    set :bind, '0.0.0.0'
+
+    configure do
+      [WORKSPACE, COPY_SOURCE_DIR].each do |dir|
+        FileUtils.mkdir_p dir unless Dir.exists?(dir)
+      end
+    end
+
+    post '/deploy' do
+      content_type :json
+      json = JSON.parse(request.body.read)
+
+      DeployJob.new.async.perform(repository: json['repository'], branch: json['branch'], callback_url: json['callback_url'])
+      { log_url: LOG_URL }.to_json
+    end
+
+    post '/destroy' do
+      content_type :json
+      json = JSON.parse(request.body.read)
+
+      DestroyJob.new.async.perform(repository: json['repository'], branch: json['branch'], callback_url: json['callback_url'])
+    end
+  end
+end
+
+ChatopsDeployer::App.run!

data/lib/chatops_deployer/command.rb

@@ -0,0 +1,33 @@
+require 'logger'
+require 'open3'
+
+module ChatopsDeployer
+  class Command
+    attr_reader :output
+
+    def self.run(command: "", logger: ::Logger.new(STDOUT))
+      new.run(command, logger)
+    end
+
+    def initialize
+      @output = ""
+      @status = nil
+    end
+
+    def run(command, logger)
+      logger.info "Running command: #{command.inspect}"
+      Open3.popen2e(*(Array(command))) do |_, out_err, thread|
+        out_err.each_line do |line|
+          logger.info line
+          @output << line
+        end
+        @status = thread.value
+      end
+      self
+    end
+
+    def success?
+      @status && @status.success?
+    end
+  end
+end

data/lib/chatops_deployer/container.rb

@@ -0,0 +1,107 @@
+require 'chatops_deployer/error'
+require 'chatops_deployer/command'
+require 'chatops_deployer/globals'
+require 'chatops_deployer/logger'
+require 'yaml'
+
+module ChatopsDeployer
+  class Container
+    include Logger
+    class Error < ChatopsDeployer::Error; end
+
+    attr_reader :urls
+    def initialize(project)
+      @sha1 = project.sha1
+      @urls = {}
+      @project = project
+    end
+
+    def build
+      @config = @project.config
+      create_docker_machine
+      setup_docker_environment
+      docker_compose_run_commands
+      docker_compose_up
+    end
+
+    def destroy
+      raise_error("Cannot destroy VM because it doesn't exist") unless vm_exists?
+      destroy_vm
+    end
+
+    private
+
+    def create_docker_machine
+      if vm_exists?
+        logger.info "VM for the branch already exists. Destroying it."
+        Command.run(command: "docker-machine rm #{@sha1}", logger: logger)
+      end
+      logger.info "Creating VM #{@sha1}"
+      mirror_config = REGISTRY_MIRROR ? " --engine-registry-mirror=#{REGISTRY_MIRROR}" : ""
+      Command.run(command: "docker-machine create --driver virtualbox #{@sha1}#{mirror_config}", logger: logger)
+      get_ip = Command.run(command: "docker-machine ip #{@sha1}", logger: logger)
+      unless get_ip.success?
+        raise_error('Cannot create VM for running docker containers')
+      end
+      @ip = get_ip.output.chomp
+    end
+
+    def setup_docker_environment
+      logger.info "Setting up docker environment for #{@sha1}"
+      docker_env = Command.run(command: "docker-machine env #{@sha1}", logger: logger)
+      raise_error('Cannot set docker environment variables') unless docker_env.success?
+
+      matches = []
+      docker_env.output.scan(/export (?<env_key>.*)="(?<env_value>.*)"\n/){ matches << $~ }
+      matches.each do |match|
+        ENV[match[:env_key]] = match[:env_value]
+      end
+    end
+
+    def docker_compose_run_commands
+      logger.info "Running commands on containers"
+      commands = @config['commands']
+      commands.each do |service_commands|
+        service = service_commands[0]
+        command = service_commands[1]
+        docker_compose_run = Command.run(command: "docker-compose run #{service} #{command}", logger: logger)
+        raise_error("docker-compose run #{service} #{command} failed") unless docker_compose_run.success?
+      end
+    end
+
+    def docker_compose_up
+      logger.info "Running docker-compose up"
+      docker_compose = Command.run(command: 'docker-compose up -d', logger: logger)
+      raise_error('docker-compose up failed') unless docker_compose.success?
+
+      if expose = @config['expose']
+        expose.each do |service, ports|
+          @urls[service] = ports.collect do |port|
+            get_url_on_vm(service, port)
+          end
+        end
+      end
+    end
+
+    def vm_exists?
+      Command.run(command: "docker-machine url #{@sha1}", logger: logger).success?
+    end
+
+    def destroy_vm
+      logger.info "Destroying VM #{@sha1}"
+      system("docker-machine stop #{@sha1}") &&
+        system("docker-machine rm #{@sha1}")
+    end
+
+    def get_url_on_vm(service, port)
+      docker_port = Command.run(command: "docker-compose port #{service} #{port}", logger: logger)
+      raise_error("Cannot find exposed port for #{port} in service #{service}") unless docker_port.success?
+      port = docker_port.output.chomp.split(':').last
+      [@ip, port]
+    end
+
+    def raise_error(message)
+      raise Error, "Container error: #{message}"
+    end
+  end
+end

data/lib/chatops_deployer/deploy_job.rb

@@ -0,0 +1,58 @@
+require 'sucker_punch'
+require 'fileutils'
+require 'httparty'
+require 'chatops_deployer/globals'
+require 'chatops_deployer/project'
+require 'chatops_deployer/nginx_config'
+require 'chatops_deployer/container'
+require 'chatops_deployer/logger'
+
+module ChatopsDeployer
+  class DeployJob
+    include SuckerPunch::Job
+
+    def perform(repository:, branch: 'master', config_file: 'chatops_deployer.yml', callback_url:)
+      @branch = branch
+      @project = Project.new(repository, branch, config_file)
+      log_file = File.open(LOG_FILE, 'a')
+      @logger = ::Logger.new(MultiIO.new($stdout, log_file)).tap do |l|
+        l.progname = @project.sha1
+      end
+
+      @nginx_config = NginxConfig.new(@project)
+      @container = Container.new(@project)
+      [@project, @nginx_config, @container].each do |obj|
+        obj.logger = @logger
+      end
+
+      Dir.chdir(@project.branch_directory) do
+        @project.fetch_repo
+        @project.read_config
+        @nginx_config.prepare_urls
+        @project.copy_files_from_deployer
+        @project.setup_cache_directories
+        @container.build
+        @project.update_cache
+      end
+      @nginx_config.add_urls(@container.urls)
+      callback(callback_url, :deployment_success)
+    rescue ChatopsDeployer::Error => e
+      @logger.error(e.message)
+      callback(callback_url, :deployment_failure, e.message)
+    end
+
+    private
+
+    def callback(callback_url, status, reason=nil)
+      body = {status: status, branch: @branch}
+      if status == :deployment_success
+        body[:urls] = @nginx_config.readable_urls
+        @logger.info "Succesfully deployed #{@branch}"
+      else
+        body[:reason] = reason
+        @logger.info "Failed deploying #{@branch}. Reason: #{reason}"
+      end
+      HTTParty.post(callback_url, body: body.to_json, headers: {'Content-Type' => 'application/json'})
+    end
+  end
+end

data/lib/chatops_deployer/destroy_job.rb

@@ -0,0 +1,53 @@
+require 'sucker_punch'
+require 'fileutils'
+require 'httparty'
+
+module ChatopsDeployer
+  class DestroyJob
+    include SuckerPunch::Job
+
+    def perform(repository:, branch:, callback_url:)
+      git_basename = repository.split('/').last
+      project = File.basename(git_basename,File.extname(git_basename))
+      @branch = branch
+      @deployment_alias = "#{project}-#{branch}"
+      project_dir = "#{WORKSPACE}/#{project}/#{branch}"
+      puts "Removing #{project_dir}"
+      FileUtils.rm_rf project_dir
+
+      #TODO: No error conditions are handled in the following methods.
+      if remove_nginx_config && dockerdown
+        callback(callback_url, :destroy_success)
+      else
+        callback(callback_url, :destroy_failure)
+      end
+    end
+
+    private
+
+    def dockerdown
+      puts "Removing docker container #{@deployment_alias}"
+      system('docker', 'stop', @deployment_alias) &&
+        system('docker', 'rm', @deployment_alias)
+    end
+
+    def remove_nginx_config
+      nginx_config = File.join(NGINX_SITES_ENABLED_DIR, @deployment_alias)
+      return false if !File.exists?(nginx_config)
+
+      puts "Removing nginx config at #{nginx_config}"
+      File.delete(nginx_config)
+      system('service nginx reload')
+    end
+
+    def callback(callback_url, status)
+      body = {status: status, branch: @branch}
+      if status == :destroy_success
+        puts "Succesfully destroyed staging env of #{@branch}"
+      else
+        puts "Failed destroying staging env of #{@branch}"
+      end
+      HTTParty.post(callback_url, body: body.to_json, headers: {'Content-Type' => 'application/json'})
+    end
+  end
+end

data/lib/chatops_deployer/error.rb

@@ -0,0 +1,3 @@
+module ChatopsDeployer
+  class Error < StandardError; end
+end

data/lib/chatops_deployer/globals.rb

@@ -0,0 +1,10 @@
+module ChatopsDeployer
+  WORKSPACE = ENV['DEPLOYER_WORKSPACE'] || '/var/www'
+  DEPLOYER_HOST = ENV['DEPLOYER_HOST'] || '127.0.0.1.xip.io'
+  NGINX_SITES_ENABLED_DIR = ENV['NGINX_SITES_ENABLED_DIR'] || '/etc/nginx/sites-enabled'
+  LOG_FILE = ENV['DEPLOYER_LOG_FILE'] || '/var/log/chatops_deployer.log'
+  COPY_SOURCE_DIR = ENV['DEPLOYER_COPY_SOURCE_DIR'] || '/etc/chatops_deployer/copy'
+  LOG_URL = ENV['DEPLOYER_LOG_URL']
+  REGISTRY_MIRROR = ENV['DEPLOYER_REGISTRY_MIRROR']
+end
+

data/lib/chatops_deployer/logger.rb

@@ -0,0 +1,39 @@
+require 'logger'
+
+module ChatopsDeployer
+  class MultiIO
+    def initialize(*targets)
+      @targets = targets
+    end
+
+    def write(*args)
+      @targets.each{|t| t.write(*args); t.flush }
+    end
+
+    def close
+      @targets.each(&:close)
+    end
+  end
+
+  module Logger
+    def self.included(base)
+      class << base
+        def logger
+          @logger ||= ::Logger.new($stdout)
+        end
+
+        def logger=(logger)
+          @logger = logger
+        end
+      end
+    end
+
+    def logger
+      self.class.logger
+    end
+
+    def logger=(logger)
+      self.class.logger = logger
+    end
+  end
+end

data/lib/chatops_deployer/nginx_config.rb

@@ -0,0 +1,125 @@
+require 'chatops_deployer/globals'
+require 'chatops_deployer/error'
+require 'chatops_deployer/command'
+require 'haikunator'
+require 'fileutils'
+require 'chatops_deployer/logger'
+
+module ChatopsDeployer
+  class NginxConfig
+    include Logger
+    attr_reader :urls
+
+    class Error < ChatopsDeployer::Error; end
+
+    def initialize(project)
+      @sha1 = project.sha1
+      @project = project
+      check_sites_enabled_dir_exists!
+      @config_path = File.join NGINX_SITES_ENABLED_DIR, @sha1
+      @urls = {}
+    end
+
+    def exists?
+      File.exists? @config_path
+    end
+
+    # service_urls is an array in the format:
+    # {"web" => [["10.1.1.2", "3000"],["10.1.1.2", "4000"]] }
+    def add_urls(service_urls)
+      return if service_urls.nil?
+      remove if exists?
+
+      service_urls.each do |service, internal_urls|
+        Array(internal_urls).each do |internal_url|
+          expose(service, internal_url)
+        end
+      end
+      logger.info "Reloading nginx"
+      nginx_reload = Command.run(command: 'service nginx reload', logger: logger)
+      unless nginx_reload.success?
+        raise_error("Cannot reload nginx after adding config. Check #{NGINX_SITES_ENABLED_DIR}/#{@sha1} for errors")
+      end
+    end
+
+    def remove
+      logger.info "Removing nginx config"
+      FileUtils.rm @config_path
+      system('service nginx reload')
+    end
+
+    def readable_urls
+      urls = {}
+      @urls.each do |service, port_exposed_urls|
+        urls[service] = port_exposed_urls.collect do |port, exposed_url|
+          "http://#{exposed_url}"
+        end
+      end
+      urls.to_json
+    end
+
+    def prepare_urls
+      @project.env['urls'] = {}
+      service_ports_from_config.each do |service, ports|
+        @urls[service] = {}
+        @project.env['urls'][service] = {}
+        ports.each do |port|
+          @urls[service][port.to_s] = generate_haikunated_url
+          @project.env['urls'][service][port.to_s] = "http://#{@urls[service][port.to_s]}"
+        end
+      end
+    end
+
+    private
+
+    def check_sites_enabled_dir_exists!
+      unless Dir.exist? NGINX_SITES_ENABLED_DIR
+        raise_error("Config directory #{NGINX_SITES_ENABLED_DIR} does not exist")
+      end
+    end
+
+    def service_ports_from_config
+      @project.config['expose'] || {}
+    end
+
+    def generate_haikunated_url
+      haiku = Haikunator.haikunate
+      "#{haiku}.#{DEPLOYER_HOST}"
+    end
+
+    # service => name of service , example: "web"
+    # internal_url => a pair of ip and port, example: ["10.1.1.2", "3000"]
+    def expose(service, internal_url)
+      raise_error("Cannot add nginx config because host is nil") if internal_url.nil? || internal_url.empty?
+      ip = internal_url[0]
+      port = internal_url[1]
+      begin
+        exposed_url = @urls[service][port]
+      rescue
+        raise_error("Cannot add nginx config because exposed ports could not be read from chatops_deployer.yml")
+      end
+      contents = <<-EOM
+        server{
+            listen 80;
+            server_name #{exposed_url};
+
+            # host error and access log
+            access_log /var/log/nginx/#{exposed_url}.access.log;
+            error_log /var/log/nginx/#{exposed_url}.error.log;
+
+            location / {
+                proxy_pass http://#{ip}:#{port};
+            }
+        }
+      EOM
+      logger.info "Adding nginx config at #{NGINX_SITES_ENABLED_DIR}/#{@sha1}"
+      File.open(@config_path, 'a') do |file|
+        file << contents
+      end
+    end
+
+    def raise_error(message)
+      raise Error, "Nginx error: #{message}"
+    end
+  end
+end

data/lib/chatops_deployer/project.rb

@@ -0,0 +1,118 @@
+require 'chatops_deployer/globals'
+require 'chatops_deployer/error'
+require 'chatops_deployer/command'
+require 'chatops_deployer/template'
+require 'chatops_deployer/logger'
+require 'digest/sha1'
+require 'fileutils'
+require 'yaml'
+
+module ChatopsDeployer
+  class Project
+    include Logger
+    class Error < ChatopsDeployer::Error; end
+
+    attr_reader :sha1, :branch_directory, :config
+    attr_accessor :env
+    def initialize(repository, branch, config_file="chatops_deployer.yml")
+      @sha1 = Digest::SHA1.hexdigest(repository + branch)
+      @repository = repository
+      @branch = branch
+      @config_file = config_file
+      @env = {}
+      setup_project_directory
+    end
+
+    def fetch_repo
+      logger.info "Fetching #{@repository}:#{@branch}"
+      unless Dir.entries('.').size == 2
+        logger.info "Branch already cloned. Deleting everything before cloning again"
+        FileUtils.rm_rf '.'
+      end
+      logger.info "Cloning branch #{@repository}:#{@branch}"
+      git_clone = Command.run(command: ['git', 'clone', "--branch=#{@branch}", '--depth=1', @repository, '.'], logger: logger)
+      unless git_clone.success?
+        raise_error("Cannot clone git repository: #{@repository}, branch: #{@branch}")
+      end
+    end
+
+    def read_config
+      @config = if File.exists?(@config_file)
+        begin
+          YAML.load_file(@config_file) || {}
+        rescue StandardError => e
+          raise_error("Cannot parse YAML content in #{@config_file}")
+        end
+      else
+        {}
+      end
+    end
+
+    def copy_files_from_deployer
+      copy_list = @config['copy'].to_a
+      return if copy_list.empty?
+      logger.info "Copying files from deployer to project"
+      copy_list.each do |copy_string|
+        source, destination = copy_string.split(':')
+        # source is from COPY_SOURCE_DIR if source doesn't start with ./
+        source = File.join(COPY_SOURCE_DIR, source) unless source.match(/^\.\//)
+        if File.extname(source) == '.erb'
+          destination ||= File.basename(source, '.erb')
+          logger.info "Processing ERB template #{source} into #{destination}"
+          Template.new(source).inject(@env).write(destination)
+        else
+          destination ||= File.basename source
+          logger.info "Copying #{source} into #{destination}"
+          FileUtils.cp source, destination
+        end
+      end
+    end
+
+    def setup_cache_directories
+      cache_directory_list = @config['cache'].to_h
+      cache_directory_list.each do |directory, _|
+        cache_dir = File.join(@common_cache_dir, directory)
+        target_cache_dir = File.join(@branch_directory, directory)
+        FileUtils.mkdir_p cache_dir
+        FileUtils.mkdir_p target_cache_dir
+        FileUtils.rm_rf target_cache_dir
+        FileUtils.cp_r(cache_dir, target_cache_dir)
+      end
+    end
+
+    def update_cache
+      cache_directory_list = @config['cache'].to_h
+      cache_directory_list.each do |directory, service_paths|
+        service_paths.each do |service, path|
+          ps = Command.run(command: ["docker-compose", "ps", "-q", service], logger: logger)
+          container = ps.output.chomp
+          next if container.empty?
+          cache_dir = File.join(@common_cache_dir, directory)
+          tmp_dir = File.join(@project_directory, 'tmp_cache')
+          copy_from_container = Command.run(command: ["docker", "cp", "#{container}:#{path}", tmp_dir], logger: logger)
+          raise_error("Cannot copy '#{path}' from container '#{container}' of service '#{service}'") unless copy_from_container.success?
+          FileUtils.rm_rf(cache_dir)
+          FileUtils.mv(tmp_dir, cache_dir)
+          FileUtils.rm_rf(tmp_dir)
+        end
+      end
+    end
+
+    private
+
+    def setup_project_directory
+      matchdata = @repository.match(/.*github.com\/(.*)\/(.*).git/)
+      raise_error("Bad github repository: #{@repository}") if matchdata.nil?
+      org, repo = matchdata.captures
+      @branch_directory = File.join(WORKSPACE, org, repo, 'repositories', @branch)
+      @project_directory = File.join(WORKSPACE, org, repo)
+      @common_cache_dir = File.join(@project_directory, 'cache')
+      FileUtils.mkdir_p @branch_directory
+      FileUtils.mkdir_p @common_cache_dir
+    end
+
+    def raise_error(message)
+      raise Error, "Project error: #{message}"
+    end
+  end
+end

data/lib/chatops_deployer/template.rb

@@ -0,0 +1,24 @@
+require 'chatops_deployer/vault'
+
+module ChatopsDeployer
+  class Template
+    attr_reader :env, :vault
+
+    def initialize(input_file_path)
+      @erb = ERB.new(File.read(input_file_path))
+      @env = {}
+      @vault = ChatopsDeployer::Vault.new
+    end
+
+    def inject(env)
+      @env = env
+      self
+    end
+
+    def write(output_file_path)
+      File.open(output_file_path, 'w') do |f|
+        f.write(@erb.result(binding))
+      end
+    end
+  end
+end

data/lib/chatops_deployer/vault.rb

@@ -0,0 +1,10 @@
+require 'vault'
+
+module ChatopsDeployer
+  class Vault
+    def read(secret, field)
+      secret = ::Vault.logical.read(secret)
+      secret ? secret.data[field.to_sym] : nil
+    end
+  end
+end

data/lib/chatops_deployer/version.rb

@@ -0,0 +1,3 @@
+module ChatopsDeployer
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: chatops_deployer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Emil Soman
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-10-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: sucker_punch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: haikunator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: vault
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: ChatopsDeployer deploys containerized services in isolated VMs and exposes
+  public facing URLs
+email:
+- emil@codemancers.com
+executables:
+- chatops_deployer
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/setup
+- chatops_deployer.gemspec
+- doc/using_vault.md
+- exe/chatops_deployer
+- lib/chatops_deployer/app.rb
+- lib/chatops_deployer/command.rb
+- lib/chatops_deployer/container.rb
+- lib/chatops_deployer/deploy_job.rb
+- lib/chatops_deployer/destroy_job.rb
+- lib/chatops_deployer/error.rb
+- lib/chatops_deployer/globals.rb
+- lib/chatops_deployer/logger.rb
+- lib/chatops_deployer/nginx_config.rb
+- lib/chatops_deployer/project.rb
+- lib/chatops_deployer/template.rb
+- lib/chatops_deployer/vault.rb
+- lib/chatops_deployer/version.rb
+homepage: https://github.com/code-mancers/chatops-deployer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.7
+signing_key: 
+specification_version: 4
+summary: An opinionated Chatops backend
+test_files: []