shipitron 1.0.1 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68328f7af2efc2a7d163280f0bb1613d685d85b09f991cc53061a5286c02c66e
-  data.tar.gz: a349443e9f92c64101ebc272f44cfd8b5256d69c83b693b303eed1c2c50ff279
+  metadata.gz: c1f02ed921fdb5107cebc789f0554daac22d275024e50b94f6059d0ab6d996c1
+  data.tar.gz: 65e5fbca1de1090e34178d44244a5e30938b7de80fe17f0ea7b3df9510e54f9a
 SHA512:
-  metadata.gz: a226f3113f356b9a475b3b64aeb9d9ccebf8598d56a29ca99e71dbe7df6c00091abdbd35e70f81c9e0913aa014547c487eb66d221249a327b41220eafac09bb9
-  data.tar.gz: ac68f08e8ef8ba579d081538ac32930c723d324e5b89c3a67c5963ffa762daeb90665e08c80d3e236646dcf4178cf1b9096489f5aabf18f784cb66448cbb16b5
+  metadata.gz: 14d5f5e245ab946538ab1aee9b5d88d88fafb7d4baa61c3634d165d42acfcf5a83e0231cc9299f37d37d38da9a96be0dfc04ba77b42499fc36bdcc5acf69980c
+  data.tar.gz: ace846368647ec022343af65af9978f2647aa7113de1795900a18091754e34e26657bd4d4d4be10e60f4bef81baffa886917c79537f834377182eadabb4a704e

data/.dockerignore

@@ -2,3 +2,4 @@
 /pkg/
 Gemfile.lock
 /tmp/
+/spec/fixtures/test_repo

data/.gitattributes

@@ -0,0 +1 @@
+spec/fixtures/test_repo.tgz filter=lfs diff=lfs merge=lfs -text

data/.gitignore

@@ -3,3 +3,4 @@
 Gemfile.lock
 /tmp/
 /cache/
+/spec/fixtures/test_repo

data/Dockerfile

@@ -1,35 +1,55 @@
-FROM ruby:2.5.0-alpine3.7 as cache
+FROM ruby:2.7.1-alpine as cache
 COPY cache/ /tmp/
 RUN   cd /usr/local/bundle && \
     ([ -f /tmp/bundler-data.tar.gz ] && \
     tar -zxf /tmp/bundler-data.tar.gz && \
     rm /tmp/bundler-data.tar.gz) || true
 
-FROM ruby:2.5.0-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.1-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
     addgroup -g 1101 docker && \
     addgroup shipitron docker
 
-ENV GOSU_VERSION 1.10
-ENV DUMB_INIT_VERSION 1.2.0
-
 RUN apk add --no-cache \
     ca-certificates \
-    openssl \
+    openssl-dev \
     tini \
     su-exec \
     build-base \
     git \
+    git-lfs \
     openssh-client \
     perl \
     bash \
     curl \
     wget \
-    jq
+    jq \
+    cmake
+
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.4.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.7.2
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.4.2
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
 
+USER root
 ENV USE_BUNDLE_EXEC true
 ENV BUNDLE_GEMFILE /shipitron/Gemfile
 

data/Dockerfile.release

@@ -1,5 +1,5 @@
-FROM ruby:2.5.0-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.1-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
@@ -20,9 +20,31 @@ RUN apk add --no-cache \
     wget \
     jq
 
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.4.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.7.2
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.4.2
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+USER root
+
 WORKDIR /app
 
-ENV SHIPITRON_VERSION=1.0.1
+ENV SHIPITRON_VERSION=1.3.1
 
 RUN gem install shipitron -v ${SHIPITRON_VERSION} && \
     mkdir -p /home/shipitron/.ssh && \

data/Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org'
 
 gemspec
+gem 'irb'

data/README.md

@@ -9,6 +9,7 @@ applications:
   dummy-app:
     repository: git@github.com:outstand/dummy-app
     cache_bucket: bucket
+    build_cache_location: tmp/build-cache.tar.gz
     image_name: outstand/dummy-app
     build_script: shipitron/build.sh
     post_builds:
@@ -30,25 +31,63 @@ applications:
 - Add deploy ref key to `shipitron/<app name>/deploy_ref_key`
 - `docker run -it --rm -v shipitron.yml:/shipitron/config/shipitron.yml outstand/shipitron:<version> deploy <app>`
 
+### New shipitron config file keys:
+
+- registry (specifies an alternate docker registry):
+```yaml
+---
+applications:
+  foobar:
+    registry: 12345.dkr.ecr.us-east-1.amazonaws.com
+```
+
+- skip_push (shipitron will skip pushing the docker images; use if the build script takes care of this):
+```yaml
+---
+applications:
+  foobar:
+    skip_push: true
+```
+Additionally, the build script will receive the named tag in use as `$2`.
+
+### Containerized tool support
+Shipitron now supports starting other containers as part of the build process. We're using this to use the aws cli to transfer files to/from s3. Shipitron's task definition needs to have a task scoped docker volume added with the name `shipitron-home`. This volume will be mounted at `/home/shipitron` and should be shared with any new containers:
+
+```
+shipitron_home_volume = FindDockerVolumeName.call!(
+  container_name: 'shipitron',
+  volume_search: /shipitron-home/
+).volume_name
+
+docker run ... -v #{shipitron_home_volume}:/home/shipitron ...
+```
+
+This allows the containers to share data with each other. ECS will automatically clean up the task scoped container.
+
+If a containerized tool requires access to AWS resources, be sure to pass the `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` to it so it can inherit any task roles.
+
 ## Development
 
-- `docker volume create --name shipitron_fog`
 - `./build_dev.sh`
-- `docker run -it --rm -v $(pwd):/shipitron -v shipitron_fog:/fog -e FOG_LOCAL=true -w /shipitron outstand/shipitron:dev rspec spec` to run specs
-- `APP_PATH=/path/to/app`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app>` to run client side
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --simulate` to get the arguments for `server_deploy` below
-- `docker run -it --rm --dns 10.10.10.2 -v $(pwd):/shipitron -v $APP_PATH:/app -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock -v shipitron_fog:/fog -e FOG_LOCAL=true -e CONSUL_HOST=consul outstand/shipitron:dev server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
+- `dev run --rm shipitron_specs` to run specs
+- Set the application path in the volumes section in `docker-compose.yml`.
+- `dev run --rm shipitron deploy <app>` to run client side
+- `dev run --rm shipitron deploy <app> --simulate` to get the arguments for `server_deploy` below
+- `dev run --rm -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock shipitron server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
 
 Running a dev version in production:
+- Update `Shipitron::Client::STARTED_BY`
 - `docker push outstand/shipitron:dev`
 - Update config to use `shipitron_task: shipitron-dev`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --debug`
+- `dev run --rm shipitron deploy <app> --debug`
 
 To release a new version:
 - Update the version number in `version.rb` and `Dockerfile.release` and commit the result.
 - `./build_dev.sh`
 - `docker run -it --rm -v ~/.gitconfig:/root/.gitconfig -v ~/.gitconfig.user:/root/.gitconfig.user -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v ~/.gem:/root/.gem -w /shipitron outstand/shipitron:dev rake release`
-- `docker build -t outstand/shipitron:VERSION -f Dockerfile.release .`
-- `docker push outstand/shipitron:VERSION`
+- `VERSION=<version>`
+- `docker build -t outstand/shipitron:$VERSION -f Dockerfile.release .`
+- `docker push outstand/shipitron:$VERSION`
+- `docker tag outstand/shipitron:$VERSION outstand/shipitron:latest`
+- `docker push outstand/shipitron:latest`
 - Update ECS task definition with new version

data/build_dev.sh

@@ -20,7 +20,7 @@ trap cleanup EXIT
 build_args=''
 mkdir -p ${bundler_data_dir}
 
-docker build -t ${build_image} -f ${dockerfile} ${build_args} .
+docker build --pull -t ${build_image} -f ${dockerfile} ${build_args} .
 
 docker run -t --cidfile=${bundler_data_dir}/cidfile -w /usr/local/bundle ${build_image} tar -zcf /tmp/bundler-data.tar.gz .
 tar_container=$(cat ${bundler_data_dir}/cidfile)

data/docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.8'
+services:
+  shipitron:
+    image: outstand/shipitron:dev
+    volumes:
+      - ~/dev/pages:/app # Set this to the application to be shipitron'd
+      - shipitron-home:/home/shipitron
+      - ~/.config/shipitron:/home/shipitron/.config/shipitron
+      - .:/shipitron
+  shipitron_specs:
+    image: outstand/shipitron:dev
+    command: rspec spec
+    environment:
+      FOG_LOCAL: 'true'
+    working_dir: /shipitron
+    volumes:
+      - fog:/fog
+      - shipitron-home:/home/shipitron
+      - ~/.config/shipitron:/home/shipitron/.config/shipitron
+      - .:/shipitron
+
+volumes:
+  fog:
+  shipitron-home:

data/lib/shipitron/cli.rb

@@ -27,20 +27,47 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
       end
     end
 
+    desc 'force_deploy <app>', 'Forces a redeploy of the app'
+    option :config_file, default: 'shipitron/config.yml'
+    option :secrets_file, default: '~/.config/shipitron/secrets.yml'
+    option :debug, type: :boolean, default: false
+    def force_deploy(app)
+      setup(
+        config_file: options[:config_file],
+        secrets_file: options[:secrets_file]
+      )
+
+      require 'shipitron/client/force_deploy'
+      result = Client::ForceDeploy.call(
+        application: app
+      )
+
+      if result.failure?
+        result.error_messages.each do |error|
+          Logger.fatal error
+        end
+        Logger.fatal 'Deploy failed.'
+      end
+    end
+
+
     desc 'server_deploy', 'Server-side component of deploy'
     option :name, required: true
     option :repository, required: true
     option :repository_branch, default: 'master'
+    option :registry, default: nil
     option :bucket, required: true
+    option :build_cache_location, default: 'tmp/build-cache.tar.gz'
     option :image_name, required: true
     option :named_tag, default: 'latest'
+    option :skip_push, type: :boolean, default: false
     option :region, required: true
     option :clusters, type: :array, required: true
     option :ecs_task_defs, type: :array, required: true
@@ -61,9 +88,12 @@ module Shipitron
         application: options[:name],
         repository_url: options[:repository],
         repository_branch: options[:repository_branch],
+        registry: options[:registry],
         s3_cache_bucket: options[:bucket],
+        build_cache_location: options[:build_cache_location],
         image_name: options[:image_name],
         named_tag: options[:named_tag],
+        skip_push: options[:skip_push],
         region: options[:region],
         clusters: options[:clusters],
         ecs_task_defs: options[:ecs_task_defs],
@@ -80,7 +110,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -111,7 +141,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Bootstrap failed.'

data/lib/shipitron/client.rb

@@ -0,0 +1,9 @@
+require 'shipitron'
+
+module Shipitron
+  module Client
+    STARTED_BY = 'shipitron'
+    # Use this for testing.
+    # STARTED_BY = 'shipitron-dev'
+  end
+end

data/lib/shipitron/client/bootstrap_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/register_ecs_task_definitions'
 require 'shipitron/client/create_ecs_services'
 

data/lib/shipitron/client/create_ecs_services.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/mustache_yaml_parser'
 require 'securerandom'

data/lib/shipitron/client/deploy_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/load_application_config'
 require 'shipitron/client/load_templates'
 require 'shipitron/client/fetch_clusters'

data/lib/shipitron/client/ensure_deploy_not_running.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 
 # Note: This is a best effort client side check to make sure there
@@ -21,7 +22,7 @@ module Shipitron
             begin
               response = ecs_client(region: cluster.region).list_tasks(
                 cluster: cluster.name,
-                started_by: 'shipitron',
+                started_by: Shipitron::Client::STARTED_BY,
                 max_results: 1,
                 desired_status: status
               )

data/lib/shipitron/client/fetch_clusters.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'resolv'
 
 module Shipitron

data/lib/shipitron/client/force_deploy.rb

@@ -0,0 +1,58 @@
+require 'shipitron'
+require 'shipitron/client'
+require 'shipitron/ecs_client'
+require 'shipitron/client/load_application_config'
+require 'shipitron/client/fetch_clusters'
+require 'shipitron/client/ensure_deploy_not_running'
+
+module Shipitron
+  module Client
+    class ForceDeploy
+      include Metaractor
+      include Interactor::Organizer
+      include EcsClient
+
+      required :application
+
+      organize [
+        LoadApplicationConfig,
+        FetchClusters,
+        EnsureDeployNotRunning
+      ]
+
+      def call
+        Logger.info "==> Force deploying #{application}"
+
+        super
+
+        context.clusters ||= []
+        context.ecs_services ||= []
+
+        begin
+          context.clusters.each do |cluster|
+            context.ecs_services.each do |service|
+              ecs_client(region: cluster.region).update_service(
+                cluster: cluster.name,
+                service: service,
+                force_new_deployment: true
+              )
+            end
+          end
+        rescue Aws::ECS::Errors::ServiceError => e
+          fail_with_errors!(messages: [
+            "Error: #{e.message}",
+            e.backtrace.join("\n")
+          ])
+        end
+
+        Logger.info "==> Done"
+      end
+
+      private
+      def application
+        context.application
+      end
+
+    end
+  end
+end

data/lib/shipitron/client/load_application_config.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/post_build'
 
 module Shipitron
@@ -11,7 +12,9 @@ module Shipitron
       def call
         context.repository_url = config.repository
         context.repository_branch = config.repository_branch
+        context.registry = config.registry
         context.s3_cache_bucket = config.cache_bucket
+        context.build_cache_location = config.build_cache_location
         context.image_name = config.image_name
         context.named_tag = begin
                               if config.named_tag.nil?
@@ -20,6 +23,7 @@ module Shipitron
                                 config.named_tag
                               end
                             end
+        context.skip_push = config.skip_push
         context.build_script = config.build_script
         context.post_builds = begin
                                 if config.post_builds.nil?