shipitron 1.1.0 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8942009b3ccf99e9b14c8676e355c8a8cd22d7def515a2903daf6707e476a235
-  data.tar.gz: 3f1d602aebc48bdc3fda47458da830626c964320b621d6d21cf0db08dfef50a1
+  metadata.gz: ffcd046204b230e7282508063bb33a58c1c68e73c4e7c8546ea15cfc48d6a2f7
+  data.tar.gz: 38ddb0adc7677e27c9afcee285d09358afb60df254994d456319bf904d518091
 SHA512:
-  metadata.gz: 8e0a2d2d5575272eada04e2cb8b836a2a71a5139e992714bec2ba45ed83dd3d0b6c7821eb92e824c4f8bcba9805f707be1b56e65d95b3ad87b69b40403758f5e
-  data.tar.gz: 45f1cdf8c7d940e0defc1ef6be4d65f772648b6c7e7d5d8759f3e19043e426d262a435a5e282eb7fd6872536041f02429d1c3822cc78979dbc444c9c53713c7c
+  metadata.gz: 28b568e8e1a5fa928beeeaf2ff1e00415671712c9415dacc317581d10d79bb320bbc05f07d9d1c7b528db4a5523f9568cb9dac33d2bc5607165b43d1896d3cc4
+  data.tar.gz: fcd1597703388b9bf8be61ab98e03faabbcf17bab98a06895344e14cb41f3ee412941fe2a27257425c3db646762130500e592150c53e35520939d04240f77fe6

data/.dockerignore

@@ -2,3 +2,4 @@
 /pkg/
 Gemfile.lock
 /tmp/
+/spec/fixtures/test_repo

data/.gitattributes

@@ -0,0 +1 @@
+spec/fixtures/test_repo.tgz filter=lfs diff=lfs merge=lfs -text

data/.gitignore

@@ -3,3 +3,4 @@
 Gemfile.lock
 /tmp/
 /cache/
+/spec/fixtures/test_repo

data/Dockerfile

@@ -1,12 +1,12 @@
-FROM ruby:2.5.1-alpine3.7 as cache
+FROM ruby:2.7.2-alpine as cache
 COPY cache/ /tmp/
 RUN   cd /usr/local/bundle && \
     ([ -f /tmp/bundler-data.tar.gz ] && \
     tar -zxf /tmp/bundler-data.tar.gz && \
     rm /tmp/bundler-data.tar.gz) || true
 
-FROM ruby:2.5.0-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.2-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
@@ -15,18 +15,41 @@ RUN addgroup -S shipitron && \
 
 RUN apk add --no-cache \
     ca-certificates \
-    openssl \
+    openssl-dev \
     tini \
     su-exec \
     build-base \
     git \
+    git-lfs \
     openssh-client \
     perl \
     bash \
     curl \
     wget \
-    jq
+    jq \
+    cmake
 
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.4.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.7.2
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.4.2
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+USER root
 ENV USE_BUNDLE_EXEC true
 ENV BUNDLE_GEMFILE /shipitron/Gemfile
 

data/Dockerfile.release

@@ -1,5 +1,5 @@
-FROM ruby:2.5.1-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.2-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
@@ -8,7 +8,7 @@ RUN addgroup -S shipitron && \
 
 RUN apk add --no-cache \
     ca-certificates \
-    openssl \
+    openssl-dev \
     tini \
     su-exec \
     build-base \
@@ -18,11 +18,34 @@ RUN apk add --no-cache \
     bash \
     curl \
     wget \
-    jq
+    jq \
+    cmake
+
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.4.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.7.2
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.4.2
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+USER root
 
 WORKDIR /app
 
-ENV SHIPITRON_VERSION=1.1.0
+ENV SHIPITRON_VERSION=1.3.2
 
 RUN gem install shipitron -v ${SHIPITRON_VERSION} && \
     mkdir -p /home/shipitron/.ssh && \

data/Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org'
 
 gemspec
+gem 'irb'

data/README.md

@@ -9,6 +9,7 @@ applications:
   dummy-app:
     repository: git@github.com:outstand/dummy-app
     cache_bucket: bucket
+    build_cache_location: tmp/build-cache.tar.gz
     image_name: outstand/dummy-app
     build_script: shipitron/build.sh
     post_builds:
@@ -30,25 +31,63 @@ applications:
 - Add deploy ref key to `shipitron/<app name>/deploy_ref_key`
 - `docker run -it --rm -v shipitron.yml:/shipitron/config/shipitron.yml outstand/shipitron:<version> deploy <app>`
 
+### New shipitron config file keys:
+
+- registry (specifies an alternate docker registry):
+```yaml
+---
+applications:
+  foobar:
+    registry: 12345.dkr.ecr.us-east-1.amazonaws.com
+```
+
+- skip_push (shipitron will skip pushing the docker images; use if the build script takes care of this):
+```yaml
+---
+applications:
+  foobar:
+    skip_push: true
+```
+Additionally, the build script will receive the named tag in use as `$2`.
+
+### Containerized tool support
+Shipitron now supports starting other containers as part of the build process. We're using this to use the aws cli to transfer files to/from s3. Shipitron's task definition needs to have a task scoped docker volume added with the name `shipitron-home`. This volume will be mounted at `/home/shipitron` and should be shared with any new containers:
+
+```
+shipitron_home_volume = FindDockerVolumeName.call!(
+  container_name: 'shipitron',
+  volume_search: /shipitron-home/
+).volume_name
+
+docker run ... -v #{shipitron_home_volume}:/home/shipitron ...
+```
+
+This allows the containers to share data with each other. ECS will automatically clean up the task scoped container.
+
+If a containerized tool requires access to AWS resources, be sure to pass the `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` to it so it can inherit any task roles.
+
 ## Development
 
-- `docker volume create --name shipitron_fog`
 - `./build_dev.sh`
-- `docker run -it --rm -v $(pwd):/shipitron -v shipitron_fog:/fog -e FOG_LOCAL=true -w /shipitron outstand/shipitron:dev rspec spec` to run specs
-- `APP_PATH=/path/to/app`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app>` to run client side
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --simulate` to get the arguments for `server_deploy` below
-- `docker run -it --rm --dns 10.10.10.2 -v $(pwd):/shipitron -v $APP_PATH:/app -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock -v shipitron_fog:/fog -e FOG_LOCAL=true -e CONSUL_HOST=consul outstand/shipitron:dev server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
+- `dev run --rm shipitron_specs` to run specs
+- Set the application path in the volumes section in `docker-compose.yml`.
+- `dev run --rm shipitron deploy <app>` to run client side
+- `dev run --rm shipitron deploy <app> --simulate` to get the arguments for `server_deploy` below
+- `dev run --rm -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock shipitron server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
 
 Running a dev version in production:
+- Update `Shipitron::Client::STARTED_BY`
 - `docker push outstand/shipitron:dev`
 - Update config to use `shipitron_task: shipitron-dev`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --debug`
+- `dev run --rm shipitron deploy <app> --debug`
 
 To release a new version:
 - Update the version number in `version.rb` and `Dockerfile.release` and commit the result.
 - `./build_dev.sh`
 - `docker run -it --rm -v ~/.gitconfig:/root/.gitconfig -v ~/.gitconfig.user:/root/.gitconfig.user -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v ~/.gem:/root/.gem -w /shipitron outstand/shipitron:dev rake release`
-- `docker build -t outstand/shipitron:VERSION -f Dockerfile.release .`
-- `docker push outstand/shipitron:VERSION`
+- `VERSION=<version>`
+- `docker build -t outstand/shipitron:$VERSION -f Dockerfile.release .`
+- `docker push outstand/shipitron:$VERSION`
+- `docker tag outstand/shipitron:$VERSION outstand/shipitron:latest`
+- `docker push outstand/shipitron:latest`
 - Update ECS task definition with new version

data/build_dev.sh

@@ -20,7 +20,7 @@ trap cleanup EXIT
 build_args=''
 mkdir -p ${bundler_data_dir}
 
-docker build -t ${build_image} -f ${dockerfile} ${build_args} .
+docker build --pull -t ${build_image} -f ${dockerfile} ${build_args} .
 
 docker run -t --cidfile=${bundler_data_dir}/cidfile -w /usr/local/bundle ${build_image} tar -zcf /tmp/bundler-data.tar.gz .
 tar_container=$(cat ${bundler_data_dir}/cidfile)

data/docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.8'
+services:
+  shipitron:
+    image: outstand/shipitron:dev
+    volumes:
+      - ~/dev/pages:/app # Set this to the application to be shipitron'd
+      - shipitron-home:/home/shipitron
+      - ~/.config/shipitron:/home/shipitron/.config/shipitron
+      - .:/shipitron
+  shipitron_specs:
+    image: outstand/shipitron:dev
+    command: rspec spec
+    environment:
+      FOG_LOCAL: 'true'
+    working_dir: /shipitron
+    volumes:
+      - fog:/fog
+      - shipitron-home:/home/shipitron
+      - ~/.config/shipitron:/home/shipitron/.config/shipitron
+      - .:/shipitron
+
+volumes:
+  fog:
+  shipitron-home:

data/lib/shipitron/cli.rb

@@ -27,7 +27,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -50,7 +50,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -62,9 +62,12 @@ module Shipitron
     option :name, required: true
     option :repository, required: true
     option :repository_branch, default: 'master'
+    option :registry, default: nil
     option :bucket, required: true
+    option :build_cache_location, default: 'tmp/build-cache.tar.gz'
     option :image_name, required: true
     option :named_tag, default: 'latest'
+    option :skip_push, type: :boolean, default: false
     option :region, required: true
     option :clusters, type: :array, required: true
     option :ecs_task_defs, type: :array, required: true
@@ -85,9 +88,12 @@ module Shipitron
         application: options[:name],
         repository_url: options[:repository],
         repository_branch: options[:repository_branch],
+        registry: options[:registry],
         s3_cache_bucket: options[:bucket],
+        build_cache_location: options[:build_cache_location],
         image_name: options[:image_name],
         named_tag: options[:named_tag],
+        skip_push: options[:skip_push],
         region: options[:region],
         clusters: options[:clusters],
         ecs_task_defs: options[:ecs_task_defs],
@@ -104,7 +110,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -135,7 +141,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Bootstrap failed.'

data/lib/shipitron/client.rb

@@ -0,0 +1,9 @@
+require 'shipitron'
+
+module Shipitron
+  module Client
+    STARTED_BY = 'shipitron'
+    # Use this for testing.
+    # STARTED_BY = 'shipitron-dev'
+  end
+end

data/lib/shipitron/client/bootstrap_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/register_ecs_task_definitions'
 require 'shipitron/client/create_ecs_services'
 

data/lib/shipitron/client/create_ecs_services.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/mustache_yaml_parser'
 require 'securerandom'
@@ -29,13 +30,12 @@ module Shipitron
 
           service_def = Smash.load(
             path.to_s,
-            parser: MustacheYamlParser.new(
-              context: {
-                cluster: cluster_name,
-                revision: nil, # ECS will default to latest ACTIVE
-                count: service_count
-              }
-            )
+            parser: MustacheYamlParser,
+            context: {
+              cluster: cluster_name,
+              revision: nil, # ECS will default to latest ACTIVE
+              count: service_count
+            }
           ).merge(
             client_token: SecureRandom.uuid
           )

data/lib/shipitron/client/deploy_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/load_application_config'
 require 'shipitron/client/load_templates'
 require 'shipitron/client/fetch_clusters'

data/lib/shipitron/client/ensure_deploy_not_running.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 
 # Note: This is a best effort client side check to make sure there
@@ -21,7 +22,7 @@ module Shipitron
             begin
               response = ecs_client(region: cluster.region).list_tasks(
                 cluster: cluster.name,
-                started_by: 'shipitron',
+                started_by: Shipitron::Client::STARTED_BY,
                 max_results: 1,
                 desired_status: status
               )

data/lib/shipitron/client/fetch_clusters.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'resolv'
 
 module Shipitron

data/lib/shipitron/client/force_deploy.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/client/load_application_config'
 require 'shipitron/client/fetch_clusters'

data/lib/shipitron/client/load_application_config.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/post_build'
 
 module Shipitron
@@ -11,7 +12,9 @@ module Shipitron
       def call
         context.repository_url = config.repository
         context.repository_branch = config.repository_branch
+        context.registry = config.registry
         context.s3_cache_bucket = config.cache_bucket
+        context.build_cache_location = config.build_cache_location
         context.image_name = config.image_name
         context.named_tag = begin
                               if config.named_tag.nil?
@@ -20,6 +23,7 @@ module Shipitron
                                 config.named_tag
                               end
                             end
+        context.skip_push = config.skip_push
         context.build_script = config.build_script
         context.post_builds = begin
                                 if config.post_builds.nil?

data/lib/shipitron/client/load_templates.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 
 module Shipitron
   module Client

data/lib/shipitron/client/register_ecs_task_definitions.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/mustache_yaml_parser'
 
@@ -26,11 +27,10 @@ module Shipitron
 
           task_def = Smash.load(
             path.to_s,
-            parser: MustacheYamlParser.new(
-              context: {
-                tag: 'latest'
-              }
-            )
+            parser: MustacheYamlParser,
+            context: {
+              tag: 'latest'
+            }
           )
 
           begin