shipitron 1.0.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: feb18a95f97617bac70d81fa89272203c5ee7aafc2e13f83f0f26bfd12bfbbd8
-  data.tar.gz: 66c40b51b8802ea914d40e8a5738066c1d3789c47ea92fcdb346e00bbf9de87c
+  metadata.gz: 9c5423fc9dbd202488e45ab382ad2bc37de889d909179b6e65a33d362ad9d20c
+  data.tar.gz: ea5bea8068a0d5d2ac0787c5590d53c3bf5a7f88ba6d646474d760b21dd4b707
 SHA512:
-  metadata.gz: 544be4870e87d518c4ac12cc0deb607921ca8f1972b970981d5e9a2ad6b00df43caed53cce5faed0627a088d1036ff8d59fd600780fed8261b2c0906b979abca
-  data.tar.gz: d935dfb8223048b941e5c77819b3e22ba9676c9f8df88ef0f97940efaf7db6772659de84e23e4755c7274032329b324c992909198d458739f10c254d2af51e4c
+  metadata.gz: 9983df8f5f01ea66a28e4bf7499bfed81bf49bc0127895669d003176861ad6db04b48a8c351215cd17ab927b1eb4b9087f203075206e2967be01429c8cd15d21
+  data.tar.gz: b5f1be2174a00e52ec0fb5df6ee04a64ac83ed73f09e6651fcf5655cb9a0c85f1746cbf30bf6bd8388f97668aac129bd1ef32980e79c44ffd645f656fe9213e5

data/Dockerfile

@@ -1,24 +1,23 @@
-FROM ruby:2.5.0-alpine3.7 as cache
+FROM ruby:2.7.1-alpine as cache
 COPY cache/ /tmp/
 RUN   cd /usr/local/bundle && \
     ([ -f /tmp/bundler-data.tar.gz ] && \
     tar -zxf /tmp/bundler-data.tar.gz && \
     rm /tmp/bundler-data.tar.gz) || true
 
-FROM ruby:2.5.0-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.1-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
     addgroup -g 1101 docker && \
     addgroup shipitron docker
 
-ENV GOSU_VERSION 1.10
-ENV DUMB_INIT_VERSION 1.2.0
-
-RUN apk add --no-cache ca-certificates openssl tini su-exec
-
 RUN apk add --no-cache \
+    ca-certificates \
+    openssl \
+    tini \
+    su-exec \
     build-base \
     git \
     openssh-client \
@@ -28,6 +27,27 @@ RUN apk add --no-cache \
     wget \
     jq
 
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.4.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.7.2
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.4.2
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+USER root
 ENV USE_BUNDLE_EXEC true
 ENV BUNDLE_GEMFILE /shipitron/Gemfile
 

data/Dockerfile.release

@@ -1,34 +1,16 @@
-FROM ruby:2.5.0-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.1-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
     addgroup -g 1101 docker && \
     addgroup shipitron docker
 
-ENV GOSU_VERSION 1.10
-ENV DUMB_INIT_VERSION 1.2.0
-
-RUN apk add --no-cache ca-certificates gnupg openssl && \
-    mkdir -p /tmp/build && \
-    cd /tmp/build && \
-    gpg --keyserver pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && \
-    wget -O gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" && \
-    wget -O gosu.asc "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64.asc" && \
-    gpg --verify gosu.asc && \
-    chmod +x gosu && \
-    cp gosu /bin/gosu && \
-    wget https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_amd64 && \
-    wget https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/sha256sums && \
-    grep dumb-init_${DUMB_INIT_VERSION}_amd64$ sha256sums | sha256sum -c && \
-    chmod +x dumb-init_${DUMB_INIT_VERSION}_amd64 && \
-    cp dumb-init_${DUMB_INIT_VERSION}_amd64 /bin/dumb-init && \
-    cd /tmp && \
-    rm -rf /tmp/build && \
-    apk del gnupg && \
-    rm -rf /root/.gnupg
-
 RUN apk add --no-cache \
+    ca-certificates \
+    openssl \
+    tini \
+    su-exec \
     build-base \
     git \
     openssh-client \
@@ -40,7 +22,7 @@ RUN apk add --no-cache \
 
 WORKDIR /app
 
-ENV SHIPITRON_VERSION=1.0.0
+ENV SHIPITRON_VERSION=1.3.0
 
 RUN gem install shipitron -v ${SHIPITRON_VERSION} && \
     mkdir -p /home/shipitron/.ssh && \
@@ -49,6 +31,5 @@ RUN gem install shipitron -v ${SHIPITRON_VERSION} && \
 
 COPY scripts/docker-entrypoint.sh /docker-entrypoint.sh
 
-ENV DUMB_INIT_SETSID 0
-ENTRYPOINT ["/docker-entrypoint.sh"]
+ENTRYPOINT ["/sbin/tini", "-g", "--", "/docker-entrypoint.sh"]
 CMD ["help"]

data/Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org'
 
 gemspec
+gem 'irb'

data/README.md

@@ -9,6 +9,7 @@ applications:
   dummy-app:
     repository: git@github.com:outstand/dummy-app
     cache_bucket: bucket
+    build_cache_location: tmp/build-cache.tar.gz
     image_name: outstand/dummy-app
     build_script: shipitron/build.sh
     post_builds:
@@ -30,20 +31,55 @@ applications:
 - Add deploy ref key to `shipitron/<app name>/deploy_ref_key`
 - `docker run -it --rm -v shipitron.yml:/shipitron/config/shipitron.yml outstand/shipitron:<version> deploy <app>`
 
+### New shipitron config file keys:
+
+- registry (specifies an alternate docker registry):
+```yaml
+---
+applications:
+  foobar:
+    registry: 12345.dkr.ecr.us-east-1.amazonaws.com
+```
+
+- skip_push (shipitron will skip pushing the docker images; use if the build script takes care of this):
+```yaml
+---
+applications:
+  foobar:
+    skip_push: true
+```
+Additionally, the build script will receive the named tag in use as `$2`.
+
+### Containerized tool support
+Shipitron now supports starting other containers as part of the build process. We're using this to use the aws cli to transfer files to/from s3. Shipitron's task definition needs to have a task scoped docker volume added with the name `shipitron-home`. This volume will be mounted at `/home/shipitron` and should be shared with any new containers:
+
+```
+shipitron_home_volume = FindDockerVolumeName.call!(
+  container_name: 'shipitron',
+  volume_search: /shipitron-home/
+).volume_name
+
+docker run ... -v #{shipitron_home_volume}:/home/shipitron ...
+```
+
+This allows the containers to share data with each other. ECS will automatically clean up the task scoped container.
+
+If a containerized tool requires access to AWS resources, be sure to pass the `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` to it so it can inherit any task roles.
+
 ## Development
 
-- `docker volume create --name shipitron_fog`
 - `./build_dev.sh`
-- `docker run -it --rm -v $(pwd):/shipitron -v shipitron_fog:/fog -e FOG_LOCAL=true -w /shipitron outstand/shipitron:dev rspec spec` to run specs
-- `APP_PATH=/path/to/app`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app>` to run client side
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --simulate` to get the arguments for `server_deploy` below
-- `docker run -it --rm --dns 10.10.10.2 -v $(pwd):/shipitron -v $APP_PATH:/app -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock -v shipitron_fog:/fog -e FOG_LOCAL=true -e CONSUL_HOST=consul outstand/shipitron:dev server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
+- `dev run --rm shipitron_specs` to run specs
+- Set the application path in the volumes section in `docker-compose.yml`.
+- `dev run --rm shipitron deploy <app>` to run client side
+- `dev run --rm shipitron deploy <app> --simulate` to get the arguments for `server_deploy` below
+- `dev run --rm -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock shipitron server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
 
 Running a dev version in production:
+- Update `Shipitron::Client::STARTED_BY`
 - `docker push outstand/shipitron:dev`
 - Update config to use `shipitron_task: shipitron-dev`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --debug`
+- `dev run --rm shipitron deploy <app> --debug`
 
 To release a new version:
 - Update the version number in `version.rb` and `Dockerfile.release` and commit the result.
@@ -51,4 +87,6 @@ To release a new version:
 - `docker run -it --rm -v ~/.gitconfig:/root/.gitconfig -v ~/.gitconfig.user:/root/.gitconfig.user -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v ~/.gem:/root/.gem -w /shipitron outstand/shipitron:dev rake release`
 - `docker build -t outstand/shipitron:VERSION -f Dockerfile.release .`
 - `docker push outstand/shipitron:VERSION`
+- `docker tag outstand/shipitron:VERSION outstand/shipitron:latest`
+- `docker push outstand/shipitron:latest`
 - Update ECS task definition with new version

data/build_dev.sh

@@ -20,7 +20,7 @@ trap cleanup EXIT
 build_args=''
 mkdir -p ${bundler_data_dir}
 
-docker build -t ${build_image} -f ${dockerfile} ${build_args} .
+docker build --pull -t ${build_image} -f ${dockerfile} ${build_args} .
 
 docker run -t --cidfile=${bundler_data_dir}/cidfile -w /usr/local/bundle ${build_image} tar -zcf /tmp/bundler-data.tar.gz .
 tar_container=$(cat ${bundler_data_dir}/cidfile)

data/docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.8'
+services:
+  shipitron:
+    image: outstand/shipitron:dev
+    volumes:
+      - ~/dev/pages:/app # Set this to the application to be shipitron'd
+      - shipitron-home:/home/shipitron
+      - ~/.config/shipitron:/home/shipitron/.config/shipitron
+      - .:/shipitron
+  shipitron_specs:
+    image: outstand/shipitron:dev
+    command: rspec spec
+    environment:
+      FOG_LOCAL: 'true'
+    working_dir: /shipitron
+    volumes:
+      - fog:/fog
+      - shipitron-home:/home/shipitron
+      - ~/.config/shipitron:/home/shipitron/.config/shipitron
+      - .:/shipitron
+
+volumes:
+  fog:
+  shipitron-home:

data/lib/shipitron/cli.rb

@@ -27,20 +27,47 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
       end
     end
 
+    desc 'force_deploy <app>', 'Forces a redeploy of the app'
+    option :config_file, default: 'shipitron/config.yml'
+    option :secrets_file, default: '~/.config/shipitron/secrets.yml'
+    option :debug, type: :boolean, default: false
+    def force_deploy(app)
+      setup(
+        config_file: options[:config_file],
+        secrets_file: options[:secrets_file]
+      )
+
+      require 'shipitron/client/force_deploy'
+      result = Client::ForceDeploy.call(
+        application: app
+      )
+
+      if result.failure?
+        result.error_messages.each do |error|
+          Logger.fatal error
+        end
+        Logger.fatal 'Deploy failed.'
+      end
+    end
+
+
     desc 'server_deploy', 'Server-side component of deploy'
     option :name, required: true
     option :repository, required: true
     option :repository_branch, default: 'master'
+    option :registry, default: nil
     option :bucket, required: true
+    option :build_cache_location, default: 'tmp/build-cache.tar.gz'
     option :image_name, required: true
     option :named_tag, default: 'latest'
+    option :skip_push, type: :boolean, default: false
     option :region, required: true
     option :clusters, type: :array, required: true
     option :ecs_task_defs, type: :array, required: true
@@ -61,9 +88,12 @@ module Shipitron
         application: options[:name],
         repository_url: options[:repository],
         repository_branch: options[:repository_branch],
+        registry: options[:registry],
         s3_cache_bucket: options[:bucket],
+        build_cache_location: options[:build_cache_location],
         image_name: options[:image_name],
         named_tag: options[:named_tag],
+        skip_push: options[:skip_push],
         region: options[:region],
         clusters: options[:clusters],
         ecs_task_defs: options[:ecs_task_defs],
@@ -80,7 +110,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -111,7 +141,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Bootstrap failed.'

data/lib/shipitron/client.rb

@@ -0,0 +1,9 @@
+require 'shipitron'
+
+module Shipitron
+  module Client
+    STARTED_BY = 'shipitron'
+    # Use this for testing.
+    # STARTED_BY = 'shipitron-dev'
+  end
+end

data/lib/shipitron/client/bootstrap_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/register_ecs_task_definitions'
 require 'shipitron/client/create_ecs_services'
 

data/lib/shipitron/client/create_ecs_services.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/mustache_yaml_parser'
 require 'securerandom'

data/lib/shipitron/client/deploy_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/load_application_config'
 require 'shipitron/client/load_templates'
 require 'shipitron/client/fetch_clusters'

data/lib/shipitron/client/ensure_deploy_not_running.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 
 # Note: This is a best effort client side check to make sure there
@@ -21,7 +22,7 @@ module Shipitron
             begin
               response = ecs_client(region: cluster.region).list_tasks(
                 cluster: cluster.name,
-                started_by: 'shipitron',
+                started_by: Shipitron::Client::STARTED_BY,
                 max_results: 1,
                 desired_status: status
               )

data/lib/shipitron/client/fetch_clusters.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'resolv'
 
 module Shipitron

data/lib/shipitron/client/force_deploy.rb

@@ -0,0 +1,58 @@
+require 'shipitron'
+require 'shipitron/client'
+require 'shipitron/ecs_client'
+require 'shipitron/client/load_application_config'
+require 'shipitron/client/fetch_clusters'
+require 'shipitron/client/ensure_deploy_not_running'
+
+module Shipitron
+  module Client
+    class ForceDeploy
+      include Metaractor
+      include Interactor::Organizer
+      include EcsClient
+
+      required :application
+
+      organize [
+        LoadApplicationConfig,
+        FetchClusters,
+        EnsureDeployNotRunning
+      ]
+
+      def call
+        Logger.info "==> Force deploying #{application}"
+
+        super
+
+        context.clusters ||= []
+        context.ecs_services ||= []
+
+        begin
+          context.clusters.each do |cluster|
+            context.ecs_services.each do |service|
+              ecs_client(region: cluster.region).update_service(
+                cluster: cluster.name,
+                service: service,
+                force_new_deployment: true
+              )
+            end
+          end
+        rescue Aws::ECS::Errors::ServiceError => e
+          fail_with_errors!(messages: [
+            "Error: #{e.message}",
+            e.backtrace.join("\n")
+          ])
+        end
+
+        Logger.info "==> Done"
+      end
+
+      private
+      def application
+        context.application
+      end
+
+    end
+  end
+end

data/lib/shipitron/client/load_application_config.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/post_build'
 
 module Shipitron
@@ -11,7 +12,9 @@ module Shipitron
       def call
         context.repository_url = config.repository
         context.repository_branch = config.repository_branch
+        context.registry = config.registry
         context.s3_cache_bucket = config.cache_bucket
+        context.build_cache_location = config.build_cache_location
         context.image_name = config.image_name
         context.named_tag = begin
                               if config.named_tag.nil?
@@ -20,6 +23,7 @@ module Shipitron
                                 config.named_tag
                               end
                             end
+        context.skip_push = config.skip_push
         context.build_script = config.build_script
         context.post_builds = begin
                                 if config.post_builds.nil?

data/lib/shipitron/client/load_templates.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 
 module Shipitron
   module Client

data/lib/shipitron/client/register_ecs_task_definitions.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/mustache_yaml_parser'
 

data/lib/shipitron/client/run_ecs_tasks.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shellwords'
 require 'base64'
@@ -16,6 +17,7 @@ module Shipitron
       required :shipitron_task
       required :repository_url
       required :s3_cache_bucket
+      required :build_cache_location
       required :image_name
       required :named_tag
       required :ecs_task_defs
@@ -23,9 +25,11 @@ module Shipitron
       optional :ecs_services
       optional :ecs_service_templates
       optional :build_script
+      optional :skip_push
       optional :post_builds
       optional :simulate
       optional :repository_branch
+      optional :registry
 
       before do
         context.post_builds ||= []
@@ -72,7 +76,7 @@ module Shipitron
               ]
             },
             count: 1,
-            started_by: 'shipitron'
+            started_by: Shipitron::Client::STARTED_BY
           )
 
           if !response.failures.empty?
@@ -108,6 +112,7 @@ module Shipitron
           '--name', context.application,
           '--repository', context.repository_url,
           '--bucket', context.s3_cache_bucket,
+          '--build-cache-location', context.build_cache_location,
           '--image-name', context.image_name,
           '--named-tag', context.named_tag,
           '--region', cluster.region,
@@ -123,10 +128,18 @@ module Shipitron
             ary.concat(context.ecs_services)
           end
 
+          if context.registry != nil
+            ary.concat ['--registry', context.registry]
+          end
+
           if context.build_script != nil
             ary.concat ['--build-script', context.build_script]
           end
 
+          if context.skip_push != nil
+            ary.concat ['--skip-push', context.skip_push.to_s]
+          end
+
           if !context.post_builds.empty?
             ary << '--post-builds'
             ary.concat(context.post_builds.map(&:to_s))

data/lib/shipitron/docker_image.rb

@@ -2,6 +2,7 @@ require 'shipitron'
 
 module Shipitron
   class DockerImage < Hashie::Dash
+    property :registry
     property :name
     property :tag
 
@@ -13,7 +14,9 @@ module Shipitron
         tag_str = tag_str.dup.prepend(':')
       end
 
-      "#{name}#{tag_str}"
+      name_with_registry = [registry, name].compact.join('/')
+
+      "#{name_with_registry}#{tag_str}"
     end
 
     def to_s

data/lib/shipitron/find_docker_volume_name.rb

@@ -0,0 +1,68 @@
+require 'excon'
+require 'json'
+
+module Shipitron
+  class FindDockerVolumeName
+    include Metaractor
+
+    required :container_name
+    required :volume_search
+
+    def call
+      volumes = container_volumes(container_name: container_name)
+
+      volume_metadata = volumes.find do |volume|
+        volume['DockerName'] =~ volume_search
+      end
+
+      if volume_metadata.nil?
+        raise 'Unable to find shipitron-home volume!'
+      end
+
+      context.volume_name = volume_metadata['DockerName']
+    end
+
+    private
+    def container_name
+      context.container_name
+    end
+
+    def volume_search
+      context.volume_search
+    end
+
+    def container_volumes(container_name:)
+      container_metadata = self.task_metadata['Containers'].find do |container|
+        container['Name'] == container_name
+      end
+
+      return {} if container_metadata.nil?
+
+      container_metadata['Volumes']
+    end
+
+    def task_metadata
+      return @task_metadata if defined?(@task_metadata)
+
+      begin
+        response = Excon.get(
+          "#{ENV['ECS_CONTAINER_METADATA_URI_V4']}/task",
+          expects: [200],
+          connect_timeout: 5,
+          read_timeout: 5,
+          write_timeout: 5,
+          tcp_nodelay: true
+        )
+
+        Logger.debug "Metadata result:"
+        Logger.debug(response.body)
+        Logger.debug "\n"
+
+        @task_metadata = JSON.parse(response.body)
+      rescue
+        Logger.info "Metadata uri failed"
+        {}
+      end
+    end
+  end
+end

data/lib/shipitron/s3_copy.rb

@@ -0,0 +1,46 @@
+require 'shipitron'
+require 'shipitron/find_docker_volume_name'
+
+module Shipitron
+  class S3Copy
+    include Metaractor
+
+    required :source
+    required :destination
+    required :region
+
+    def call
+      if ENV['FOG_LOCAL']
+        Logger.info `cp #{source.gsub('s3://', '/fog/')} #{destination.gsub('s3://', '/fog/')}`
+        if $? != 0
+          fail_with_error!(message: 'Failed to transfer to/from s3 (mocked).')
+        end
+      else
+        Logger.info "S3 Copy from #{source} to #{destination}"
+
+        shipitron_home_volume = FindDockerVolumeName.call!(
+          container_name: 'shipitron',
+          volume_search: /shipitron-home/
+        ).volume_name
+
+        Logger.info `docker run --rm -t -v #{shipitron_home_volume}:/home/shipitron -e AWS_CONTAINER_CREDENTIALS_RELATIVE_URI amazon/aws-cli:latest --region #{region} s3 cp #{source} #{destination} --quiet --only-show-errors`
+        if $? != 0
+          fail_with_error!(message: 'Failed to transfer to/from s3.')
+        end
+      end
+    end
+
+    private
+    def source
+      context.source
+    end
+
+    def destination
+      context.destination
+    end
+
+    def region
+      context.region
+    end
+  end
+end

data/lib/shipitron/server/deploy_application.rb

@@ -19,6 +19,7 @@ module Shipitron
       required :application
       required :repository_url
       required :s3_cache_bucket
+      required :build_cache_location
       required :docker_image
       required :named_tag
       required :region
@@ -30,6 +31,8 @@ module Shipitron
       optional :build_script
       optional :post_builds
       optional :repository_branch
+      optional :skip_push, default: false
+      optional :registry
 
       around do |interactor|
         if ENV['CONSUL_HOST'].nil?

data/lib/shipitron/server/docker/build_image.rb

@@ -13,6 +13,9 @@ module Shipitron
         required :application
         required :docker_image
         required :git_sha
+        required :named_tag
+        required :region
+        optional :registry
 
         organize [
           DownloadBuildCache,

data/lib/shipitron/server/docker/configure.rb

@@ -1,5 +1,6 @@
 require 'shipitron'
 require 'shipitron/consul_keys'
+require 'json'
 
 module Shipitron
   module Server
@@ -9,22 +10,48 @@ module Shipitron
         include ConsulKeys
 
         required :application
+        optional :registry
 
         before do
           configure_consul_client!
         end
 
         def call
-          docker_auth = begin
-                          key = fetch_key(key: "shipitron/#{application}/docker_auth")
-                          key = fetch_key!(key: 'shipitron/docker_auth') if key.nil?
-                          key
-                        end
-          auth_file = Pathname.new('/home/shipitron/.docker/config.json')
-          auth_file.parent.mkpath
-          auth_file.open('wb') do |file|
-            file.puts(docker_auth.to_s)
-            file.chmod(0600)
+          username = fetch_scoped_key('docker_user')
+          password = fetch_scoped_key('docker_password')
+
+          if username && password
+            Logger.info `docker login --username #{username} --password #{password}`
+            if $? != 0
+              fail_with_error!(message: 'Docker login failed.')
+            end
+          end
+
+          if registry
+            case registry
+            when /docker\.io/
+              # do nothing
+            when /\d+\.dkr\.ecr\.us-east-1\.amazonaws\.com/
+              # ECR
+              config_file = Pathname.new('/home/shipitron/.docker/config.json')
+              config_file.parent.mkpath
+
+              config_hash = {}
+              if config_file.file?
+                config_file.open('rb') do |file|
+                  json = file.read
+                  config_hash = JSON.parse(json) rescue {}
+                end
+              end
+
+              config_hash['credHelpers'] ||= {}
+              config_hash['credHelpers'][registry] = 'ecr-login'
+
+              config_file.open('wb') do |file|
+                file.puts(JSON.generate(config_hash))
+                file.chmod(0600)
+              end
+            end
           end
         end
 
@@ -33,6 +60,15 @@ module Shipitron
           context.application
         end
 
+        def registry
+          context.registry
+        end
+
+        def fetch_scoped_key(key)
+          value = fetch_key(key: "shipitron/#{application}/#{key}")
+          value = fetch_key(key: "shipitron/#{key}") if value.nil?
+          value
+        end
       end
     end
   end

data/lib/shipitron/server/docker/push_image.rb

@@ -8,8 +8,11 @@ module Shipitron
 
         required :docker_image
         required :named_tag
+        optional :skip_push, default: false
 
         def call
+          return if context.skip_push
+
           Logger.info "Pushing docker image #{docker_image} and #{docker_image.name_with_tag(named_tag)}"
 
           Logger.info `docker tag #{docker_image} #{docker_image.name_with_tag(named_tag)}`

data/lib/shipitron/server/docker/run_build_script.rb

@@ -10,15 +10,14 @@ module Shipitron
         required :application
         required :docker_image
         required :git_sha
-        optional :build_script
-
-        before do
-          context.build_script ||= 'shipitron/build.sh'
-        end
+        required :named_tag
+        optional :build_script, default: 'shipitron/build.sh'
+        optional :registry
 
         def call
           Logger.info 'Building docker image'
 
+          docker_image.registry = registry if registry != nil
           docker_image.tag = git_sha
 
           FileUtils.cd("/home/shipitron/#{application}") do
@@ -27,7 +26,7 @@ module Shipitron
             end
 
             cmd = TTY::Command.new
-            result = cmd.run!("#{build_script} #{docker_image}")
+            result = cmd.run!("#{build_script} #{docker_image} #{named_tag}")
 
             if result.failure?
               fail_with_error!(message: "build script exited with non-zero code: #{result.exit_status}")
@@ -48,9 +47,17 @@ module Shipitron
           context.git_sha
         end
 
+        def named_tag
+          context.named_tag
+        end
+
         def build_script
           context.build_script
         end
+
+        def registry
+          context.registry
+        end
       end
     end
   end

data/lib/shipitron/server/download_build_cache.rb

@@ -1,5 +1,6 @@
 require 'shipitron'
 require 'shipitron/fetch_bucket'
+require 'shipitron/s3_copy'
 
 module Shipitron
   module Server
@@ -8,20 +9,28 @@ module Shipitron
 
       required :application
       required :s3_cache_bucket
+      required :build_cache_location
+      required :region
 
       def call
         Logger.info "Downloading build cache from bucket #{s3_cache_bucket}"
 
-        s3_file = bucket.files.get("#{application}.build-cache.tar.gz")
+        s3_file = bucket.files.head("#{application}.build-cache.archive")
         if s3_file.nil?
           Logger.warn 'Build cache not found.'
           return
         end
 
-        build_cache = Pathname.new("/home/shipitron/#{application}/tmp/build-cache.tar.gz")
+        build_cache = Pathname.new("/home/shipitron/#{application}/#{build_cache_location}")
         build_cache.parent.mkpath
-        build_cache.open('wb') do |local_file|
-          local_file.write(s3_file.body)
+
+        result = S3Copy.call(
+          source: "s3://#{s3_cache_bucket}/#{application}.build-cache.archive",
+          destination: build_cache.to_s,
+          region: context.region
+        )
+        if result.failure?
+          fail_with_error!(message: 'Failed to download build cache!')
         end
 
         Logger.info 'Download complete.'
@@ -36,6 +45,10 @@ module Shipitron
         context.s3_cache_bucket
       end
 
+      def build_cache_location
+        context.build_cache_location
+      end
+
       def bucket
         @bucket ||= FetchBucket.call!(name: s3_cache_bucket).bucket
       end

data/lib/shipitron/server/git/update_cache.rb

@@ -25,6 +25,7 @@ module Shipitron
           else
             Logger.info 'Fetching new git commits'
             FileUtils.cd('/home/shipitron/git-cache') do
+              `git gc --auto`
               `git fetch -f #{Shellwords.escape repository_url} #{Shellwords.escape repository_branch}:#{Shellwords.escape repository_branch}`
             end
           end

data/lib/shipitron/server/transform_cli_args.rb

@@ -12,7 +12,9 @@ module Shipitron
       required :application
       required :repository_url
       optional :repository_branch
+      optional :registry
       required :s3_cache_bucket
+      required :build_cache_location
       required :image_name
       required :named_tag
       required :region
@@ -22,6 +24,7 @@ module Shipitron
       optional :ecs_services
       optional :ecs_service_templates
       optional :build_script
+      optional :skip_push, default: false
       optional :post_builds
 
       before do
@@ -36,12 +39,15 @@ module Shipitron
           application
           repository_url
           repository_branch
+          registry
           s3_cache_bucket
+          build_cache_location
           named_tag
           region
           clusters
           ecs_services
           build_script
+          skip_push
         ].each_with_object(cli_args) { |k, args| args[k] = context[k] }
 
         cli_args.docker_image = DockerImage.new(name: context.image_name)

data/lib/shipitron/server/update_ecs_task_definitions.rb

@@ -14,11 +14,12 @@ module Shipitron
       required :docker_image
       required :ecs_task_defs
       optional :ecs_task_def_templates
+      optional :registry
 
       before do
         context.ecs_task_def_templates ||= []
         context.templates = context.ecs_task_def_templates
-        context.template_context = { tag: docker_image.tag }
+        context.template_context = { tag: docker_image.tag, registry: context.registry }
       end
 
       organize [

data/lib/shipitron/server/upload_build_cache.rb

@@ -8,21 +8,27 @@ module Shipitron
 
       required :application
       required :s3_cache_bucket
+      required :build_cache_location
+      required :region
 
       def call
         Logger.info "Uploading build cache to bucket #{s3_cache_bucket}"
 
-        build_cache = Pathname.new("/home/shipitron/#{application}/tmp/build-cache.tar.gz")
+        build_cache = Pathname.new("/home/shipitron/#{application}/#{build_cache_location}")
         unless build_cache.exist?
           Logger.warn 'Build cache not found.'
           return
         end
 
-        build_cache.open('rb') do |local_file|
-          bucket.files.create(
-            key: "#{application}.build-cache.tar.gz",
-            body: local_file.read
-          )
+        result = S3Copy.call(
+          source: build_cache.to_s,
+          destination: "s3://#{s3_cache_bucket}/#{application}.build-cache.archive",
+          region: context.region
+        )
+        if result.failure?
+          Logger.warn 'Failed to upload build cache!'
+        else
+          Logger.info 'Upload complete.'
         end
       end
 
@@ -35,8 +41,8 @@ module Shipitron
         context.s3_cache_bucket
       end
 
-      def bucket
-        @bucket ||= FetchBucket.call!(name: s3_cache_bucket).bucket
+      def build_cache_location
+        context.build_cache_location
       end
     end
   end

data/lib/shipitron/version.rb

@@ -1,3 +1,3 @@
 module Shipitron
-  VERSION = '1.0.0'
+  VERSION = '1.3.0'
 end

data/scripts/docker-entrypoint.sh

@@ -1,6 +1,15 @@
 #!/bin/sh
 set -e
 
+chown_dir() {
+  dir=$1
+  if [ "$(stat -c %u ${dir})" = '0' ]; then
+    chown -R shipitron:shipitron $dir
+  fi
+}
+
+chown_dir /home/shipitron
+
 if [ -n "$USE_BUNDLE_EXEC" ]; then
   BINARY="bundle exec shipitron"
 else

data/shipitron.gemspec

@@ -17,21 +17,22 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency 'thor', '~> 0.20'
+  spec.add_runtime_dependency 'thor', '~> 1.0'
   spec.add_runtime_dependency 'aws-sdk-ecs', '~> 1.8'
-  spec.add_runtime_dependency 'hashie', '~> 3.5'
-  spec.add_runtime_dependency 'metaractor', '~> 0.5'
+  spec.add_runtime_dependency 'hashie', '~> 4.1'
+  spec.add_runtime_dependency 'metaractor', '~> 3.0'
   spec.add_runtime_dependency 'diplomat', '~> 2.0'
-  spec.add_runtime_dependency 'fog-aws', '~> 2.0'
+  spec.add_runtime_dependency 'fog-aws', '~> 3.6'
   spec.add_runtime_dependency 'mime-types', '~> 3.1'
   spec.add_runtime_dependency 'minitar', '~> 0.6'
   spec.add_runtime_dependency 'mustache', '~> 1.0'
   spec.add_runtime_dependency 'tty-command', '~> 0.7'
   spec.add_runtime_dependency 'tty-table', '~> 0.9'
   spec.add_runtime_dependency 'pastel', '~> 0.7'
+  spec.add_runtime_dependency 'excon', '~> 0.76'
 
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "pry-byebug", "~> 3.5"
   spec.add_development_dependency "rspec", "~> 3.7"
   spec.add_development_dependency "fivemat", "~> 1.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shipitron
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Ryan Schlesinger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ecs
   requirement: !ruby/object:Gem::Requirement
@@ -44,28 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: metaractor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: diplomat
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
@@ -178,34 +178,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.76'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.76'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -280,14 +294,17 @@ files:
 - README.md
 - Rakefile
 - build_dev.sh
+- docker-compose.yml
 - exe/shipitron
 - lib/shipitron.rb
 - lib/shipitron/cli.rb
+- lib/shipitron/client.rb
 - lib/shipitron/client/bootstrap_application.rb
 - lib/shipitron/client/create_ecs_services.rb
 - lib/shipitron/client/deploy_application.rb
 - lib/shipitron/client/ensure_deploy_not_running.rb
 - lib/shipitron/client/fetch_clusters.rb
+- lib/shipitron/client/force_deploy.rb
 - lib/shipitron/client/load_application_config.rb
 - lib/shipitron/client/load_templates.rb
 - lib/shipitron/client/register_ecs_task_definitions.rb
@@ -298,10 +315,12 @@ files:
 - lib/shipitron/ecs_client.rb
 - lib/shipitron/ecs_task_def.rb
 - lib/shipitron/fetch_bucket.rb
+- lib/shipitron/find_docker_volume_name.rb
 - lib/shipitron/logger.rb
 - lib/shipitron/mustache_yaml_parser.rb
 - lib/shipitron/parse_templates.rb
 - lib/shipitron/post_build.rb
+- lib/shipitron/s3_copy.rb
 - lib/shipitron/server/deploy_application.rb
 - lib/shipitron/server/docker/build_image.rb
 - lib/shipitron/server/docker/configure.rb
@@ -345,8 +364,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A deployment tool for use with Docker and ECS.