shipitron 1.2.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bd4d376f639bb5ebd80df8a467b8aaca8eac7f2d50ebb6a404683dbaea2fde5
-  data.tar.gz: d368aa28072880ae2580ed73fca2c78d40328b497d21d3a1d02139ccdfd77af5
+  metadata.gz: ce2ad45258aa11acc35b0c3041aab0fe3f20bb56f076764120f1b6466233f310
+  data.tar.gz: 938b06da85c43614c316c7d63a755440e8d7d61e6396ee0bc574040fda9bd817
 SHA512:
-  metadata.gz: 26eafd5c78aafa2946e78e400e0c132820ca9b7b57dbbf0846fce2e11f290db649c7257ffb76283fa2bd86176fe38fe322a79077836ef9afd1eaefa306b43a3d
-  data.tar.gz: cf99dd7b011c3beb293b428418930f55aa0990a73027deda2c6c1b7e6d420db7b5584af89457acc0a0318553ebc0ed3bc53c3f18d49860c5c947647164f2ea9f
+  metadata.gz: 5a93f308e3065ada355022a5a18977d903e00ae56387e98218bf26557101ca8ccbb1a2d7c1950c6c2849b0bc47da860a57dccb6d0365216df99670a90245ce2c
+  data.tar.gz: 2a041845cb6bc87c29366de8e39702fb691bec9efb7a2e53df8673d30bbd9988edaa3147fe68a98c09fe6be80b81aa8a3094ded4fef464df23622c421ea63017

data/.buildkite/pipeline.yml

@@ -0,0 +1,94 @@
+env:
+  BUILDKITE_PLUGIN_DOCKER_CACHE_S3_BUCKET: "outstand-buildkite-cache"
+  # BUILDKITE_PLUGIN_DOCKER_CACHE_VOLUME_DEBUG: "true"
+  BUILDKITE_PLUGIN_DOCKER_COMPOSE_SHELL: "false"
+  # BUILDKITE_PLUGIN_DOCKER_COMPOSE_UPLOAD_CONTAINER_LOGS: "always"
+  BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_RETRIES: 5
+  BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_RETRIES: 5
+  PLUGIN_DOCKER_COMPOSE_VERSION: "17bac3aaee1360e39381b89bb45513b11838238e"
+  PLUGIN_DOCKER_CACHE_VERSION: "f3d8feb52a25c69c75565e9f0b80375eae51850a"
+
+steps:
+  - label: ":docker: Build"
+    key: build
+    plugins:
+      - seek-oss/aws-sm#v2.2.1:
+          env:
+            DOCKER_LOGIN_PASSWORD: "/buildkite/docker_password"
+
+      - docker-login#v2.0.1:
+          username: outstandci
+
+      - ecr#v2.1.1:
+          login: true
+          region: "us-east-1"
+
+      - https://github.com/outstand/docker-compose-buildkite-plugin.git#${PLUGIN_DOCKER_COMPOSE_VERSION}:
+          build: specs
+          image-repository: 786715713882.dkr.ecr.us-east-1.amazonaws.com/ci-images
+          config:
+            - docker-compose.yml
+            - test.yml
+
+  - label: ":bundler: :rubygems:"
+    key: bundle_install
+    command: bundle install
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.2.1:
+          env:
+            DOCKER_LOGIN_PASSWORD: "/buildkite/docker_password"
+
+      - docker-login#v2.0.1:
+          username: outstandci
+
+      - ecr#v2.1.1:
+          login: true
+          region: "us-east-1"
+
+      - https://github.com/outstand/docker-compose-buildkite-plugin.git#${PLUGIN_DOCKER_COMPOSE_VERSION}:
+          run: specs
+          dependencies: false
+          config:
+            - docker-compose.yml
+            - test.yml
+
+      - https://github.com/outstand/docker-cache-buildkite-plugin.git#${PLUGIN_DOCKER_CACHE_VERSION}:
+          name: bundler-cache
+          keys:
+            - v1-bundler-cache-{{ arch }}-{{ checksum "shipitron.gemspec" }}-{{ checksum "Gemfile" }}
+            - v1-bundler-cache-{{ arch }}-
+          save: true
+          volumes:
+            - bundler-data
+
+  - label: ":ruby: Specs"
+    command: rspec spec
+    depends_on: bundle_install
+    plugins:
+      - seek-oss/aws-sm#v2.2.1:
+          env:
+            DOCKER_LOGIN_PASSWORD: "/buildkite/docker_password"
+          json-to-env:
+            - secret-id: "/buildkite/rails/env_vars"
+
+      - docker-login#v2.0.1:
+          username: outstandci
+
+      - ecr#v2.1.1:
+          login: true
+          region: "us-east-1"
+
+      - https://github.com/outstand/docker-compose-buildkite-plugin.git#${PLUGIN_DOCKER_COMPOSE_VERSION}:
+          run: specs
+          config:
+            - docker-compose.yml
+            - test.yml
+
+      - https://github.com/outstand/docker-cache-buildkite-plugin.git#${PLUGIN_DOCKER_CACHE_VERSION}:
+          name: bundler-cache
+          keys:
+            - v1-bundler-cache-{{ arch }}-{{ checksum "shipitron.gemspec" }}-{{ checksum "Gemfile" }}
+            - v1-bundler-cache-{{ arch }}-
+          volumes:
+            - bundler-data

data/.dockerignore

@@ -2,3 +2,4 @@
 /pkg/
 Gemfile.lock
 /tmp/
+/spec/fixtures/test_repo

data/.gitattributes

@@ -0,0 +1 @@
+spec/fixtures/test_repo.tgz filter=lfs diff=lfs merge=lfs -text

data/.gitignore

@@ -3,3 +3,4 @@
 Gemfile.lock
 /tmp/
 /cache/
+/spec/fixtures/test_repo

data/.rspec

@@ -1,3 +1,4 @@
 --color
 --require spec_helper
 --format Fivemat
+--format RSpec::Buildkite::AnnotationFormatter

data/Deskfile

@@ -0,0 +1,15 @@
+function bundle() {
+  dev run --rm shipitron bundle "$@"
+}
+
+function shipitron() {
+  dev run --rm shipitron "$@"
+}
+
+function rspec() {
+  dev run --rm specs rspec "$@"
+}
+
+function release_gem() {
+  dev run --rm release_gem
+}

data/Dockerfile

@@ -1,12 +1,7 @@
-FROM ruby:2.5.1-alpine3.7 as cache
-COPY cache/ /tmp/
-RUN   cd /usr/local/bundle && \
-    ([ -f /tmp/bundler-data.tar.gz ] && \
-    tar -zxf /tmp/bundler-data.tar.gz && \
-    rm /tmp/bundler-data.tar.gz) || true
+FROM outstand/fixuid as fixuid
 
-FROM ruby:2.5.0-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.3-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
@@ -15,31 +10,61 @@ RUN addgroup -S shipitron && \
 
 RUN apk add --no-cache \
     ca-certificates \
-    openssl \
+    openssl-dev \
     tini \
     su-exec \
     build-base \
     git \
+    git-lfs \
     openssh-client \
     perl \
     bash \
     curl \
     wget \
-    jq
+    jq \
+    cmake
 
+COPY --from=fixuid /usr/local/bin/fixuid /usr/local/bin/fixuid
+RUN chmod 4755 /usr/local/bin/fixuid && \
+      USER=shipitron && \
+      GROUP=shipitron && \
+      mkdir -p /etc/fixuid && \
+      printf "user: $USER\ngroup: $GROUP\n" > /etc/fixuid/config.yml
+
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.5.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.8.3
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.5.1
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+USER root
 ENV USE_BUNDLE_EXEC true
 ENV BUNDLE_GEMFILE /shipitron/Gemfile
 
+ENV BUNDLER_VERSION 2.2.21
+RUN gem install bundler -v ${BUNDLER_VERSION} -i /usr/local/lib/ruby/gems/$(ls /usr/local/lib/ruby/gems) --force
+
 WORKDIR /app
 COPY Gemfile shipitron.gemspec /shipitron/
 COPY lib/shipitron/version.rb /shipitron/lib/shipitron/
 
-COPY --from=cache /usr/local/bundle /usr/local/bundle
-RUN (bundle check || bundle install) && \
-      git config --global push.default simple
+RUN git config --global push.default simple
 COPY . /shipitron/
-RUN ln -s /shipitron/exe/shipitron /usr/local/bin/shipitron && \
-    mkdir -p /home/shipitron/.ssh && \
+RUN mkdir -p /home/shipitron/.ssh && \
     chown shipitron:shipitron /home/shipitron/.ssh && \
     chmod 700 /home/shipitron/.ssh
 

data/Dockerfile.release

@@ -1,5 +1,5 @@
-FROM ruby:2.5.1-alpine3.7
-MAINTAINER Ryan Schlesinger <ryan@outstand.com>
+FROM ruby:2.7.3-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
 
 RUN addgroup -S shipitron && \
     adduser -S -G shipitron shipitron && \
@@ -8,7 +8,7 @@ RUN addgroup -S shipitron && \
 
 RUN apk add --no-cache \
     ca-certificates \
-    openssl \
+    openssl-dev \
     tini \
     su-exec \
     build-base \
@@ -18,18 +18,41 @@ RUN apk add --no-cache \
     bash \
     curl \
     wget \
-    jq
+    jq \
+    cmake
+
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.5.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.8.3
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.5.1
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+USER root
 
 WORKDIR /app
 
-ENV SHIPITRON_VERSION=1.2.0
+ENV SHIPITRON_VERSION=1.4.0
 
 RUN gem install shipitron -v ${SHIPITRON_VERSION} && \
     mkdir -p /home/shipitron/.ssh && \
     chown shipitron:shipitron /home/shipitron/.ssh && \
     chmod 700 /home/shipitron/.ssh
 
-COPY scripts/docker-entrypoint.sh /docker-entrypoint.sh
+COPY scripts/release-entrypoint.sh /docker-entrypoint.sh
 
 ENTRYPOINT ["/sbin/tini", "-g", "--", "/docker-entrypoint.sh"]
 CMD ["help"]

data/Dockerfile.staging

@@ -0,0 +1,70 @@
+FROM ruby:2.7.3-alpine
+LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
+
+RUN addgroup -S shipitron && \
+    adduser -S -G shipitron shipitron && \
+    addgroup -g 1101 docker && \
+    addgroup shipitron docker
+
+RUN apk add --no-cache \
+    ca-certificates \
+    openssl-dev \
+    tini \
+    su-exec \
+    build-base \
+    git \
+    git-lfs \
+    openssh-client \
+    perl \
+    bash \
+    curl \
+    wget \
+    jq \
+    cmake
+
+ENV ECR_CREDENTIAL_HELPER_VERSION 0.5.0
+RUN cd /usr/local/bin && \
+      wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
+      chmod +x docker-credential-ecr-login
+
+ENV BUILDKIT_VERSION v0.8.3
+RUN cd /usr/local/bin && \
+      wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
+      tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
+      chmod +x buildctl && \
+      rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
+
+USER shipitron
+ENV BUILDX_VERSION v0.5.1
+RUN cd /home/shipitron && \
+      wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
+      mkdir -p ~/.docker/cli-plugins && \
+      mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
+      chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+USER root
+ENV USE_BUNDLE_EXEC true
+ENV BUNDLE_GEMFILE /shipitron/Gemfile
+
+ENV BUNDLER_VERSION 2.2.21
+RUN gem install bundler -v ${BUNDLER_VERSION} -i /usr/local/lib/ruby/gems/$(ls /usr/local/lib/ruby/gems) --force
+
+WORKDIR /shipitron
+COPY Gemfile shipitron.gemspec /shipitron/
+COPY lib/shipitron/version.rb /shipitron/lib/shipitron/
+COPY exe/shipitron /shipitron/exe/shipitron
+COPY .git/ /shipitron/
+
+RUN bundle install
+
+COPY . /shipitron/
+RUN mkdir -p /home/shipitron/.ssh && \
+    chown shipitron:shipitron /home/shipitron/.ssh && \
+    chmod 700 /home/shipitron/.ssh
+
+WORKDIR /app
+
+COPY scripts/release-entrypoint.sh /docker-entrypoint.sh
+
+ENTRYPOINT ["/sbin/tini", "-g", "--", "/docker-entrypoint.sh"]
+CMD ["help"]

data/Gemfile

@@ -1,3 +1,5 @@
 source 'https://rubygems.org'
 
 gemspec
+gem 'irb'
+gem 'rspec-buildkite', github: 'outstand/rspec-buildkite', branch: 'error-output'

data/README.md

@@ -31,27 +31,65 @@ applications:
 - Add deploy ref key to `shipitron/<app name>/deploy_ref_key`
 - `docker run -it --rm -v shipitron.yml:/shipitron/config/shipitron.yml outstand/shipitron:<version> deploy <app>`
 
+### New shipitron config file keys:
+
+- registry (specifies an alternate docker registry):
+```yaml
+---
+applications:
+  foobar:
+    registry: 12345.dkr.ecr.us-east-1.amazonaws.com
+```
+
+- skip_push (shipitron will skip pushing the docker images; use if the build script takes care of this):
+```yaml
+---
+applications:
+  foobar:
+    skip_push: true
+```
+Additionally, the build script will receive the named tag in use as `$2`.
+
+### Containerized tool support
+Shipitron now supports starting other containers as part of the build process. We're using this to use the aws cli to transfer files to/from s3. Shipitron's task definition needs to have a task scoped docker volume added with the name `shipitron-home`. This volume will be mounted at `/home/shipitron` and should be shared with any new containers:
+
+```
+shipitron_home_volume = FindDockerVolumeName.call!(
+  container_name: 'shipitron',
+  volume_search: /shipitron-home/
+).volume_name
+
+docker run ... -v #{shipitron_home_volume}:/home/shipitron ...
+```
+
+This allows the containers to share data with each other. ECS will automatically clean up the task scoped container.
+
+If a containerized tool requires access to AWS resources, be sure to pass the `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` to it so it can inherit any task roles.
+
 ## Development
 
-- `docker volume create --name shipitron_fog`
-- `./build_dev.sh`
-- `docker run -it --rm -v $(pwd):/shipitron -v shipitron_fog:/fog -e FOG_LOCAL=true -w /shipitron outstand/shipitron:dev rspec spec` to run specs
-- `APP_PATH=/path/to/app`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app>` to run client side
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --simulate` to get the arguments for `server_deploy` below
-- `docker run -it --rm --dns 10.10.10.2 -v $(pwd):/shipitron -v $APP_PATH:/app -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock -v shipitron_fog:/fog -e FOG_LOCAL=true -e CONSUL_HOST=consul outstand/shipitron:dev server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
+- `desk go`
+- `dev build --pull shipitron`
+- `rspec specs` to run specs
+- Set the application path in the volumes section in `docker-compose.yml`.
+- `shipitron deploy <app>` to run client side
+- `shipitron deploy <app> --simulate-store-deploy` to save the deploy options to S3 and generate deploy id
+- `dev run --rm -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock shipitron server_deploy --deploy-id <deploy_id>` to run server side
 
 Running a dev version in production:
-- `docker push outstand/shipitron:dev`
-- Update config to use `shipitron_task: shipitron-dev`
-- `docker run -it --rm -v $(pwd):/shipitron -v $HOME/.config/shipitron:/home/shipitron/.config/shipitron -v $APP_PATH:/app outstand/shipitron:dev deploy <app> --debug`
+- Ensure `SHIPITRON_STARTED_BY` env var is set for your `shipitron-staging` task def
+- `docker build --pull -t outstand/shipitron:staging -f Dockerfile.staging .`
+- `docker push outstand/shipitron:staging`
+- Update config to use `shipitron_task: shipitron-staging` and `outstand/shipitron:staging`
+- `shipitron deploy <app> --debug`
 
 To release a new version:
 - Update the version number in `version.rb` and `Dockerfile.release` and commit the result.
-- `./build_dev.sh`
-- `docker run -it --rm -v ~/.gitconfig:/root/.gitconfig -v ~/.gitconfig.user:/root/.gitconfig.user -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v ~/.gem:/root/.gem -w /shipitron outstand/shipitron:dev rake release`
-- `docker build -t outstand/shipitron:VERSION -f Dockerfile.release .`
-- `docker push outstand/shipitron:VERSION`
-- `docker tag outstand/shipitron:VERSION outstand/shipitron:latest`
+- `dev build --pull shipitron`
+- `release_gem`
+- `VERSION=<version>`
+- `docker build -t outstand/shipitron:$VERSION -f Dockerfile.release .`
+- `docker push outstand/shipitron:$VERSION`
+- `docker tag outstand/shipitron:$VERSION outstand/shipitron:latest`
 - `docker push outstand/shipitron:latest`
 - Update ECS task definition with new version