shipitron 1.2.1 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +23 -2
- data/Dockerfile.release +2 -2
- data/Gemfile +1 -0
- data/README.md +42 -7
- data/docker-compose.yml +24 -0
- data/lib/shipitron/cli.rb +8 -4
- data/lib/shipitron/client.rb +9 -0
- data/lib/shipitron/client/bootstrap_application.rb +1 -0
- data/lib/shipitron/client/create_ecs_services.rb +1 -0
- data/lib/shipitron/client/deploy_application.rb +1 -0
- data/lib/shipitron/client/ensure_deploy_not_running.rb +2 -1
- data/lib/shipitron/client/fetch_clusters.rb +1 -0
- data/lib/shipitron/client/force_deploy.rb +1 -0
- data/lib/shipitron/client/load_application_config.rb +3 -0
- data/lib/shipitron/client/load_templates.rb +1 -0
- data/lib/shipitron/client/register_ecs_task_definitions.rb +1 -0
- data/lib/shipitron/client/run_ecs_tasks.rb +12 -1
- data/lib/shipitron/docker_image.rb +4 -1
- data/lib/shipitron/find_docker_volume_name.rb +68 -0
- data/lib/shipitron/s3_copy.rb +46 -0
- data/lib/shipitron/server/deploy_application.rb +2 -0
- data/lib/shipitron/server/docker/build_image.rb +3 -0
- data/lib/shipitron/server/docker/configure.rb +46 -10
- data/lib/shipitron/server/docker/push_image.rb +3 -0
- data/lib/shipitron/server/docker/run_build_script.rb +13 -6
- data/lib/shipitron/server/download_build_cache.rb +11 -3
- data/lib/shipitron/server/transform_cli_args.rb +4 -0
- data/lib/shipitron/server/update_ecs_task_definitions.rb +2 -1
- data/lib/shipitron/server/upload_build_cache.rb +10 -9
- data/lib/shipitron/version.rb +1 -1
- data/scripts/docker-entrypoint.sh +9 -0
- data/shipitron.gemspec +7 -6
- metadata +33 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9c5423fc9dbd202488e45ab382ad2bc37de889d909179b6e65a33d362ad9d20c
|
4
|
+
data.tar.gz: ea5bea8068a0d5d2ac0787c5590d53c3bf5a7f88ba6d646474d760b21dd4b707
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9983df8f5f01ea66a28e4bf7499bfed81bf49bc0127895669d003176861ad6db04b48a8c351215cd17ab927b1eb4b9087f203075206e2967be01429c8cd15d21
|
7
|
+
data.tar.gz: b5f1be2174a00e52ec0fb5df6ee04a64ac83ed73f09e6651fcf5655cb9a0c85f1746cbf30bf6bd8388f97668aac129bd1ef32980e79c44ffd645f656fe9213e5
|
data/Dockerfile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
FROM ruby:2.
|
1
|
+
FROM ruby:2.7.1-alpine as cache
|
2
2
|
COPY cache/ /tmp/
|
3
3
|
RUN cd /usr/local/bundle && \
|
4
4
|
([ -f /tmp/bundler-data.tar.gz ] && \
|
5
5
|
tar -zxf /tmp/bundler-data.tar.gz && \
|
6
6
|
rm /tmp/bundler-data.tar.gz) || true
|
7
7
|
|
8
|
-
FROM ruby:2.
|
8
|
+
FROM ruby:2.7.1-alpine
|
9
9
|
LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
|
10
10
|
|
11
11
|
RUN addgroup -S shipitron && \
|
@@ -27,6 +27,27 @@ RUN apk add --no-cache \
|
|
27
27
|
wget \
|
28
28
|
jq
|
29
29
|
|
30
|
+
ENV ECR_CREDENTIAL_HELPER_VERSION 0.4.0
|
31
|
+
RUN cd /usr/local/bin && \
|
32
|
+
wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${ECR_CREDENTIAL_HELPER_VERSION}/linux-amd64/docker-credential-ecr-login && \
|
33
|
+
chmod +x docker-credential-ecr-login
|
34
|
+
|
35
|
+
ENV BUILDKIT_VERSION v0.7.2
|
36
|
+
RUN cd /usr/local/bin && \
|
37
|
+
wget -nv https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz && \
|
38
|
+
tar --strip-components=1 -zxvf buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz bin/buildctl && \
|
39
|
+
chmod +x buildctl && \
|
40
|
+
rm -f buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz
|
41
|
+
|
42
|
+
USER shipitron
|
43
|
+
ENV BUILDX_VERSION v0.4.2
|
44
|
+
RUN cd /home/shipitron && \
|
45
|
+
wget -nv https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 && \
|
46
|
+
mkdir -p ~/.docker/cli-plugins && \
|
47
|
+
mv buildx-${BUILDX_VERSION}.linux-amd64 ~/.docker/cli-plugins/docker-buildx && \
|
48
|
+
chmod a+x ~/.docker/cli-plugins/docker-buildx
|
49
|
+
|
50
|
+
USER root
|
30
51
|
ENV USE_BUNDLE_EXEC true
|
31
52
|
ENV BUNDLE_GEMFILE /shipitron/Gemfile
|
32
53
|
|
data/Dockerfile.release
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
FROM ruby:2.
|
1
|
+
FROM ruby:2.7.1-alpine
|
2
2
|
LABEL maintainer="Ryan Schlesinger <ryan@outstand.com>"
|
3
3
|
|
4
4
|
RUN addgroup -S shipitron && \
|
@@ -22,7 +22,7 @@ RUN apk add --no-cache \
|
|
22
22
|
|
23
23
|
WORKDIR /app
|
24
24
|
|
25
|
-
ENV SHIPITRON_VERSION=1.
|
25
|
+
ENV SHIPITRON_VERSION=1.3.0
|
26
26
|
|
27
27
|
RUN gem install shipitron -v ${SHIPITRON_VERSION} && \
|
28
28
|
mkdir -p /home/shipitron/.ssh && \
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -31,20 +31,55 @@ applications:
|
|
31
31
|
- Add deploy ref key to `shipitron/<app name>/deploy_ref_key`
|
32
32
|
- `docker run -it --rm -v shipitron.yml:/shipitron/config/shipitron.yml outstand/shipitron:<version> deploy <app>`
|
33
33
|
|
34
|
+
### New shipitron config file keys:
|
35
|
+
|
36
|
+
- registry (specifies an alternate docker registry):
|
37
|
+
```yaml
|
38
|
+
---
|
39
|
+
applications:
|
40
|
+
foobar:
|
41
|
+
registry: 12345.dkr.ecr.us-east-1.amazonaws.com
|
42
|
+
```
|
43
|
+
|
44
|
+
- skip_push (shipitron will skip pushing the docker images; use if the build script takes care of this):
|
45
|
+
```yaml
|
46
|
+
---
|
47
|
+
applications:
|
48
|
+
foobar:
|
49
|
+
skip_push: true
|
50
|
+
```
|
51
|
+
Additionally, the build script will receive the named tag in use as `$2`.
|
52
|
+
|
53
|
+
### Containerized tool support
|
54
|
+
Shipitron now supports starting other containers as part of the build process. We're using this to use the aws cli to transfer files to/from s3. Shipitron's task definition needs to have a task scoped docker volume added with the name `shipitron-home`. This volume will be mounted at `/home/shipitron` and should be shared with any new containers:
|
55
|
+
|
56
|
+
```
|
57
|
+
shipitron_home_volume = FindDockerVolumeName.call!(
|
58
|
+
container_name: 'shipitron',
|
59
|
+
volume_search: /shipitron-home/
|
60
|
+
).volume_name
|
61
|
+
|
62
|
+
docker run ... -v #{shipitron_home_volume}:/home/shipitron ...
|
63
|
+
```
|
64
|
+
|
65
|
+
This allows the containers to share data with each other. ECS will automatically clean up the task scoped container.
|
66
|
+
|
67
|
+
If a containerized tool requires access to AWS resources, be sure to pass the `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` to it so it can inherit any task roles.
|
68
|
+
|
34
69
|
## Development
|
35
70
|
|
36
|
-
- `docker volume create --name shipitron_fog`
|
37
71
|
- `./build_dev.sh`
|
38
|
-
- `
|
39
|
-
-
|
40
|
-
- `
|
41
|
-
- `
|
42
|
-
- `
|
72
|
+
- `dev run --rm shipitron_specs` to run specs
|
73
|
+
- Set the application path in the volumes section in `docker-compose.yml`.
|
74
|
+
- `dev run --rm shipitron deploy <app>` to run client side
|
75
|
+
- `dev run --rm shipitron deploy <app> --simulate` to get the arguments for `server_deploy` below
|
76
|
+
- `dev run --rm -v /bin/docker:/bin/docker -v /var/run/docker.sock:/var/run/docker.sock shipitron server_deploy --name dummy-app --repository git@github.com:outstand/dummy-app --bucket outstand-shipitron --image-name outstand/dummy-app --region us-east-1 --cluster-name us-east-1-prod-blue --ecs-task-defs dummy-app --ecs-services dummy-app --build-script shipitron/build.sh --post-builds 'ecs_task:dummy-app,container_name:dummy-app,command:echo postbuild' --ecs-task-def-templates LS0tCmZhbWlseTogZHVtbXktYXBwCmNvbnRhaW5lcl9kZWZpbml0aW9uczoKICAtIG5hbWU6IGR1bW15LWFwcAogICAgaW1hZ2U6IG91dHN0YW5kL2R1bW15LWFwcDp7e3RhZ319CiAgICBtZW1vcnk6IDEyOAogICAgZXNzZW50aWFsOiB0cnVlCiAgICBwb3J0X21hcHBpbmdzOgogICAgICAtIGNvbnRhaW5lcl9wb3J0OiA4MAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gbmFtZTogU0VSVklDRV84MF9OQU1FCiAgICAgICAgdmFsdWU6IGR1bW15Cg== --ecs-service-templates LS0tCmNsdXN0ZXI6IHt7Y2x1c3Rlcn19CnNlcnZpY2VfbmFtZTogZHVtbXktYXBwCnRhc2tfZGVmaW5pdGlvbjogZHVtbXktYXBwe3tyZXZpc2lvbn19CmRlc2lyZWRfY291bnQ6IHt7Y291bnR9fQojcm9sZToge3tyb2xlfX0KZGVwbG95bWVudF9jb25maWd1cmF0aW9uOgogIG1heGltdW1fcGVyY2VudDogMjAwCiAgbWluaW11bV9oZWFsdGh5X3BlcmNlbnQ6IDUwCg== --debug` to run server side (dummy-app is an example)
|
43
77
|
|
44
78
|
Running a dev version in production:
|
79
|
+
- Update `Shipitron::Client::STARTED_BY`
|
45
80
|
- `docker push outstand/shipitron:dev`
|
46
81
|
- Update config to use `shipitron_task: shipitron-dev`
|
47
|
-
- `
|
82
|
+
- `dev run --rm shipitron deploy <app> --debug`
|
48
83
|
|
49
84
|
To release a new version:
|
50
85
|
- Update the version number in `version.rb` and `Dockerfile.release` and commit the result.
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
version: '3.8'
|
2
|
+
services:
|
3
|
+
shipitron:
|
4
|
+
image: outstand/shipitron:dev
|
5
|
+
volumes:
|
6
|
+
- ~/dev/pages:/app # Set this to the application to be shipitron'd
|
7
|
+
- shipitron-home:/home/shipitron
|
8
|
+
- ~/.config/shipitron:/home/shipitron/.config/shipitron
|
9
|
+
- .:/shipitron
|
10
|
+
shipitron_specs:
|
11
|
+
image: outstand/shipitron:dev
|
12
|
+
command: rspec spec
|
13
|
+
environment:
|
14
|
+
FOG_LOCAL: 'true'
|
15
|
+
working_dir: /shipitron
|
16
|
+
volumes:
|
17
|
+
- fog:/fog
|
18
|
+
- shipitron-home:/home/shipitron
|
19
|
+
- ~/.config/shipitron:/home/shipitron/.config/shipitron
|
20
|
+
- .:/shipitron
|
21
|
+
|
22
|
+
volumes:
|
23
|
+
fog:
|
24
|
+
shipitron-home:
|
data/lib/shipitron/cli.rb
CHANGED
@@ -27,7 +27,7 @@ module Shipitron
|
|
27
27
|
)
|
28
28
|
|
29
29
|
if result.failure?
|
30
|
-
result.
|
30
|
+
result.error_messages.each do |error|
|
31
31
|
Logger.fatal error
|
32
32
|
end
|
33
33
|
Logger.fatal 'Deploy failed.'
|
@@ -50,7 +50,7 @@ module Shipitron
|
|
50
50
|
)
|
51
51
|
|
52
52
|
if result.failure?
|
53
|
-
result.
|
53
|
+
result.error_messages.each do |error|
|
54
54
|
Logger.fatal error
|
55
55
|
end
|
56
56
|
Logger.fatal 'Deploy failed.'
|
@@ -62,10 +62,12 @@ module Shipitron
|
|
62
62
|
option :name, required: true
|
63
63
|
option :repository, required: true
|
64
64
|
option :repository_branch, default: 'master'
|
65
|
+
option :registry, default: nil
|
65
66
|
option :bucket, required: true
|
66
67
|
option :build_cache_location, default: 'tmp/build-cache.tar.gz'
|
67
68
|
option :image_name, required: true
|
68
69
|
option :named_tag, default: 'latest'
|
70
|
+
option :skip_push, type: :boolean, default: false
|
69
71
|
option :region, required: true
|
70
72
|
option :clusters, type: :array, required: true
|
71
73
|
option :ecs_task_defs, type: :array, required: true
|
@@ -86,10 +88,12 @@ module Shipitron
|
|
86
88
|
application: options[:name],
|
87
89
|
repository_url: options[:repository],
|
88
90
|
repository_branch: options[:repository_branch],
|
91
|
+
registry: options[:registry],
|
89
92
|
s3_cache_bucket: options[:bucket],
|
90
93
|
build_cache_location: options[:build_cache_location],
|
91
94
|
image_name: options[:image_name],
|
92
95
|
named_tag: options[:named_tag],
|
96
|
+
skip_push: options[:skip_push],
|
93
97
|
region: options[:region],
|
94
98
|
clusters: options[:clusters],
|
95
99
|
ecs_task_defs: options[:ecs_task_defs],
|
@@ -106,7 +110,7 @@ module Shipitron
|
|
106
110
|
)
|
107
111
|
|
108
112
|
if result.failure?
|
109
|
-
result.
|
113
|
+
result.error_messages.each do |error|
|
110
114
|
Logger.fatal error
|
111
115
|
end
|
112
116
|
Logger.fatal 'Deploy failed.'
|
@@ -137,7 +141,7 @@ module Shipitron
|
|
137
141
|
)
|
138
142
|
|
139
143
|
if result.failure?
|
140
|
-
result.
|
144
|
+
result.error_messages.each do |error|
|
141
145
|
Logger.fatal error
|
142
146
|
end
|
143
147
|
Logger.fatal 'Bootstrap failed.'
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'shipitron'
|
2
|
+
require 'shipitron/client'
|
2
3
|
require 'shipitron/ecs_client'
|
3
4
|
|
4
5
|
# Note: This is a best effort client side check to make sure there
|
@@ -21,7 +22,7 @@ module Shipitron
|
|
21
22
|
begin
|
22
23
|
response = ecs_client(region: cluster.region).list_tasks(
|
23
24
|
cluster: cluster.name,
|
24
|
-
started_by:
|
25
|
+
started_by: Shipitron::Client::STARTED_BY,
|
25
26
|
max_results: 1,
|
26
27
|
desired_status: status
|
27
28
|
)
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'shipitron'
|
2
|
+
require 'shipitron/client'
|
2
3
|
require 'shipitron/post_build'
|
3
4
|
|
4
5
|
module Shipitron
|
@@ -11,6 +12,7 @@ module Shipitron
|
|
11
12
|
def call
|
12
13
|
context.repository_url = config.repository
|
13
14
|
context.repository_branch = config.repository_branch
|
15
|
+
context.registry = config.registry
|
14
16
|
context.s3_cache_bucket = config.cache_bucket
|
15
17
|
context.build_cache_location = config.build_cache_location
|
16
18
|
context.image_name = config.image_name
|
@@ -21,6 +23,7 @@ module Shipitron
|
|
21
23
|
config.named_tag
|
22
24
|
end
|
23
25
|
end
|
26
|
+
context.skip_push = config.skip_push
|
24
27
|
context.build_script = config.build_script
|
25
28
|
context.post_builds = begin
|
26
29
|
if config.post_builds.nil?
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'shipitron'
|
2
|
+
require 'shipitron/client'
|
2
3
|
require 'shipitron/ecs_client'
|
3
4
|
require 'shellwords'
|
4
5
|
require 'base64'
|
@@ -24,9 +25,11 @@ module Shipitron
|
|
24
25
|
optional :ecs_services
|
25
26
|
optional :ecs_service_templates
|
26
27
|
optional :build_script
|
28
|
+
optional :skip_push
|
27
29
|
optional :post_builds
|
28
30
|
optional :simulate
|
29
31
|
optional :repository_branch
|
32
|
+
optional :registry
|
30
33
|
|
31
34
|
before do
|
32
35
|
context.post_builds ||= []
|
@@ -73,7 +76,7 @@ module Shipitron
|
|
73
76
|
]
|
74
77
|
},
|
75
78
|
count: 1,
|
76
|
-
started_by:
|
79
|
+
started_by: Shipitron::Client::STARTED_BY
|
77
80
|
)
|
78
81
|
|
79
82
|
if !response.failures.empty?
|
@@ -125,10 +128,18 @@ module Shipitron
|
|
125
128
|
ary.concat(context.ecs_services)
|
126
129
|
end
|
127
130
|
|
131
|
+
if context.registry != nil
|
132
|
+
ary.concat ['--registry', context.registry]
|
133
|
+
end
|
134
|
+
|
128
135
|
if context.build_script != nil
|
129
136
|
ary.concat ['--build-script', context.build_script]
|
130
137
|
end
|
131
138
|
|
139
|
+
if context.skip_push != nil
|
140
|
+
ary.concat ['--skip-push', context.skip_push.to_s]
|
141
|
+
end
|
142
|
+
|
132
143
|
if !context.post_builds.empty?
|
133
144
|
ary << '--post-builds'
|
134
145
|
ary.concat(context.post_builds.map(&:to_s))
|
@@ -2,6 +2,7 @@ require 'shipitron'
|
|
2
2
|
|
3
3
|
module Shipitron
|
4
4
|
class DockerImage < Hashie::Dash
|
5
|
+
property :registry
|
5
6
|
property :name
|
6
7
|
property :tag
|
7
8
|
|
@@ -13,7 +14,9 @@ module Shipitron
|
|
13
14
|
tag_str = tag_str.dup.prepend(':')
|
14
15
|
end
|
15
16
|
|
16
|
-
|
17
|
+
name_with_registry = [registry, name].compact.join('/')
|
18
|
+
|
19
|
+
"#{name_with_registry}#{tag_str}"
|
17
20
|
end
|
18
21
|
|
19
22
|
def to_s
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'excon'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class FindDockerVolumeName
|
6
|
+
include Metaractor
|
7
|
+
|
8
|
+
required :container_name
|
9
|
+
required :volume_search
|
10
|
+
|
11
|
+
def call
|
12
|
+
volumes = container_volumes(container_name: container_name)
|
13
|
+
|
14
|
+
volume_metadata = volumes.find do |volume|
|
15
|
+
volume['DockerName'] =~ volume_search
|
16
|
+
end
|
17
|
+
|
18
|
+
if volume_metadata.nil?
|
19
|
+
raise 'Unable to find shipitron-home volume!'
|
20
|
+
end
|
21
|
+
|
22
|
+
context.volume_name = volume_metadata['DockerName']
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
def container_name
|
27
|
+
context.container_name
|
28
|
+
end
|
29
|
+
|
30
|
+
def volume_search
|
31
|
+
context.volume_search
|
32
|
+
end
|
33
|
+
|
34
|
+
def container_volumes(container_name:)
|
35
|
+
container_metadata = self.task_metadata['Containers'].find do |container|
|
36
|
+
container['Name'] == container_name
|
37
|
+
end
|
38
|
+
|
39
|
+
return {} if container_metadata.nil?
|
40
|
+
|
41
|
+
container_metadata['Volumes']
|
42
|
+
end
|
43
|
+
|
44
|
+
def task_metadata
|
45
|
+
return @task_metadata if defined?(@task_metadata)
|
46
|
+
|
47
|
+
begin
|
48
|
+
response = Excon.get(
|
49
|
+
"#{ENV['ECS_CONTAINER_METADATA_URI_V4']}/task",
|
50
|
+
expects: [200],
|
51
|
+
connect_timeout: 5,
|
52
|
+
read_timeout: 5,
|
53
|
+
write_timeout: 5,
|
54
|
+
tcp_nodelay: true
|
55
|
+
)
|
56
|
+
|
57
|
+
Logger.debug "Metadata result:"
|
58
|
+
Logger.debug(response.body)
|
59
|
+
Logger.debug "\n"
|
60
|
+
|
61
|
+
@task_metadata = JSON.parse(response.body)
|
62
|
+
rescue
|
63
|
+
Logger.info "Metadata uri failed"
|
64
|
+
{}
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'shipitron'
|
2
|
+
require 'shipitron/find_docker_volume_name'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class S3Copy
|
6
|
+
include Metaractor
|
7
|
+
|
8
|
+
required :source
|
9
|
+
required :destination
|
10
|
+
required :region
|
11
|
+
|
12
|
+
def call
|
13
|
+
if ENV['FOG_LOCAL']
|
14
|
+
Logger.info `cp #{source.gsub('s3://', '/fog/')} #{destination.gsub('s3://', '/fog/')}`
|
15
|
+
if $? != 0
|
16
|
+
fail_with_error!(message: 'Failed to transfer to/from s3 (mocked).')
|
17
|
+
end
|
18
|
+
else
|
19
|
+
Logger.info "S3 Copy from #{source} to #{destination}"
|
20
|
+
|
21
|
+
shipitron_home_volume = FindDockerVolumeName.call!(
|
22
|
+
container_name: 'shipitron',
|
23
|
+
volume_search: /shipitron-home/
|
24
|
+
).volume_name
|
25
|
+
|
26
|
+
Logger.info `docker run --rm -t -v #{shipitron_home_volume}:/home/shipitron -e AWS_CONTAINER_CREDENTIALS_RELATIVE_URI amazon/aws-cli:latest --region #{region} s3 cp #{source} #{destination} --quiet --only-show-errors`
|
27
|
+
if $? != 0
|
28
|
+
fail_with_error!(message: 'Failed to transfer to/from s3.')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
def source
|
35
|
+
context.source
|
36
|
+
end
|
37
|
+
|
38
|
+
def destination
|
39
|
+
context.destination
|
40
|
+
end
|
41
|
+
|
42
|
+
def region
|
43
|
+
context.region
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'shipitron'
|
2
2
|
require 'shipitron/consul_keys'
|
3
|
+
require 'json'
|
3
4
|
|
4
5
|
module Shipitron
|
5
6
|
module Server
|
@@ -9,22 +10,48 @@ module Shipitron
|
|
9
10
|
include ConsulKeys
|
10
11
|
|
11
12
|
required :application
|
13
|
+
optional :registry
|
12
14
|
|
13
15
|
before do
|
14
16
|
configure_consul_client!
|
15
17
|
end
|
16
18
|
|
17
19
|
def call
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
20
|
+
username = fetch_scoped_key('docker_user')
|
21
|
+
password = fetch_scoped_key('docker_password')
|
22
|
+
|
23
|
+
if username && password
|
24
|
+
Logger.info `docker login --username #{username} --password #{password}`
|
25
|
+
if $? != 0
|
26
|
+
fail_with_error!(message: 'Docker login failed.')
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
if registry
|
31
|
+
case registry
|
32
|
+
when /docker\.io/
|
33
|
+
# do nothing
|
34
|
+
when /\d+\.dkr\.ecr\.us-east-1\.amazonaws\.com/
|
35
|
+
# ECR
|
36
|
+
config_file = Pathname.new('/home/shipitron/.docker/config.json')
|
37
|
+
config_file.parent.mkpath
|
38
|
+
|
39
|
+
config_hash = {}
|
40
|
+
if config_file.file?
|
41
|
+
config_file.open('rb') do |file|
|
42
|
+
json = file.read
|
43
|
+
config_hash = JSON.parse(json) rescue {}
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
config_hash['credHelpers'] ||= {}
|
48
|
+
config_hash['credHelpers'][registry] = 'ecr-login'
|
49
|
+
|
50
|
+
config_file.open('wb') do |file|
|
51
|
+
file.puts(JSON.generate(config_hash))
|
52
|
+
file.chmod(0600)
|
53
|
+
end
|
54
|
+
end
|
28
55
|
end
|
29
56
|
end
|
30
57
|
|
@@ -33,6 +60,15 @@ module Shipitron
|
|
33
60
|
context.application
|
34
61
|
end
|
35
62
|
|
63
|
+
def registry
|
64
|
+
context.registry
|
65
|
+
end
|
66
|
+
|
67
|
+
def fetch_scoped_key(key)
|
68
|
+
value = fetch_key(key: "shipitron/#{application}/#{key}")
|
69
|
+
value = fetch_key(key: "shipitron/#{key}") if value.nil?
|
70
|
+
value
|
71
|
+
end
|
36
72
|
end
|
37
73
|
end
|
38
74
|
end
|
@@ -8,8 +8,11 @@ module Shipitron
|
|
8
8
|
|
9
9
|
required :docker_image
|
10
10
|
required :named_tag
|
11
|
+
optional :skip_push, default: false
|
11
12
|
|
12
13
|
def call
|
14
|
+
return if context.skip_push
|
15
|
+
|
13
16
|
Logger.info "Pushing docker image #{docker_image} and #{docker_image.name_with_tag(named_tag)}"
|
14
17
|
|
15
18
|
Logger.info `docker tag #{docker_image} #{docker_image.name_with_tag(named_tag)}`
|
@@ -10,15 +10,14 @@ module Shipitron
|
|
10
10
|
required :application
|
11
11
|
required :docker_image
|
12
12
|
required :git_sha
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
context.build_script ||= 'shipitron/build.sh'
|
17
|
-
end
|
13
|
+
required :named_tag
|
14
|
+
optional :build_script, default: 'shipitron/build.sh'
|
15
|
+
optional :registry
|
18
16
|
|
19
17
|
def call
|
20
18
|
Logger.info 'Building docker image'
|
21
19
|
|
20
|
+
docker_image.registry = registry if registry != nil
|
22
21
|
docker_image.tag = git_sha
|
23
22
|
|
24
23
|
FileUtils.cd("/home/shipitron/#{application}") do
|
@@ -27,7 +26,7 @@ module Shipitron
|
|
27
26
|
end
|
28
27
|
|
29
28
|
cmd = TTY::Command.new
|
30
|
-
result = cmd.run!("#{build_script} #{docker_image}")
|
29
|
+
result = cmd.run!("#{build_script} #{docker_image} #{named_tag}")
|
31
30
|
|
32
31
|
if result.failure?
|
33
32
|
fail_with_error!(message: "build script exited with non-zero code: #{result.exit_status}")
|
@@ -48,9 +47,17 @@ module Shipitron
|
|
48
47
|
context.git_sha
|
49
48
|
end
|
50
49
|
|
50
|
+
def named_tag
|
51
|
+
context.named_tag
|
52
|
+
end
|
53
|
+
|
51
54
|
def build_script
|
52
55
|
context.build_script
|
53
56
|
end
|
57
|
+
|
58
|
+
def registry
|
59
|
+
context.registry
|
60
|
+
end
|
54
61
|
end
|
55
62
|
end
|
56
63
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'shipitron'
|
2
2
|
require 'shipitron/fetch_bucket'
|
3
|
+
require 'shipitron/s3_copy'
|
3
4
|
|
4
5
|
module Shipitron
|
5
6
|
module Server
|
@@ -9,11 +10,12 @@ module Shipitron
|
|
9
10
|
required :application
|
10
11
|
required :s3_cache_bucket
|
11
12
|
required :build_cache_location
|
13
|
+
required :region
|
12
14
|
|
13
15
|
def call
|
14
16
|
Logger.info "Downloading build cache from bucket #{s3_cache_bucket}"
|
15
17
|
|
16
|
-
s3_file = bucket.files.
|
18
|
+
s3_file = bucket.files.head("#{application}.build-cache.archive")
|
17
19
|
if s3_file.nil?
|
18
20
|
Logger.warn 'Build cache not found.'
|
19
21
|
return
|
@@ -21,8 +23,14 @@ module Shipitron
|
|
21
23
|
|
22
24
|
build_cache = Pathname.new("/home/shipitron/#{application}/#{build_cache_location}")
|
23
25
|
build_cache.parent.mkpath
|
24
|
-
|
25
|
-
|
26
|
+
|
27
|
+
result = S3Copy.call(
|
28
|
+
source: "s3://#{s3_cache_bucket}/#{application}.build-cache.archive",
|
29
|
+
destination: build_cache.to_s,
|
30
|
+
region: context.region
|
31
|
+
)
|
32
|
+
if result.failure?
|
33
|
+
fail_with_error!(message: 'Failed to download build cache!')
|
26
34
|
end
|
27
35
|
|
28
36
|
Logger.info 'Download complete.'
|
@@ -12,6 +12,7 @@ module Shipitron
|
|
12
12
|
required :application
|
13
13
|
required :repository_url
|
14
14
|
optional :repository_branch
|
15
|
+
optional :registry
|
15
16
|
required :s3_cache_bucket
|
16
17
|
required :build_cache_location
|
17
18
|
required :image_name
|
@@ -23,6 +24,7 @@ module Shipitron
|
|
23
24
|
optional :ecs_services
|
24
25
|
optional :ecs_service_templates
|
25
26
|
optional :build_script
|
27
|
+
optional :skip_push, default: false
|
26
28
|
optional :post_builds
|
27
29
|
|
28
30
|
before do
|
@@ -37,6 +39,7 @@ module Shipitron
|
|
37
39
|
application
|
38
40
|
repository_url
|
39
41
|
repository_branch
|
42
|
+
registry
|
40
43
|
s3_cache_bucket
|
41
44
|
build_cache_location
|
42
45
|
named_tag
|
@@ -44,6 +47,7 @@ module Shipitron
|
|
44
47
|
clusters
|
45
48
|
ecs_services
|
46
49
|
build_script
|
50
|
+
skip_push
|
47
51
|
].each_with_object(cli_args) { |k, args| args[k] = context[k] }
|
48
52
|
|
49
53
|
cli_args.docker_image = DockerImage.new(name: context.image_name)
|
@@ -14,11 +14,12 @@ module Shipitron
|
|
14
14
|
required :docker_image
|
15
15
|
required :ecs_task_defs
|
16
16
|
optional :ecs_task_def_templates
|
17
|
+
optional :registry
|
17
18
|
|
18
19
|
before do
|
19
20
|
context.ecs_task_def_templates ||= []
|
20
21
|
context.templates = context.ecs_task_def_templates
|
21
|
-
context.template_context = { tag: docker_image.tag }
|
22
|
+
context.template_context = { tag: docker_image.tag, registry: context.registry }
|
22
23
|
end
|
23
24
|
|
24
25
|
organize [
|
@@ -9,6 +9,7 @@ module Shipitron
|
|
9
9
|
required :application
|
10
10
|
required :s3_cache_bucket
|
11
11
|
required :build_cache_location
|
12
|
+
required :region
|
12
13
|
|
13
14
|
def call
|
14
15
|
Logger.info "Uploading build cache to bucket #{s3_cache_bucket}"
|
@@ -19,11 +20,15 @@ module Shipitron
|
|
19
20
|
return
|
20
21
|
end
|
21
22
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
result = S3Copy.call(
|
24
|
+
source: build_cache.to_s,
|
25
|
+
destination: "s3://#{s3_cache_bucket}/#{application}.build-cache.archive",
|
26
|
+
region: context.region
|
27
|
+
)
|
28
|
+
if result.failure?
|
29
|
+
Logger.warn 'Failed to upload build cache!'
|
30
|
+
else
|
31
|
+
Logger.info 'Upload complete.'
|
27
32
|
end
|
28
33
|
end
|
29
34
|
|
@@ -39,10 +44,6 @@ module Shipitron
|
|
39
44
|
def build_cache_location
|
40
45
|
context.build_cache_location
|
41
46
|
end
|
42
|
-
|
43
|
-
def bucket
|
44
|
-
@bucket ||= FetchBucket.call!(name: s3_cache_bucket).bucket
|
45
|
-
end
|
46
47
|
end
|
47
48
|
end
|
48
49
|
end
|
data/lib/shipitron/version.rb
CHANGED
data/shipitron.gemspec
CHANGED
@@ -17,21 +17,22 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
|
-
spec.add_runtime_dependency 'thor', '~> 0
|
20
|
+
spec.add_runtime_dependency 'thor', '~> 1.0'
|
21
21
|
spec.add_runtime_dependency 'aws-sdk-ecs', '~> 1.8'
|
22
|
-
spec.add_runtime_dependency 'hashie', '~>
|
23
|
-
spec.add_runtime_dependency 'metaractor', '~> 0
|
22
|
+
spec.add_runtime_dependency 'hashie', '~> 4.1'
|
23
|
+
spec.add_runtime_dependency 'metaractor', '~> 3.0'
|
24
24
|
spec.add_runtime_dependency 'diplomat', '~> 2.0'
|
25
|
-
spec.add_runtime_dependency 'fog-aws', '~>
|
25
|
+
spec.add_runtime_dependency 'fog-aws', '~> 3.6'
|
26
26
|
spec.add_runtime_dependency 'mime-types', '~> 3.1'
|
27
27
|
spec.add_runtime_dependency 'minitar', '~> 0.6'
|
28
28
|
spec.add_runtime_dependency 'mustache', '~> 1.0'
|
29
29
|
spec.add_runtime_dependency 'tty-command', '~> 0.7'
|
30
30
|
spec.add_runtime_dependency 'tty-table', '~> 0.9'
|
31
31
|
spec.add_runtime_dependency 'pastel', '~> 0.7'
|
32
|
+
spec.add_runtime_dependency 'excon', '~> 0.76'
|
32
33
|
|
33
|
-
spec.add_development_dependency "bundler", "~> 1
|
34
|
-
spec.add_development_dependency "rake", "~>
|
34
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
35
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
35
36
|
spec.add_development_dependency "pry-byebug", "~> 3.5"
|
36
37
|
spec.add_development_dependency "rspec", "~> 3.7"
|
37
38
|
spec.add_development_dependency "fivemat", "~> 1.3"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shipitron
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Schlesinger
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0
|
19
|
+
version: '1.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0
|
26
|
+
version: '1.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: aws-sdk-ecs
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,28 +44,28 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '4.1'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '4.1'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: metaractor
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '0
|
61
|
+
version: '3.0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '0
|
68
|
+
version: '3.0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: diplomat
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,14 +86,14 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '3.6'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '3.6'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: mime-types
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,34 +178,48 @@ dependencies:
|
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '0.7'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: excon
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0.76'
|
188
|
+
type: :runtime
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0.76'
|
181
195
|
- !ruby/object:Gem::Dependency
|
182
196
|
name: bundler
|
183
197
|
requirement: !ruby/object:Gem::Requirement
|
184
198
|
requirements:
|
185
199
|
- - "~>"
|
186
200
|
- !ruby/object:Gem::Version
|
187
|
-
version: '1
|
201
|
+
version: '2.1'
|
188
202
|
type: :development
|
189
203
|
prerelease: false
|
190
204
|
version_requirements: !ruby/object:Gem::Requirement
|
191
205
|
requirements:
|
192
206
|
- - "~>"
|
193
207
|
- !ruby/object:Gem::Version
|
194
|
-
version: '1
|
208
|
+
version: '2.1'
|
195
209
|
- !ruby/object:Gem::Dependency
|
196
210
|
name: rake
|
197
211
|
requirement: !ruby/object:Gem::Requirement
|
198
212
|
requirements:
|
199
213
|
- - "~>"
|
200
214
|
- !ruby/object:Gem::Version
|
201
|
-
version: '
|
215
|
+
version: '13.0'
|
202
216
|
type: :development
|
203
217
|
prerelease: false
|
204
218
|
version_requirements: !ruby/object:Gem::Requirement
|
205
219
|
requirements:
|
206
220
|
- - "~>"
|
207
221
|
- !ruby/object:Gem::Version
|
208
|
-
version: '
|
222
|
+
version: '13.0'
|
209
223
|
- !ruby/object:Gem::Dependency
|
210
224
|
name: pry-byebug
|
211
225
|
requirement: !ruby/object:Gem::Requirement
|
@@ -280,9 +294,11 @@ files:
|
|
280
294
|
- README.md
|
281
295
|
- Rakefile
|
282
296
|
- build_dev.sh
|
297
|
+
- docker-compose.yml
|
283
298
|
- exe/shipitron
|
284
299
|
- lib/shipitron.rb
|
285
300
|
- lib/shipitron/cli.rb
|
301
|
+
- lib/shipitron/client.rb
|
286
302
|
- lib/shipitron/client/bootstrap_application.rb
|
287
303
|
- lib/shipitron/client/create_ecs_services.rb
|
288
304
|
- lib/shipitron/client/deploy_application.rb
|
@@ -299,10 +315,12 @@ files:
|
|
299
315
|
- lib/shipitron/ecs_client.rb
|
300
316
|
- lib/shipitron/ecs_task_def.rb
|
301
317
|
- lib/shipitron/fetch_bucket.rb
|
318
|
+
- lib/shipitron/find_docker_volume_name.rb
|
302
319
|
- lib/shipitron/logger.rb
|
303
320
|
- lib/shipitron/mustache_yaml_parser.rb
|
304
321
|
- lib/shipitron/parse_templates.rb
|
305
322
|
- lib/shipitron/post_build.rb
|
323
|
+
- lib/shipitron/s3_copy.rb
|
306
324
|
- lib/shipitron/server/deploy_application.rb
|
307
325
|
- lib/shipitron/server/docker/build_image.rb
|
308
326
|
- lib/shipitron/server/docker/configure.rb
|
@@ -346,7 +364,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
346
364
|
- !ruby/object:Gem::Version
|
347
365
|
version: '0'
|
348
366
|
requirements: []
|
349
|
-
rubygems_version: 3.
|
367
|
+
rubygems_version: 3.1.2
|
350
368
|
signing_key:
|
351
369
|
specification_version: 4
|
352
370
|
summary: A deployment tool for use with Docker and ECS.
|