shipitron 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.buildkite/pipeline.yml +94 -0
- data/.dockerignore +1 -0
- data/.gitattributes +1 -0
- data/.gitignore +1 -0
- data/.rspec +1 -0
- data/Deskfile +15 -0
- data/Dockerfile +40 -15
- data/Dockerfile.release +29 -6
- data/Dockerfile.staging +70 -0
- data/Gemfile +2 -0
- data/README.md +53 -15
- data/docker-compose.yml +54 -0
- data/lib/shipitron.rb +19 -2
- data/lib/shipitron/cli.rb +42 -42
- data/lib/shipitron/client.rb +9 -0
- data/lib/shipitron/client/bootstrap_application.rb +1 -0
- data/lib/shipitron/client/create_ecs_services.rb +7 -7
- data/lib/shipitron/client/deploy_application.rb +2 -0
- data/lib/shipitron/client/ensure_deploy_not_running.rb +2 -1
- data/lib/shipitron/client/fetch_clusters.rb +1 -0
- data/lib/shipitron/client/force_deploy.rb +1 -0
- data/lib/shipitron/client/generate_deploy.rb +41 -0
- data/lib/shipitron/client/load_application_config.rb +17 -0
- data/lib/shipitron/client/load_templates.rb +1 -0
- data/lib/shipitron/client/register_ecs_task_definitions.rb +5 -5
- data/lib/shipitron/client/run_ecs_tasks.rb +101 -72
- data/lib/shipitron/docker_image.rb +4 -1
- data/lib/shipitron/find_docker_volume_name.rb +68 -0
- data/lib/shipitron/git_info.rb +57 -0
- data/lib/shipitron/mustache_yaml_parser.rb +12 -8
- data/lib/shipitron/s3_copy.rb +46 -0
- data/lib/shipitron/server/deploy_application.rb +2 -0
- data/lib/shipitron/server/docker/build_image.rb +4 -1
- data/lib/shipitron/server/docker/configure.rb +46 -10
- data/lib/shipitron/server/docker/push_image.rb +3 -0
- data/lib/shipitron/server/docker/run_build_script.rb +17 -10
- data/lib/shipitron/server/download_build_cache.rb +11 -3
- data/lib/shipitron/server/fetch_deploy.rb +50 -0
- data/lib/shipitron/server/git/clone_local_copy.rb +4 -5
- data/lib/shipitron/server/git/update_cache.rb +2 -1
- data/lib/shipitron/server/run_post_build.rb +40 -1
- data/lib/shipitron/server/transform_cli_args.rb +4 -0
- data/lib/shipitron/server/update_ecs_task_definitions.rb +2 -1
- data/lib/shipitron/server/upload_build_cache.rb +10 -9
- data/lib/shipitron/version.rb +1 -1
- data/scripts/docker-entrypoint.sh +37 -3
- data/scripts/release-entrypoint.sh +20 -0
- data/shipitron.gemspec +11 -8
- data/test.yml +5 -0
- metadata +78 -25
- data/build_dev.sh +0 -27
@@ -1,9 +1,12 @@
|
|
1
1
|
require 'shipitron'
|
2
|
+
require 'shipitron/client'
|
2
3
|
require 'shipitron/ecs_client'
|
4
|
+
require 'shipitron/client/generate_deploy'
|
3
5
|
require 'shellwords'
|
4
6
|
require 'base64'
|
5
7
|
require 'tty-table'
|
6
8
|
require 'pastel'
|
9
|
+
require 'securerandom'
|
7
10
|
|
8
11
|
module Shipitron
|
9
12
|
module Client
|
@@ -20,20 +23,16 @@ module Shipitron
|
|
20
23
|
required :image_name
|
21
24
|
required :named_tag
|
22
25
|
required :ecs_task_defs
|
23
|
-
optional :ecs_task_def_templates
|
24
|
-
optional :ecs_services
|
25
|
-
optional :ecs_service_templates
|
26
|
+
optional :ecs_task_def_templates, default: []
|
27
|
+
optional :ecs_services, default: []
|
28
|
+
optional :ecs_service_templates, default: []
|
26
29
|
optional :build_script
|
27
|
-
optional :
|
30
|
+
optional :skip_push
|
31
|
+
optional :post_builds, default: []
|
28
32
|
optional :simulate
|
33
|
+
optional :simulate_store_deploy
|
29
34
|
optional :repository_branch
|
30
|
-
|
31
|
-
before do
|
32
|
-
context.post_builds ||= []
|
33
|
-
context.ecs_task_def_templates ||= {}
|
34
|
-
context.ecs_services ||= []
|
35
|
-
context.ecs_service_templates ||= {}
|
36
|
-
end
|
35
|
+
optional :registry
|
37
36
|
|
38
37
|
def call
|
39
38
|
Logger.info "Skipping ECS run_task calls due to --simulate" if simulate?
|
@@ -53,27 +52,31 @@ module Shipitron
|
|
53
52
|
Logger.info line.chomp
|
54
53
|
end
|
55
54
|
|
56
|
-
cluster = clusters.first
|
55
|
+
@cluster = clusters.first
|
57
56
|
|
58
57
|
begin
|
59
58
|
if simulate?
|
60
|
-
|
59
|
+
server_deploy_opts
|
60
|
+
generate_deploy! if context.simulate_store_deploy == true
|
61
|
+
|
61
62
|
return
|
62
63
|
end
|
63
64
|
|
64
|
-
|
65
|
-
|
65
|
+
generate_deploy!
|
66
|
+
|
67
|
+
response = ecs_client(region: @cluster.region).run_task(
|
68
|
+
cluster: @cluster.name,
|
66
69
|
task_definition: shipitron_task,
|
67
70
|
overrides: {
|
68
71
|
container_overrides: [
|
69
72
|
{
|
70
73
|
name: 'shipitron',
|
71
|
-
command: command_args(
|
74
|
+
command: command_args(deploy_id: deploy_id)
|
72
75
|
}
|
73
76
|
]
|
74
77
|
},
|
75
78
|
count: 1,
|
76
|
-
started_by:
|
79
|
+
started_by: Shipitron::Client.started_by
|
77
80
|
)
|
78
81
|
|
79
82
|
if !response.failures.empty?
|
@@ -103,73 +106,99 @@ module Shipitron
|
|
103
106
|
context.shipitron_task
|
104
107
|
end
|
105
108
|
|
106
|
-
def
|
109
|
+
def deploy_id
|
110
|
+
return @_deploy_id if defined?(@_deploy_id)
|
111
|
+
|
112
|
+
@_deploy_id = SecureRandom.uuid
|
113
|
+
end
|
114
|
+
|
115
|
+
def generate_deploy!
|
116
|
+
Shipitron::Client::GenerateDeploy.call!(
|
117
|
+
server_deploy_opts: server_deploy_opts,
|
118
|
+
deploy_id: deploy_id
|
119
|
+
)
|
120
|
+
end
|
121
|
+
|
122
|
+
def command_args(deploy_id:)
|
107
123
|
[
|
108
124
|
'server_deploy',
|
109
|
-
'--
|
110
|
-
|
111
|
-
|
112
|
-
'--build-cache-location', context.build_cache_location,
|
113
|
-
'--image-name', context.image_name,
|
114
|
-
'--named-tag', context.named_tag,
|
115
|
-
'--region', cluster.region,
|
116
|
-
].tap do |ary|
|
117
|
-
ary << '--clusters'
|
118
|
-
ary.concat(context.clusters.map(&:name))
|
119
|
-
|
120
|
-
ary << '--ecs-task-defs'
|
121
|
-
ary.concat(context.ecs_task_defs)
|
122
|
-
|
123
|
-
unless context.ecs_services.empty?
|
124
|
-
ary << '--ecs-services'
|
125
|
-
ary.concat(context.ecs_services)
|
126
|
-
end
|
125
|
+
'--deploy-id', deploy_id
|
126
|
+
]
|
127
|
+
end
|
127
128
|
|
128
|
-
|
129
|
-
|
130
|
-
|
129
|
+
def server_deploy_opts
|
130
|
+
return @_server_deploy_opts if defined?(@_server_deploy_opts)
|
131
|
+
|
132
|
+
@_server_deploy_opts =
|
133
|
+
{
|
134
|
+
name: context.application,
|
135
|
+
repository: context.repository_url,
|
136
|
+
bucket: context.s3_cache_bucket,
|
137
|
+
build_cache_location: context.build_cache_location,
|
138
|
+
image_name: context.image_name,
|
139
|
+
named_tag: context.named_tag,
|
140
|
+
region: @cluster.region
|
141
|
+
}.tap do |opts|
|
142
|
+
opts[:clusters] =
|
143
|
+
context.clusters.map(&:name)
|
144
|
+
|
145
|
+
opts[:ecs_task_defs] =
|
146
|
+
context.ecs_task_defs
|
147
|
+
|
148
|
+
unless context.ecs_services.empty?
|
149
|
+
opts[:ecs_services] =
|
150
|
+
context.ecs_services
|
151
|
+
end
|
131
152
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
end
|
153
|
+
if context.registry != nil
|
154
|
+
opts[:registry] = context.registry
|
155
|
+
end
|
136
156
|
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
context.ecs_task_def_templates.map do |name, data|
|
141
|
-
if context.ecs_task_defs.include?(name)
|
142
|
-
Base64.urlsafe_encode64(data)
|
143
|
-
end
|
144
|
-
end.compact
|
145
|
-
)
|
146
|
-
end
|
157
|
+
if context.build_script != nil
|
158
|
+
opts[:build_script] = context.build_script
|
159
|
+
end
|
147
160
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
context.ecs_service_templates.map do |name, data|
|
152
|
-
if context.ecs_services.include?(name)
|
153
|
-
Base64.urlsafe_encode64(data)
|
154
|
-
end
|
155
|
-
end.compact
|
156
|
-
)
|
157
|
-
end
|
161
|
+
if context.skip_push != nil
|
162
|
+
opts[:skip_push] = context.skip_push.to_s
|
163
|
+
end
|
158
164
|
|
159
|
-
|
160
|
-
|
161
|
-
|
165
|
+
if !context.post_builds.empty?
|
166
|
+
opts[:post_builds] =
|
167
|
+
context.post_builds.map(&:to_s)
|
168
|
+
end
|
162
169
|
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
170
|
+
if !context.ecs_task_def_templates.empty?
|
171
|
+
opts[:ecs_task_def_templates] =
|
172
|
+
context.ecs_task_def_templates.map do |name, data|
|
173
|
+
if context.ecs_task_defs.include?(name)
|
174
|
+
Base64.urlsafe_encode64(data)
|
175
|
+
end
|
176
|
+
end.compact
|
177
|
+
end
|
178
|
+
|
179
|
+
if !context.ecs_service_templates.empty?
|
180
|
+
opts[:ecs_service_templates] =
|
181
|
+
context.ecs_service_templates.map do |name, data|
|
182
|
+
if context.ecs_services.include?(name)
|
183
|
+
Base64.urlsafe_encode64(data)
|
184
|
+
end
|
185
|
+
end.compact
|
186
|
+
end
|
187
|
+
|
188
|
+
unless context.repository_branch.nil?
|
189
|
+
opts[:repository_branch] = context.repository_branch
|
190
|
+
end
|
191
|
+
|
192
|
+
if simulate?
|
193
|
+
Logger.info "server_deploy opts:\n#{JSON.pretty_generate(opts)}"
|
194
|
+
else
|
195
|
+
Logger.debug "server_deploy opts:\n#{JSON.pretty_generate(opts)}"
|
196
|
+
end
|
167
197
|
end
|
168
|
-
end
|
169
198
|
end
|
170
199
|
|
171
200
|
def simulate?
|
172
|
-
context.simulate == true
|
201
|
+
context.simulate == true || context.simulate_store_deploy == true
|
173
202
|
end
|
174
203
|
end
|
175
204
|
end
|
@@ -2,6 +2,7 @@ require 'shipitron'
|
|
2
2
|
|
3
3
|
module Shipitron
|
4
4
|
class DockerImage < Hashie::Dash
|
5
|
+
property :registry
|
5
6
|
property :name
|
6
7
|
property :tag
|
7
8
|
|
@@ -13,7 +14,9 @@ module Shipitron
|
|
13
14
|
tag_str = tag_str.dup.prepend(':')
|
14
15
|
end
|
15
16
|
|
16
|
-
|
17
|
+
name_with_registry = [registry, name].compact.join('/')
|
18
|
+
|
19
|
+
"#{name_with_registry}#{tag_str}"
|
17
20
|
end
|
18
21
|
|
19
22
|
def to_s
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'excon'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class FindDockerVolumeName
|
6
|
+
include Metaractor
|
7
|
+
|
8
|
+
required :container_name
|
9
|
+
required :volume_search
|
10
|
+
|
11
|
+
def call
|
12
|
+
volumes = container_volumes(container_name: container_name)
|
13
|
+
|
14
|
+
volume_metadata = volumes.find do |volume|
|
15
|
+
volume['DockerName'] =~ volume_search
|
16
|
+
end
|
17
|
+
|
18
|
+
if volume_metadata.nil?
|
19
|
+
raise 'Unable to find shipitron-home volume!'
|
20
|
+
end
|
21
|
+
|
22
|
+
context.volume_name = volume_metadata['DockerName']
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
def container_name
|
27
|
+
context.container_name
|
28
|
+
end
|
29
|
+
|
30
|
+
def volume_search
|
31
|
+
context.volume_search
|
32
|
+
end
|
33
|
+
|
34
|
+
def container_volumes(container_name:)
|
35
|
+
container_metadata = self.task_metadata['Containers'].find do |container|
|
36
|
+
container['Name'] == container_name
|
37
|
+
end
|
38
|
+
|
39
|
+
return {} if container_metadata.nil?
|
40
|
+
|
41
|
+
container_metadata['Volumes']
|
42
|
+
end
|
43
|
+
|
44
|
+
def task_metadata
|
45
|
+
return @task_metadata if defined?(@task_metadata)
|
46
|
+
|
47
|
+
begin
|
48
|
+
response = Excon.get(
|
49
|
+
"#{ENV['ECS_CONTAINER_METADATA_URI_V4']}/task",
|
50
|
+
expects: [200],
|
51
|
+
connect_timeout: 5,
|
52
|
+
read_timeout: 5,
|
53
|
+
write_timeout: 5,
|
54
|
+
tcp_nodelay: true
|
55
|
+
)
|
56
|
+
|
57
|
+
Logger.debug "Metadata result:"
|
58
|
+
Logger.debug(response.body)
|
59
|
+
Logger.debug "\n"
|
60
|
+
|
61
|
+
@task_metadata = JSON.parse(response.body)
|
62
|
+
rescue
|
63
|
+
Logger.info "Metadata uri failed"
|
64
|
+
{}
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'shipitron'
|
2
|
+
require 'rugged'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class GitInfo < Hashie::Dash
|
6
|
+
property :sha
|
7
|
+
property :short_sha
|
8
|
+
property :email
|
9
|
+
property :name
|
10
|
+
property :summary
|
11
|
+
property :timestamp
|
12
|
+
property :branch
|
13
|
+
property :tag
|
14
|
+
|
15
|
+
def one_liner
|
16
|
+
"#{name} (#{short_sha}): #{summary}"
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.from_path(path:)
|
20
|
+
repo = Rugged::Repository.new(path)
|
21
|
+
commit = repo.last_commit
|
22
|
+
self.new(
|
23
|
+
sha: commit.oid,
|
24
|
+
short_sha: commit.oid[0, 12],
|
25
|
+
email: commit.author.dig(:email),
|
26
|
+
name: commit.author.dig(:name),
|
27
|
+
summary: commit.summary,
|
28
|
+
timestamp: commit.epoch_time.to_s,
|
29
|
+
branch: branch_name(repo: repo),
|
30
|
+
tag: tag_name(repo: repo)
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.branch_name(repo:)
|
35
|
+
ref = repo.head
|
36
|
+
ref.branch? ? ref.name.sub('refs/heads/', '') : nil
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.tag_name(repo:)
|
40
|
+
ref = repo.head
|
41
|
+
tags = repo.tags
|
42
|
+
|
43
|
+
tags.each do |tag|
|
44
|
+
target_id =
|
45
|
+
if tag.annotated?
|
46
|
+
tag.annotation.target_id
|
47
|
+
else
|
48
|
+
tag.target.oid
|
49
|
+
end
|
50
|
+
|
51
|
+
return tag.name if ref.target_id == target_id
|
52
|
+
end
|
53
|
+
|
54
|
+
nil
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -3,20 +3,24 @@ require 'yaml'
|
|
3
3
|
|
4
4
|
module Shipitron
|
5
5
|
class MustacheYamlParser
|
6
|
-
def initialize(
|
7
|
-
|
6
|
+
def initialize(file_path, options = {})
|
7
|
+
@context = options[:context]
|
8
|
+
@view = options[:view]
|
9
|
+
|
10
|
+
if (@context.nil? && @view.nil?) || (!@context.nil? && !@view.nil?)
|
8
11
|
raise ArgumentError, 'Either context or view required'
|
9
12
|
end
|
10
13
|
|
11
|
-
@
|
12
|
-
@view = view
|
13
|
-
|
14
|
+
@file_path = file_path.is_a?(Pathname) ? file_path.to_s : file_path
|
14
15
|
@view ||= Mustache
|
15
16
|
end
|
16
17
|
|
17
|
-
def perform
|
18
|
-
|
19
|
-
|
18
|
+
def perform
|
19
|
+
YAML.load(@view.render(File.read(@file_path), @context))
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.perform(file_path, options = {})
|
23
|
+
new(file_path, options).perform
|
20
24
|
end
|
21
25
|
end
|
22
26
|
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'shipitron'
|
2
|
+
require 'shipitron/find_docker_volume_name'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class S3Copy
|
6
|
+
include Metaractor
|
7
|
+
|
8
|
+
required :source
|
9
|
+
required :destination
|
10
|
+
required :region
|
11
|
+
|
12
|
+
def call
|
13
|
+
if ENV['FOG_LOCAL']
|
14
|
+
Logger.info `cp #{source.gsub('s3://', '/fog/')} #{destination.gsub('s3://', '/fog/')}`
|
15
|
+
if $? != 0
|
16
|
+
fail_with_error!(message: 'Failed to transfer to/from s3 (mocked).')
|
17
|
+
end
|
18
|
+
else
|
19
|
+
Logger.info "S3 Copy from #{source} to #{destination}"
|
20
|
+
|
21
|
+
shipitron_home_volume = FindDockerVolumeName.call!(
|
22
|
+
container_name: 'shipitron',
|
23
|
+
volume_search: /shipitron-home/
|
24
|
+
).volume_name
|
25
|
+
|
26
|
+
Logger.info `docker run --rm -t -v #{shipitron_home_volume}:/home/shipitron -e AWS_CONTAINER_CREDENTIALS_RELATIVE_URI amazon/aws-cli:latest --region #{region} s3 cp #{source} #{destination} --quiet --only-show-errors`
|
27
|
+
if $? != 0
|
28
|
+
fail_with_error!(message: 'Failed to transfer to/from s3.')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
def source
|
35
|
+
context.source
|
36
|
+
end
|
37
|
+
|
38
|
+
def destination
|
39
|
+
context.destination
|
40
|
+
end
|
41
|
+
|
42
|
+
def region
|
43
|
+
context.region
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|