shipitron 1.2.1 → 1.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.buildkite/pipeline.yml +94 -0
- data/.dockerignore +1 -0
- data/.gitattributes +1 -0
- data/.gitignore +1 -0
- data/.rspec +1 -0
- data/Deskfile +15 -0
- data/Dockerfile +40 -15
- data/Dockerfile.release +28 -5
- data/Dockerfile.staging +70 -0
- data/Gemfile +2 -0
- data/README.md +53 -15
- data/docker-compose.yml +54 -0
- data/lib/shipitron.rb +19 -2
- data/lib/shipitron/cli.rb +42 -41
- data/lib/shipitron/client.rb +9 -0
- data/lib/shipitron/client/bootstrap_application.rb +1 -0
- data/lib/shipitron/client/create_ecs_services.rb +7 -7
- data/lib/shipitron/client/deploy_application.rb +2 -0
- data/lib/shipitron/client/ensure_deploy_not_running.rb +2 -1
- data/lib/shipitron/client/fetch_clusters.rb +1 -0
- data/lib/shipitron/client/force_deploy.rb +1 -0
- data/lib/shipitron/client/generate_deploy.rb +41 -0
- data/lib/shipitron/client/load_application_config.rb +17 -0
- data/lib/shipitron/client/load_templates.rb +1 -0
- data/lib/shipitron/client/register_ecs_task_definitions.rb +5 -5
- data/lib/shipitron/client/run_ecs_tasks.rb +101 -72
- data/lib/shipitron/docker_image.rb +4 -1
- data/lib/shipitron/find_docker_volume_name.rb +68 -0
- data/lib/shipitron/git_info.rb +57 -0
- data/lib/shipitron/mustache_yaml_parser.rb +12 -8
- data/lib/shipitron/s3_copy.rb +46 -0
- data/lib/shipitron/server/deploy_application.rb +2 -0
- data/lib/shipitron/server/docker/build_image.rb +4 -1
- data/lib/shipitron/server/docker/configure.rb +46 -10
- data/lib/shipitron/server/docker/push_image.rb +3 -0
- data/lib/shipitron/server/docker/run_build_script.rb +17 -10
- data/lib/shipitron/server/download_build_cache.rb +11 -3
- data/lib/shipitron/server/fetch_deploy.rb +50 -0
- data/lib/shipitron/server/git/clone_local_copy.rb +4 -5
- data/lib/shipitron/server/git/update_cache.rb +1 -1
- data/lib/shipitron/server/run_post_build.rb +40 -1
- data/lib/shipitron/server/transform_cli_args.rb +4 -0
- data/lib/shipitron/server/update_ecs_task_definitions.rb +2 -1
- data/lib/shipitron/server/upload_build_cache.rb +10 -9
- data/lib/shipitron/version.rb +1 -1
- data/scripts/docker-entrypoint.sh +38 -4
- data/scripts/release-entrypoint.sh +20 -0
- data/shipitron.gemspec +11 -8
- data/test.yml +5 -0
- metadata +78 -24
- data/build_dev.sh +0 -27
@@ -1,9 +1,12 @@
|
|
1
1
|
require 'shipitron'
|
2
|
+
require 'shipitron/client'
|
2
3
|
require 'shipitron/ecs_client'
|
4
|
+
require 'shipitron/client/generate_deploy'
|
3
5
|
require 'shellwords'
|
4
6
|
require 'base64'
|
5
7
|
require 'tty-table'
|
6
8
|
require 'pastel'
|
9
|
+
require 'securerandom'
|
7
10
|
|
8
11
|
module Shipitron
|
9
12
|
module Client
|
@@ -20,20 +23,16 @@ module Shipitron
|
|
20
23
|
required :image_name
|
21
24
|
required :named_tag
|
22
25
|
required :ecs_task_defs
|
23
|
-
optional :ecs_task_def_templates
|
24
|
-
optional :ecs_services
|
25
|
-
optional :ecs_service_templates
|
26
|
+
optional :ecs_task_def_templates, default: []
|
27
|
+
optional :ecs_services, default: []
|
28
|
+
optional :ecs_service_templates, default: []
|
26
29
|
optional :build_script
|
27
|
-
optional :
|
30
|
+
optional :skip_push
|
31
|
+
optional :post_builds, default: []
|
28
32
|
optional :simulate
|
33
|
+
optional :simulate_store_deploy
|
29
34
|
optional :repository_branch
|
30
|
-
|
31
|
-
before do
|
32
|
-
context.post_builds ||= []
|
33
|
-
context.ecs_task_def_templates ||= {}
|
34
|
-
context.ecs_services ||= []
|
35
|
-
context.ecs_service_templates ||= {}
|
36
|
-
end
|
35
|
+
optional :registry
|
37
36
|
|
38
37
|
def call
|
39
38
|
Logger.info "Skipping ECS run_task calls due to --simulate" if simulate?
|
@@ -53,27 +52,31 @@ module Shipitron
|
|
53
52
|
Logger.info line.chomp
|
54
53
|
end
|
55
54
|
|
56
|
-
cluster = clusters.first
|
55
|
+
@cluster = clusters.first
|
57
56
|
|
58
57
|
begin
|
59
58
|
if simulate?
|
60
|
-
|
59
|
+
server_deploy_opts
|
60
|
+
generate_deploy! if context.simulate_store_deploy == true
|
61
|
+
|
61
62
|
return
|
62
63
|
end
|
63
64
|
|
64
|
-
|
65
|
-
|
65
|
+
generate_deploy!
|
66
|
+
|
67
|
+
response = ecs_client(region: @cluster.region).run_task(
|
68
|
+
cluster: @cluster.name,
|
66
69
|
task_definition: shipitron_task,
|
67
70
|
overrides: {
|
68
71
|
container_overrides: [
|
69
72
|
{
|
70
73
|
name: 'shipitron',
|
71
|
-
command: command_args(
|
74
|
+
command: command_args(deploy_id: deploy_id)
|
72
75
|
}
|
73
76
|
]
|
74
77
|
},
|
75
78
|
count: 1,
|
76
|
-
started_by:
|
79
|
+
started_by: Shipitron::Client.started_by
|
77
80
|
)
|
78
81
|
|
79
82
|
if !response.failures.empty?
|
@@ -103,73 +106,99 @@ module Shipitron
|
|
103
106
|
context.shipitron_task
|
104
107
|
end
|
105
108
|
|
106
|
-
def
|
109
|
+
def deploy_id
|
110
|
+
return @_deploy_id if defined?(@_deploy_id)
|
111
|
+
|
112
|
+
@_deploy_id = SecureRandom.uuid
|
113
|
+
end
|
114
|
+
|
115
|
+
def generate_deploy!
|
116
|
+
Shipitron::Client::GenerateDeploy.call!(
|
117
|
+
server_deploy_opts: server_deploy_opts,
|
118
|
+
deploy_id: deploy_id
|
119
|
+
)
|
120
|
+
end
|
121
|
+
|
122
|
+
def command_args(deploy_id:)
|
107
123
|
[
|
108
124
|
'server_deploy',
|
109
|
-
'--
|
110
|
-
|
111
|
-
|
112
|
-
'--build-cache-location', context.build_cache_location,
|
113
|
-
'--image-name', context.image_name,
|
114
|
-
'--named-tag', context.named_tag,
|
115
|
-
'--region', cluster.region,
|
116
|
-
].tap do |ary|
|
117
|
-
ary << '--clusters'
|
118
|
-
ary.concat(context.clusters.map(&:name))
|
119
|
-
|
120
|
-
ary << '--ecs-task-defs'
|
121
|
-
ary.concat(context.ecs_task_defs)
|
122
|
-
|
123
|
-
unless context.ecs_services.empty?
|
124
|
-
ary << '--ecs-services'
|
125
|
-
ary.concat(context.ecs_services)
|
126
|
-
end
|
125
|
+
'--deploy-id', deploy_id
|
126
|
+
]
|
127
|
+
end
|
127
128
|
|
128
|
-
|
129
|
-
|
130
|
-
|
129
|
+
def server_deploy_opts
|
130
|
+
return @_server_deploy_opts if defined?(@_server_deploy_opts)
|
131
|
+
|
132
|
+
@_server_deploy_opts =
|
133
|
+
{
|
134
|
+
name: context.application,
|
135
|
+
repository: context.repository_url,
|
136
|
+
bucket: context.s3_cache_bucket,
|
137
|
+
build_cache_location: context.build_cache_location,
|
138
|
+
image_name: context.image_name,
|
139
|
+
named_tag: context.named_tag,
|
140
|
+
region: @cluster.region
|
141
|
+
}.tap do |opts|
|
142
|
+
opts[:clusters] =
|
143
|
+
context.clusters.map(&:name)
|
144
|
+
|
145
|
+
opts[:ecs_task_defs] =
|
146
|
+
context.ecs_task_defs
|
147
|
+
|
148
|
+
unless context.ecs_services.empty?
|
149
|
+
opts[:ecs_services] =
|
150
|
+
context.ecs_services
|
151
|
+
end
|
131
152
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
end
|
153
|
+
if context.registry != nil
|
154
|
+
opts[:registry] = context.registry
|
155
|
+
end
|
136
156
|
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
context.ecs_task_def_templates.map do |name, data|
|
141
|
-
if context.ecs_task_defs.include?(name)
|
142
|
-
Base64.urlsafe_encode64(data)
|
143
|
-
end
|
144
|
-
end.compact
|
145
|
-
)
|
146
|
-
end
|
157
|
+
if context.build_script != nil
|
158
|
+
opts[:build_script] = context.build_script
|
159
|
+
end
|
147
160
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
context.ecs_service_templates.map do |name, data|
|
152
|
-
if context.ecs_services.include?(name)
|
153
|
-
Base64.urlsafe_encode64(data)
|
154
|
-
end
|
155
|
-
end.compact
|
156
|
-
)
|
157
|
-
end
|
161
|
+
if context.skip_push != nil
|
162
|
+
opts[:skip_push] = context.skip_push.to_s
|
163
|
+
end
|
158
164
|
|
159
|
-
|
160
|
-
|
161
|
-
|
165
|
+
if !context.post_builds.empty?
|
166
|
+
opts[:post_builds] =
|
167
|
+
context.post_builds.map(&:to_s)
|
168
|
+
end
|
162
169
|
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
170
|
+
if !context.ecs_task_def_templates.empty?
|
171
|
+
opts[:ecs_task_def_templates] =
|
172
|
+
context.ecs_task_def_templates.map do |name, data|
|
173
|
+
if context.ecs_task_defs.include?(name)
|
174
|
+
Base64.urlsafe_encode64(data)
|
175
|
+
end
|
176
|
+
end.compact
|
177
|
+
end
|
178
|
+
|
179
|
+
if !context.ecs_service_templates.empty?
|
180
|
+
opts[:ecs_service_templates] =
|
181
|
+
context.ecs_service_templates.map do |name, data|
|
182
|
+
if context.ecs_services.include?(name)
|
183
|
+
Base64.urlsafe_encode64(data)
|
184
|
+
end
|
185
|
+
end.compact
|
186
|
+
end
|
187
|
+
|
188
|
+
unless context.repository_branch.nil?
|
189
|
+
opts[:repository_branch] = context.repository_branch
|
190
|
+
end
|
191
|
+
|
192
|
+
if simulate?
|
193
|
+
Logger.info "server_deploy opts:\n#{JSON.pretty_generate(opts)}"
|
194
|
+
else
|
195
|
+
Logger.debug "server_deploy opts:\n#{JSON.pretty_generate(opts)}"
|
196
|
+
end
|
167
197
|
end
|
168
|
-
end
|
169
198
|
end
|
170
199
|
|
171
200
|
def simulate?
|
172
|
-
context.simulate == true
|
201
|
+
context.simulate == true || context.simulate_store_deploy == true
|
173
202
|
end
|
174
203
|
end
|
175
204
|
end
|
@@ -2,6 +2,7 @@ require 'shipitron'
|
|
2
2
|
|
3
3
|
module Shipitron
|
4
4
|
class DockerImage < Hashie::Dash
|
5
|
+
property :registry
|
5
6
|
property :name
|
6
7
|
property :tag
|
7
8
|
|
@@ -13,7 +14,9 @@ module Shipitron
|
|
13
14
|
tag_str = tag_str.dup.prepend(':')
|
14
15
|
end
|
15
16
|
|
16
|
-
|
17
|
+
name_with_registry = [registry, name].compact.join('/')
|
18
|
+
|
19
|
+
"#{name_with_registry}#{tag_str}"
|
17
20
|
end
|
18
21
|
|
19
22
|
def to_s
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'excon'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class FindDockerVolumeName
|
6
|
+
include Metaractor
|
7
|
+
|
8
|
+
required :container_name
|
9
|
+
required :volume_search
|
10
|
+
|
11
|
+
def call
|
12
|
+
volumes = container_volumes(container_name: container_name)
|
13
|
+
|
14
|
+
volume_metadata = volumes.find do |volume|
|
15
|
+
volume['DockerName'] =~ volume_search
|
16
|
+
end
|
17
|
+
|
18
|
+
if volume_metadata.nil?
|
19
|
+
raise 'Unable to find shipitron-home volume!'
|
20
|
+
end
|
21
|
+
|
22
|
+
context.volume_name = volume_metadata['DockerName']
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
def container_name
|
27
|
+
context.container_name
|
28
|
+
end
|
29
|
+
|
30
|
+
def volume_search
|
31
|
+
context.volume_search
|
32
|
+
end
|
33
|
+
|
34
|
+
def container_volumes(container_name:)
|
35
|
+
container_metadata = self.task_metadata['Containers'].find do |container|
|
36
|
+
container['Name'] == container_name
|
37
|
+
end
|
38
|
+
|
39
|
+
return {} if container_metadata.nil?
|
40
|
+
|
41
|
+
container_metadata['Volumes']
|
42
|
+
end
|
43
|
+
|
44
|
+
def task_metadata
|
45
|
+
return @task_metadata if defined?(@task_metadata)
|
46
|
+
|
47
|
+
begin
|
48
|
+
response = Excon.get(
|
49
|
+
"#{ENV['ECS_CONTAINER_METADATA_URI_V4']}/task",
|
50
|
+
expects: [200],
|
51
|
+
connect_timeout: 5,
|
52
|
+
read_timeout: 5,
|
53
|
+
write_timeout: 5,
|
54
|
+
tcp_nodelay: true
|
55
|
+
)
|
56
|
+
|
57
|
+
Logger.debug "Metadata result:"
|
58
|
+
Logger.debug(response.body)
|
59
|
+
Logger.debug "\n"
|
60
|
+
|
61
|
+
@task_metadata = JSON.parse(response.body)
|
62
|
+
rescue
|
63
|
+
Logger.info "Metadata uri failed"
|
64
|
+
{}
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'shipitron'
|
2
|
+
require 'rugged'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class GitInfo < Hashie::Dash
|
6
|
+
property :sha
|
7
|
+
property :short_sha
|
8
|
+
property :email
|
9
|
+
property :name
|
10
|
+
property :summary
|
11
|
+
property :timestamp
|
12
|
+
property :branch
|
13
|
+
property :tag
|
14
|
+
|
15
|
+
def one_liner
|
16
|
+
"#{name} (#{short_sha}): #{summary}"
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.from_path(path:)
|
20
|
+
repo = Rugged::Repository.new(path)
|
21
|
+
commit = repo.last_commit
|
22
|
+
self.new(
|
23
|
+
sha: commit.oid,
|
24
|
+
short_sha: commit.oid[0, 12],
|
25
|
+
email: commit.author.dig(:email),
|
26
|
+
name: commit.author.dig(:name),
|
27
|
+
summary: commit.summary,
|
28
|
+
timestamp: commit.epoch_time.to_s,
|
29
|
+
branch: branch_name(repo: repo),
|
30
|
+
tag: tag_name(repo: repo)
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.branch_name(repo:)
|
35
|
+
ref = repo.head
|
36
|
+
ref.branch? ? ref.name.sub('refs/heads/', '') : nil
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.tag_name(repo:)
|
40
|
+
ref = repo.head
|
41
|
+
tags = repo.tags
|
42
|
+
|
43
|
+
tags.each do |tag|
|
44
|
+
target_id =
|
45
|
+
if tag.annotated?
|
46
|
+
tag.annotation.target_id
|
47
|
+
else
|
48
|
+
tag.target.oid
|
49
|
+
end
|
50
|
+
|
51
|
+
return tag.name if ref.target_id == target_id
|
52
|
+
end
|
53
|
+
|
54
|
+
nil
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -3,20 +3,24 @@ require 'yaml'
|
|
3
3
|
|
4
4
|
module Shipitron
|
5
5
|
class MustacheYamlParser
|
6
|
-
def initialize(
|
7
|
-
|
6
|
+
def initialize(file_path, options = {})
|
7
|
+
@context = options[:context]
|
8
|
+
@view = options[:view]
|
9
|
+
|
10
|
+
if (@context.nil? && @view.nil?) || (!@context.nil? && !@view.nil?)
|
8
11
|
raise ArgumentError, 'Either context or view required'
|
9
12
|
end
|
10
13
|
|
11
|
-
@
|
12
|
-
@view = view
|
13
|
-
|
14
|
+
@file_path = file_path.is_a?(Pathname) ? file_path.to_s : file_path
|
14
15
|
@view ||= Mustache
|
15
16
|
end
|
16
17
|
|
17
|
-
def perform
|
18
|
-
|
19
|
-
|
18
|
+
def perform
|
19
|
+
YAML.load(@view.render(File.read(@file_path), @context))
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.perform(file_path, options = {})
|
23
|
+
new(file_path, options).perform
|
20
24
|
end
|
21
25
|
end
|
22
26
|
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'shipitron'
|
2
|
+
require 'shipitron/find_docker_volume_name'
|
3
|
+
|
4
|
+
module Shipitron
|
5
|
+
class S3Copy
|
6
|
+
include Metaractor
|
7
|
+
|
8
|
+
required :source
|
9
|
+
required :destination
|
10
|
+
required :region
|
11
|
+
|
12
|
+
def call
|
13
|
+
if ENV['FOG_LOCAL']
|
14
|
+
Logger.info `cp #{source.gsub('s3://', '/fog/')} #{destination.gsub('s3://', '/fog/')}`
|
15
|
+
if $? != 0
|
16
|
+
fail_with_error!(message: 'Failed to transfer to/from s3 (mocked).')
|
17
|
+
end
|
18
|
+
else
|
19
|
+
Logger.info "S3 Copy from #{source} to #{destination}"
|
20
|
+
|
21
|
+
shipitron_home_volume = FindDockerVolumeName.call!(
|
22
|
+
container_name: 'shipitron',
|
23
|
+
volume_search: /shipitron-home/
|
24
|
+
).volume_name
|
25
|
+
|
26
|
+
Logger.info `docker run --rm -t -v #{shipitron_home_volume}:/home/shipitron -e AWS_CONTAINER_CREDENTIALS_RELATIVE_URI amazon/aws-cli:latest --region #{region} s3 cp #{source} #{destination} --quiet --only-show-errors`
|
27
|
+
if $? != 0
|
28
|
+
fail_with_error!(message: 'Failed to transfer to/from s3.')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
def source
|
35
|
+
context.source
|
36
|
+
end
|
37
|
+
|
38
|
+
def destination
|
39
|
+
context.destination
|
40
|
+
end
|
41
|
+
|
42
|
+
def region
|
43
|
+
context.region
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|