shipitron 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.buildkite/pipeline.yml +94 -0
  3. data/.dockerignore +1 -0
  4. data/.gitattributes +1 -0
  5. data/.gitignore +1 -0
  6. data/.rspec +1 -0
  7. data/Deskfile +15 -0
  8. data/Dockerfile +40 -15
  9. data/Dockerfile.release +29 -6
  10. data/Dockerfile.staging +70 -0
  11. data/Gemfile +2 -0
  12. data/README.md +53 -15
  13. data/docker-compose.yml +54 -0
  14. data/lib/shipitron.rb +19 -2
  15. data/lib/shipitron/cli.rb +42 -42
  16. data/lib/shipitron/client.rb +9 -0
  17. data/lib/shipitron/client/bootstrap_application.rb +1 -0
  18. data/lib/shipitron/client/create_ecs_services.rb +7 -7
  19. data/lib/shipitron/client/deploy_application.rb +2 -0
  20. data/lib/shipitron/client/ensure_deploy_not_running.rb +2 -1
  21. data/lib/shipitron/client/fetch_clusters.rb +1 -0
  22. data/lib/shipitron/client/force_deploy.rb +1 -0
  23. data/lib/shipitron/client/generate_deploy.rb +41 -0
  24. data/lib/shipitron/client/load_application_config.rb +17 -0
  25. data/lib/shipitron/client/load_templates.rb +1 -0
  26. data/lib/shipitron/client/register_ecs_task_definitions.rb +5 -5
  27. data/lib/shipitron/client/run_ecs_tasks.rb +101 -72
  28. data/lib/shipitron/docker_image.rb +4 -1
  29. data/lib/shipitron/find_docker_volume_name.rb +68 -0
  30. data/lib/shipitron/git_info.rb +57 -0
  31. data/lib/shipitron/mustache_yaml_parser.rb +12 -8
  32. data/lib/shipitron/s3_copy.rb +46 -0
  33. data/lib/shipitron/server/deploy_application.rb +2 -0
  34. data/lib/shipitron/server/docker/build_image.rb +4 -1
  35. data/lib/shipitron/server/docker/configure.rb +46 -10
  36. data/lib/shipitron/server/docker/push_image.rb +3 -0
  37. data/lib/shipitron/server/docker/run_build_script.rb +17 -10
  38. data/lib/shipitron/server/download_build_cache.rb +11 -3
  39. data/lib/shipitron/server/fetch_deploy.rb +50 -0
  40. data/lib/shipitron/server/git/clone_local_copy.rb +4 -5
  41. data/lib/shipitron/server/git/update_cache.rb +2 -1
  42. data/lib/shipitron/server/run_post_build.rb +40 -1
  43. data/lib/shipitron/server/transform_cli_args.rb +4 -0
  44. data/lib/shipitron/server/update_ecs_task_definitions.rb +2 -1
  45. data/lib/shipitron/server/upload_build_cache.rb +10 -9
  46. data/lib/shipitron/version.rb +1 -1
  47. data/scripts/docker-entrypoint.sh +37 -3
  48. data/scripts/release-entrypoint.sh +20 -0
  49. data/shipitron.gemspec +11 -8
  50. data/test.yml +5 -0
  51. metadata +78 -25
  52. data/build_dev.sh +0 -27
@@ -1,9 +1,12 @@
1
1
  require 'shipitron'
2
+ require 'shipitron/client'
2
3
  require 'shipitron/ecs_client'
4
+ require 'shipitron/client/generate_deploy'
3
5
  require 'shellwords'
4
6
  require 'base64'
5
7
  require 'tty-table'
6
8
  require 'pastel'
9
+ require 'securerandom'
7
10
 
8
11
  module Shipitron
9
12
  module Client
@@ -20,20 +23,16 @@ module Shipitron
20
23
  required :image_name
21
24
  required :named_tag
22
25
  required :ecs_task_defs
23
- optional :ecs_task_def_templates
24
- optional :ecs_services
25
- optional :ecs_service_templates
26
+ optional :ecs_task_def_templates, default: []
27
+ optional :ecs_services, default: []
28
+ optional :ecs_service_templates, default: []
26
29
  optional :build_script
27
- optional :post_builds
30
+ optional :skip_push
31
+ optional :post_builds, default: []
28
32
  optional :simulate
33
+ optional :simulate_store_deploy
29
34
  optional :repository_branch
30
-
31
- before do
32
- context.post_builds ||= []
33
- context.ecs_task_def_templates ||= {}
34
- context.ecs_services ||= []
35
- context.ecs_service_templates ||= {}
36
- end
35
+ optional :registry
37
36
 
38
37
  def call
39
38
  Logger.info "Skipping ECS run_task calls due to --simulate" if simulate?
@@ -53,27 +52,31 @@ module Shipitron
53
52
  Logger.info line.chomp
54
53
  end
55
54
 
56
- cluster = clusters.first
55
+ @cluster = clusters.first
57
56
 
58
57
  begin
59
58
  if simulate?
60
- command_args(cluster)
59
+ server_deploy_opts
60
+ generate_deploy! if context.simulate_store_deploy == true
61
+
61
62
  return
62
63
  end
63
64
 
64
- response = ecs_client(region: cluster.region).run_task(
65
- cluster: cluster.name,
65
+ generate_deploy!
66
+
67
+ response = ecs_client(region: @cluster.region).run_task(
68
+ cluster: @cluster.name,
66
69
  task_definition: shipitron_task,
67
70
  overrides: {
68
71
  container_overrides: [
69
72
  {
70
73
  name: 'shipitron',
71
- command: command_args(cluster)
74
+ command: command_args(deploy_id: deploy_id)
72
75
  }
73
76
  ]
74
77
  },
75
78
  count: 1,
76
- started_by: 'shipitron'
79
+ started_by: Shipitron::Client.started_by
77
80
  )
78
81
 
79
82
  if !response.failures.empty?
@@ -103,73 +106,99 @@ module Shipitron
103
106
  context.shipitron_task
104
107
  end
105
108
 
106
- def command_args(cluster)
109
+ def deploy_id
110
+ return @_deploy_id if defined?(@_deploy_id)
111
+
112
+ @_deploy_id = SecureRandom.uuid
113
+ end
114
+
115
+ def generate_deploy!
116
+ Shipitron::Client::GenerateDeploy.call!(
117
+ server_deploy_opts: server_deploy_opts,
118
+ deploy_id: deploy_id
119
+ )
120
+ end
121
+
122
+ def command_args(deploy_id:)
107
123
  [
108
124
  'server_deploy',
109
- '--name', context.application,
110
- '--repository', context.repository_url,
111
- '--bucket', context.s3_cache_bucket,
112
- '--build-cache-location', context.build_cache_location,
113
- '--image-name', context.image_name,
114
- '--named-tag', context.named_tag,
115
- '--region', cluster.region,
116
- ].tap do |ary|
117
- ary << '--clusters'
118
- ary.concat(context.clusters.map(&:name))
119
-
120
- ary << '--ecs-task-defs'
121
- ary.concat(context.ecs_task_defs)
122
-
123
- unless context.ecs_services.empty?
124
- ary << '--ecs-services'
125
- ary.concat(context.ecs_services)
126
- end
125
+ '--deploy-id', deploy_id
126
+ ]
127
+ end
127
128
 
128
- if context.build_script != nil
129
- ary.concat ['--build-script', context.build_script]
130
- end
129
+ def server_deploy_opts
130
+ return @_server_deploy_opts if defined?(@_server_deploy_opts)
131
+
132
+ @_server_deploy_opts =
133
+ {
134
+ name: context.application,
135
+ repository: context.repository_url,
136
+ bucket: context.s3_cache_bucket,
137
+ build_cache_location: context.build_cache_location,
138
+ image_name: context.image_name,
139
+ named_tag: context.named_tag,
140
+ region: @cluster.region
141
+ }.tap do |opts|
142
+ opts[:clusters] =
143
+ context.clusters.map(&:name)
144
+
145
+ opts[:ecs_task_defs] =
146
+ context.ecs_task_defs
147
+
148
+ unless context.ecs_services.empty?
149
+ opts[:ecs_services] =
150
+ context.ecs_services
151
+ end
131
152
 
132
- if !context.post_builds.empty?
133
- ary << '--post-builds'
134
- ary.concat(context.post_builds.map(&:to_s))
135
- end
153
+ if context.registry != nil
154
+ opts[:registry] = context.registry
155
+ end
136
156
 
137
- if !context.ecs_task_def_templates.empty?
138
- ary << '--ecs-task-def-templates'
139
- ary.concat(
140
- context.ecs_task_def_templates.map do |name, data|
141
- if context.ecs_task_defs.include?(name)
142
- Base64.urlsafe_encode64(data)
143
- end
144
- end.compact
145
- )
146
- end
157
+ if context.build_script != nil
158
+ opts[:build_script] = context.build_script
159
+ end
147
160
 
148
- if !context.ecs_service_templates.empty?
149
- ary << '--ecs-service-templates'
150
- ary.concat(
151
- context.ecs_service_templates.map do |name, data|
152
- if context.ecs_services.include?(name)
153
- Base64.urlsafe_encode64(data)
154
- end
155
- end.compact
156
- )
157
- end
161
+ if context.skip_push != nil
162
+ opts[:skip_push] = context.skip_push.to_s
163
+ end
158
164
 
159
- unless context.repository_branch.nil?
160
- ary.concat ['--repository-branch', context.repository_branch]
161
- end
165
+ if !context.post_builds.empty?
166
+ opts[:post_builds] =
167
+ context.post_builds.map(&:to_s)
168
+ end
162
169
 
163
- if simulate?
164
- Logger.info "server_deploy command: #{ary.shelljoin}"
165
- else
166
- Logger.debug "server_deploy command: #{ary.shelljoin}"
170
+ if !context.ecs_task_def_templates.empty?
171
+ opts[:ecs_task_def_templates] =
172
+ context.ecs_task_def_templates.map do |name, data|
173
+ if context.ecs_task_defs.include?(name)
174
+ Base64.urlsafe_encode64(data)
175
+ end
176
+ end.compact
177
+ end
178
+
179
+ if !context.ecs_service_templates.empty?
180
+ opts[:ecs_service_templates] =
181
+ context.ecs_service_templates.map do |name, data|
182
+ if context.ecs_services.include?(name)
183
+ Base64.urlsafe_encode64(data)
184
+ end
185
+ end.compact
186
+ end
187
+
188
+ unless context.repository_branch.nil?
189
+ opts[:repository_branch] = context.repository_branch
190
+ end
191
+
192
+ if simulate?
193
+ Logger.info "server_deploy opts:\n#{JSON.pretty_generate(opts)}"
194
+ else
195
+ Logger.debug "server_deploy opts:\n#{JSON.pretty_generate(opts)}"
196
+ end
167
197
  end
168
- end
169
198
  end
170
199
 
171
200
  def simulate?
172
- context.simulate == true
201
+ context.simulate == true || context.simulate_store_deploy == true
173
202
  end
174
203
  end
175
204
  end
@@ -2,6 +2,7 @@ require 'shipitron'
2
2
 
3
3
  module Shipitron
4
4
  class DockerImage < Hashie::Dash
5
+ property :registry
5
6
  property :name
6
7
  property :tag
7
8
 
@@ -13,7 +14,9 @@ module Shipitron
13
14
  tag_str = tag_str.dup.prepend(':')
14
15
  end
15
16
 
16
- "#{name}#{tag_str}"
17
+ name_with_registry = [registry, name].compact.join('/')
18
+
19
+ "#{name_with_registry}#{tag_str}"
17
20
  end
18
21
 
19
22
  def to_s
@@ -0,0 +1,68 @@
1
+ require 'excon'
2
+ require 'json'
3
+
4
+ module Shipitron
5
+ class FindDockerVolumeName
6
+ include Metaractor
7
+
8
+ required :container_name
9
+ required :volume_search
10
+
11
+ def call
12
+ volumes = container_volumes(container_name: container_name)
13
+
14
+ volume_metadata = volumes.find do |volume|
15
+ volume['DockerName'] =~ volume_search
16
+ end
17
+
18
+ if volume_metadata.nil?
19
+ raise 'Unable to find shipitron-home volume!'
20
+ end
21
+
22
+ context.volume_name = volume_metadata['DockerName']
23
+ end
24
+
25
+ private
26
+ def container_name
27
+ context.container_name
28
+ end
29
+
30
+ def volume_search
31
+ context.volume_search
32
+ end
33
+
34
+ def container_volumes(container_name:)
35
+ container_metadata = self.task_metadata['Containers'].find do |container|
36
+ container['Name'] == container_name
37
+ end
38
+
39
+ return {} if container_metadata.nil?
40
+
41
+ container_metadata['Volumes']
42
+ end
43
+
44
+ def task_metadata
45
+ return @task_metadata if defined?(@task_metadata)
46
+
47
+ begin
48
+ response = Excon.get(
49
+ "#{ENV['ECS_CONTAINER_METADATA_URI_V4']}/task",
50
+ expects: [200],
51
+ connect_timeout: 5,
52
+ read_timeout: 5,
53
+ write_timeout: 5,
54
+ tcp_nodelay: true
55
+ )
56
+
57
+ Logger.debug "Metadata result:"
58
+ Logger.debug(response.body)
59
+ Logger.debug "\n"
60
+
61
+ @task_metadata = JSON.parse(response.body)
62
+ rescue
63
+ Logger.info "Metadata uri failed"
64
+ {}
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,57 @@
1
+ require 'shipitron'
2
+ require 'rugged'
3
+
4
+ module Shipitron
5
+ class GitInfo < Hashie::Dash
6
+ property :sha
7
+ property :short_sha
8
+ property :email
9
+ property :name
10
+ property :summary
11
+ property :timestamp
12
+ property :branch
13
+ property :tag
14
+
15
+ def one_liner
16
+ "#{name} (#{short_sha}): #{summary}"
17
+ end
18
+
19
+ def self.from_path(path:)
20
+ repo = Rugged::Repository.new(path)
21
+ commit = repo.last_commit
22
+ self.new(
23
+ sha: commit.oid,
24
+ short_sha: commit.oid[0, 12],
25
+ email: commit.author.dig(:email),
26
+ name: commit.author.dig(:name),
27
+ summary: commit.summary,
28
+ timestamp: commit.epoch_time.to_s,
29
+ branch: branch_name(repo: repo),
30
+ tag: tag_name(repo: repo)
31
+ )
32
+ end
33
+
34
+ def self.branch_name(repo:)
35
+ ref = repo.head
36
+ ref.branch? ? ref.name.sub('refs/heads/', '') : nil
37
+ end
38
+
39
+ def self.tag_name(repo:)
40
+ ref = repo.head
41
+ tags = repo.tags
42
+
43
+ tags.each do |tag|
44
+ target_id =
45
+ if tag.annotated?
46
+ tag.annotation.target_id
47
+ else
48
+ tag.target.oid
49
+ end
50
+
51
+ return tag.name if ref.target_id == target_id
52
+ end
53
+
54
+ nil
55
+ end
56
+ end
57
+ end
@@ -3,20 +3,24 @@ require 'yaml'
3
3
 
4
4
  module Shipitron
5
5
  class MustacheYamlParser
6
- def initialize(context:nil, view:nil)
7
- if (context.nil? && view.nil?) || (!context.nil? && !view.nil?)
6
+ def initialize(file_path, options = {})
7
+ @context = options[:context]
8
+ @view = options[:view]
9
+
10
+ if (@context.nil? && @view.nil?) || (!@context.nil? && !@view.nil?)
8
11
  raise ArgumentError, 'Either context or view required'
9
12
  end
10
13
 
11
- @context = context
12
- @view = view
13
-
14
+ @file_path = file_path.is_a?(Pathname) ? file_path.to_s : file_path
14
15
  @view ||= Mustache
15
16
  end
16
17
 
17
- def perform(file_path)
18
- file_path = file_path.is_a?(Pathname) ? file_path.to_s : file_path
19
- YAML.load(@view.render(File.read(file_path), @context))
18
+ def perform
19
+ YAML.load(@view.render(File.read(@file_path), @context))
20
+ end
21
+
22
+ def self.perform(file_path, options = {})
23
+ new(file_path, options).perform
20
24
  end
21
25
  end
22
26
  end
@@ -0,0 +1,46 @@
1
+ require 'shipitron'
2
+ require 'shipitron/find_docker_volume_name'
3
+
4
+ module Shipitron
5
+ class S3Copy
6
+ include Metaractor
7
+
8
+ required :source
9
+ required :destination
10
+ required :region
11
+
12
+ def call
13
+ if ENV['FOG_LOCAL']
14
+ Logger.info `cp #{source.gsub('s3://', '/fog/')} #{destination.gsub('s3://', '/fog/')}`
15
+ if $? != 0
16
+ fail_with_error!(message: 'Failed to transfer to/from s3 (mocked).')
17
+ end
18
+ else
19
+ Logger.info "S3 Copy from #{source} to #{destination}"
20
+
21
+ shipitron_home_volume = FindDockerVolumeName.call!(
22
+ container_name: 'shipitron',
23
+ volume_search: /shipitron-home/
24
+ ).volume_name
25
+
26
+ Logger.info `docker run --rm -t -v #{shipitron_home_volume}:/home/shipitron -e AWS_CONTAINER_CREDENTIALS_RELATIVE_URI amazon/aws-cli:latest --region #{region} s3 cp #{source} #{destination} --quiet --only-show-errors`
27
+ if $? != 0
28
+ fail_with_error!(message: 'Failed to transfer to/from s3.')
29
+ end
30
+ end
31
+ end
32
+
33
+ private
34
+ def source
35
+ context.source
36
+ end
37
+
38
+ def destination
39
+ context.destination
40
+ end
41
+
42
+ def region
43
+ context.region
44
+ end
45
+ end
46
+ end