hippo-cli 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,274 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'open3'
4
+ require 'hippo/error'
5
+ require 'hippo/recipe'
6
+
7
+ module Hippo
8
+ class CLISteps
9
+ attr_reader :recipe
10
+ attr_reader :stage
11
+
12
+ def initialize(recipe, stage)
13
+ @recipe = recipe
14
+ @stage = stage
15
+ end
16
+
17
+ # Prepare the repository for this build by getting the latest
18
+ # version from the remote and checking out the branch.
19
+ def prepare_repository(fetch: true)
20
+ info "Using repository #{@recipe.repository.url}"
21
+ if fetch
22
+ if @recipe.repository.cloned?
23
+ info 'Repository is already cloned'
24
+ action 'Fetching the latest repository data...'
25
+ @recipe.repository.fetch
26
+ else
27
+ info 'Repository is not yet cloned.'
28
+ action 'Cloning repository...'
29
+ @recipe.repository.clone
30
+ end
31
+
32
+ elsif !fetch && !@recipe.repository.cloned?
33
+ raise Error, 'Repository is not cloned yet so cannot continue'
34
+ else
35
+ info 'Not fetching latest repository, using cached copy'
36
+ end
37
+
38
+ action "Checking out '#{@stage.branch}' branch..."
39
+ @recipe.repository.checkout(@stage.branch)
40
+ @commit = @recipe.repository.commit
41
+ info "Latest commit on branch is #{@commit.objectish}"
42
+ info "Message: #{@commit.message.split("\n").first}"
43
+ @commit
44
+ end
45
+
46
+ def build
47
+ if @commit.nil?
48
+ raise Error, 'You cannot build without first preparing the repository'
49
+ end
50
+
51
+ Dir.chdir(@recipe.repository.path) do
52
+ @recipe.build_specs.each do |_, build_spec|
53
+ if build_spec.image_name.nil?
54
+ raise Error, "No image-name has been specified for build #{build_spec.name}"
55
+ end
56
+
57
+ image_name = build_spec.image_name_for_commit(@commit)
58
+ action "Building #{build_spec.name} with tag #{image_name}"
59
+
60
+ command = [
61
+ 'docker', 'build', '.',
62
+ '-f', build_spec.dockerfile,
63
+ '-t', image_name
64
+ ]
65
+ external_command do
66
+ if system(*command)
67
+ @built_commit = @commit
68
+ success "Successfully built image #{build_spec.image_name} for #{build_spec.name}"
69
+ else
70
+ raise Error, "Image for #{build_spec.name} did not succeed. Check output and try again."
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
76
+
77
+ def publish
78
+ if @built_commit.nil?
79
+ raise Error, 'You cannot publish without first building the image'
80
+ end
81
+
82
+ Dir.chdir(@recipe.repository.path) do
83
+ @recipe.build_specs.each do |_, build_spec|
84
+ if build_spec.image_name.nil?
85
+ raise Error, "No image-name has been specified for build #{build_spec.name}"
86
+ end
87
+
88
+ image_name = build_spec.image_name_for_commit(@built_commit.objectish)
89
+ action "Publishing #{build_spec.name} with tag #{image_name}"
90
+
91
+ command = ['docker', 'push', image_name]
92
+ external_command do
93
+ if system(*command)
94
+ success "Successfully published image #{image_name} for #{build_spec.name}"
95
+ else
96
+ raise Error, "Image for #{build_spec.name} was not published successfully. Check output and try again."
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
102
+
103
+ def run_install_jobs
104
+ run_jobs('install')
105
+ end
106
+
107
+ def run_deploy_jobs
108
+ run_jobs('deploy')
109
+ end
110
+
111
+ def deploy
112
+ action 'Applying deployments'
113
+ deployments = @recipe.kubernetes.objects('deployments', @stage, @commit)
114
+
115
+ if deployments.nil?
116
+ info 'No deployments file configured. Not applying any deployments'
117
+ end
118
+
119
+ external_command do
120
+ @recipe.kubernetes.apply_with_kubectl(deployments)
121
+ end
122
+ success 'Deployments applied successfully'
123
+ puts 'You can watch the deployment progressing using the command below:'
124
+ puts
125
+ puts " ⏰ #{@stage.kubectl('get pods --watch')}"
126
+ deployments.each do |deployment|
127
+ puts ' 👩🏼‍💻 ' + @stage.kubectl("describe deployment #{deployment['metadata']['name']}")
128
+ end
129
+ puts
130
+ end
131
+
132
+ def apply_services
133
+ action 'Applying services'
134
+ objects = @recipe.kubernetes.objects('services', @stage, @commit)
135
+ if objects.empty?
136
+ info 'No services have been defined'
137
+ else
138
+ external_command do
139
+ @recipe.kubernetes.apply_with_kubectl(objects)
140
+ end
141
+ success 'Services applied successfully'
142
+ end
143
+ end
144
+
145
+ def apply_namespace
146
+ action 'Applying namespace'
147
+ external_command { @recipe.kubernetes.apply_namespace(@stage) }
148
+ success 'Namespace applied successfully'
149
+ end
150
+
151
+ def apply_config
152
+ action 'Applying configuration'
153
+ objects = @recipe.kubernetes.objects('config', @stage, @commit)
154
+ if objects.empty?
155
+ info 'No configuration files have been defined'
156
+ else
157
+ external_command do
158
+ @recipe.kubernetes.apply_with_kubectl(objects)
159
+ end
160
+ success 'Configuration applied successfully'
161
+ end
162
+ end
163
+
164
+ def apply_secrets
165
+ require 'hippo/secret_manager'
166
+ action 'Applying secrets'
167
+ manager = SecretManager.new(@recipe, @stage)
168
+ unless manager.key_available?
169
+ error 'No secret encryption key was available. Not applying secrets.'
170
+ return
171
+ end
172
+
173
+ yamls = manager.secrets.map(&:to_secret_yaml).join("---\n")
174
+ external_command do
175
+ @recipe.kubernetes.apply_with_kubectl(yamls)
176
+ end
177
+ success 'Secrets applicated successfully'
178
+ end
179
+
180
+ private
181
+
182
+ def info(text)
183
+ puts text
184
+ end
185
+
186
+ def success(text)
187
+ puts "\e[32m#{text}\e[0m"
188
+ end
189
+
190
+ def action(text)
191
+ puts "\e[33m#{text}\e[0m"
192
+ end
193
+
194
+ def error(text)
195
+ puts "\e[31m#{text}\e[0m"
196
+ end
197
+
198
+ def external_command
199
+ $stdout.print "\e[37m"
200
+ yield
201
+ ensure
202
+ $stdout.print "\e[0m"
203
+ end
204
+
205
+ def run_jobs(type)
206
+ objects = @recipe.kubernetes.objects("jobs/#{type}", @stage, @commit)
207
+ if objects.empty?
208
+ info "No #{type} jobs exist so not applying anything"
209
+ return true
210
+ end
211
+
212
+ action "Applying #{type} job objects objects to Kubernetes"
213
+
214
+ result = nil
215
+ external_command do
216
+ # Remove any previous jobs that might have been running before
217
+ objects.each do |job|
218
+ @recipe.kubernetes.delete_job(@stage, job['metadata']['name'])
219
+ end
220
+
221
+ result = @recipe.kubernetes.apply_with_kubectl(objects)
222
+ end
223
+
224
+ puts 'Waiting for all scheduled jobs to finish...'
225
+ timeout, jobs = @recipe.kubernetes.wait_for_jobs(@stage, result.keys)
226
+ success_jobs = []
227
+ failed_jobs = []
228
+ jobs.each do |job|
229
+ if job['status']['succeeded']
230
+ success_jobs << job
231
+ else
232
+ failed_jobs << job
233
+ end
234
+ end
235
+
236
+ if success_jobs.size == jobs.size
237
+ success 'All jobs completed successfully.'
238
+ puts 'You can review the logs for these by running the commands below.'
239
+ puts
240
+ result = true
241
+ else
242
+ error 'Not all install jobs completed successfully.'
243
+ puts 'You should review the logs for these using the commands below.'
244
+ puts
245
+ result = false
246
+ end
247
+
248
+ jobs.each do |job|
249
+ icon = if job['status']['succeeded']
250
+ '✅'
251
+ else
252
+ '❌'
253
+ end
254
+ puts " #{icon} " + @stage.kubectl("logs job/#{job['metadata']['name']}")
255
+ end
256
+ puts
257
+
258
+ result
259
+ end
260
+
261
+ class << self
262
+ def setup(context)
263
+ recipe = Hippo::Recipe.load_from_file(context.options[:hippofile] || './Hippofile')
264
+
265
+ stage = recipe.stages[CURRENT_STAGE]
266
+ if stage.nil?
267
+ raise Error, "Invalid stage name `#{CURRENT_STAGE}`. Check this has been defined in in your stages directory with a matching name?"
268
+ end
269
+
270
+ new(recipe, stage)
271
+ end
272
+ end
273
+ end
274
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Hippo
4
+ class Error < StandardError
5
+ end
6
+
7
+ class RepositoryAlreadyClonedError < Error
8
+ end
9
+
10
+ class RepositoryCloneError < Error
11
+ end
12
+
13
+ class RepositoryFetchError < Error
14
+ end
15
+
16
+ class RepositoryCheckoutError < Error
17
+ end
18
+ end
@@ -0,0 +1,200 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'liquid'
4
+ require 'hippo/util'
5
+
6
+ module Hippo
7
+ class Kubernetes
8
+ OBJECT_DIRECTORY_NAMES = %w[config deployments jobs/install jobs/deploy services].freeze
9
+
10
+ include Hippo::Util
11
+
12
+ def initialize(recipe, options)
13
+ @recipe = recipe
14
+ @options = options
15
+ end
16
+
17
+ # Load and return a set of objects from a given path.
18
+ # Parse them through the templating and return them in the appropriate
19
+ # context.
20
+ #
21
+ # @param stage [Hippo::Stage]
22
+ # @param commit [String] the commit ref
23
+ # @param path [String]
24
+ # @return
25
+ def objects(path, stage, commit)
26
+ time = Time.now
27
+
28
+ yamls = load_yaml_from_directory(path)
29
+ yamls |= load_yaml_from_directory(File.join(path, stage.name))
30
+
31
+ yamls.map do |yaml_part|
32
+ object = yaml_part.parse(@recipe, stage, commit)
33
+
34
+ # Unless a namespace has been specified in the metadata we will
35
+ # want to add the namespace for the current stage.
36
+ if object['metadata'].nil? || object['metadata']['namespace'].nil?
37
+ object['metadata'] ||= {}
38
+ object['metadata']['namespace'] = stage.namespace
39
+ end
40
+
41
+ # Add our own details to the metadata of all objets created by us so
42
+ # we know where they came from.
43
+ object['metadata']['annotations'] ||= {}
44
+ object['metadata']['annotations']['hippo.adam.ac/builtAt'] ||= time.to_s
45
+ object['metadata']['annotations']['hippo.adam.ac/builtBy'] ||= ENV['USER'] || 'unknown'
46
+
47
+ add_default_labels(object, stage)
48
+
49
+ # Add some information to Deployments to reflect the latest
50
+ # information about this deployment.
51
+ if object['kind'] == 'Deployment'
52
+ object['metadata']['annotations']['hippo.adam.ac/deployID'] ||= time.to_i.to_s
53
+ if commit
54
+ object['metadata']['annotations']['hippo.adam.ac/commitRef'] ||= commit.objectish
55
+ object['metadata']['annotations']['hippo.adam.ac/commitMessage'] ||= commit.message
56
+ end
57
+
58
+ if pod_metadata = object.dig('spec', 'template', 'metadata')
59
+ pod_metadata['annotations'] ||= {}
60
+ pod_metadata['annotations']['hippo.adam.ac/deployID'] ||= time.to_i.to_s
61
+ if commit
62
+ pod_metadata['annotations']['hippo.adam.ac/commitRef'] ||= commit.objectish
63
+ end
64
+ end
65
+ end
66
+
67
+ object
68
+ end
69
+ end
70
+
71
+ def apply_namespace(stage)
72
+ namespace = {
73
+ 'kind' => 'Namespace',
74
+ 'apiVersion' => 'v1',
75
+ 'metadata' => {
76
+ 'name' => stage.namespace
77
+ }
78
+ }
79
+ add_default_labels(namespace, stage)
80
+ apply_with_kubectl(namespace.to_yaml)
81
+ end
82
+
83
+ # Apply the given configuration with kubectl
84
+ #
85
+ # @param config [Array<Hippo::YAMLPart>, String]
86
+ # @return [void]
87
+ def apply_with_kubectl(yaml_parts)
88
+ unless yaml_parts.is_a?(String)
89
+ yaml_parts = [yaml_parts] unless yaml_parts.is_a?(Array)
90
+ yaml_parts = yaml_parts.map { |yp| yp.hash.to_yaml }.join("\n---\n")
91
+ end
92
+
93
+ Open3.popen3('kubectl apply -f -') do |stdin, stdout, stderr, wt|
94
+ stdin.puts yaml_parts
95
+ stdin.close
96
+
97
+ stdout = stdout.read.strip
98
+ stderr = stderr.read.strip
99
+
100
+ if wt.value.success?
101
+ puts stdout
102
+ stdout.split("\n").each_with_object({}) do |line, hash|
103
+ if line =~ %r{\A([\w\/\-\.]+) (\w+)\z}
104
+ hash[Regexp.last_match(1)] = Regexp.last_match(2)
105
+ end
106
+ end
107
+ else
108
+ raise Error, "[kubectl] #{stderr}"
109
+ end
110
+ end
111
+ end
112
+
113
+ # Get details of objects using kubectl.
114
+ #
115
+ # @param stage [Hippo::Stage]
116
+ # @param names [Array<String>]
117
+ # @raises [Hippo::Error]
118
+ # @return [Array<Hash>]
119
+ def get_with_kubectl(stage, *names)
120
+ command = [
121
+ 'kubectl',
122
+ '-n', stage.namespace,
123
+ 'get',
124
+ names,
125
+ '-o', 'yaml'
126
+ ].flatten.reject(&:nil?)
127
+
128
+ Open3.popen3(*command) do |_, stdout, stderr, wt|
129
+ if wt.value.success?
130
+ yaml = YAML.safe_load(stdout.read, permitted_classes: [Time])
131
+ yaml['items'] || [yaml]
132
+ else
133
+ raise Error, "[kutectl] #{stderr.read}"
134
+ end
135
+ end
136
+ end
137
+
138
+ # Delete a named job from the cluster
139
+ #
140
+ # @param stage [Hippo::Stage]
141
+ # @param name [String]
142
+ # @raises [Hippo::Error]
143
+ # @return [void]
144
+ def delete_job(stage, name)
145
+ command = [
146
+ 'kubectl',
147
+ '-n', stage.namespace,
148
+ 'delete',
149
+ 'job',
150
+ name
151
+ ]
152
+
153
+ Open3.popen3(*command) do |_, stdout, stderr, wt|
154
+ if wt.value.success?
155
+ puts stdout.read
156
+ true
157
+ else
158
+ stderr = stderr.read
159
+ if stderr =~ /\"#{name}\" not found/
160
+ false
161
+ else
162
+ raise Error, "[kutectl] #{stderr.read}"
163
+ end
164
+ end
165
+ end
166
+ end
167
+
168
+ # Poll the named jobs and return them when all are complete
169
+ # or the number of checks is exceeded.
170
+ #
171
+ # @param stage [Hippo::Stage]
172
+ # @param names [Array<String>]
173
+ # @param times [Integer]
174
+ # @return [Array<Boolean, Array<Hash>]
175
+ def wait_for_jobs(stage, names, times = 120)
176
+ jobs = nil
177
+ times.times do
178
+ jobs = get_with_kubectl(stage, *names)
179
+
180
+ # Are all the jobs completed?
181
+ if jobs.all? { |j| j['status']['active'].nil? }
182
+ return [false, jobs]
183
+ else
184
+ sleep 2
185
+ end
186
+ end
187
+ [true, jobs]
188
+ end
189
+
190
+ private
191
+
192
+ def add_default_labels(object, stage)
193
+ object['metadata']['labels'] ||= {}
194
+ object['metadata']['labels']['app.kubernetes.io/name'] = @recipe.name
195
+ object['metadata']['labels']['app.kubernetes.io/instance'] = stage.name
196
+ object['metadata']['labels']['app.kubernetes.io/managed-by'] = 'hippo'
197
+ object
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,126 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'yaml'
4
+ require 'hippo/build_spec'
5
+ require 'hippo/error'
6
+ require 'hippo/kubernetes'
7
+ require 'hippo/repository'
8
+ require 'hippo/stage'
9
+ require 'hippo/util'
10
+
11
+ module Hippo
12
+ class Recipe
13
+ include Util
14
+
15
+ class RecipeNotFound < Error
16
+ end
17
+
18
+ class << self
19
+ # Load a new Recipe class from a given file.
20
+ #
21
+ # @param path [String] path to recipe file.
22
+ # @return [Hippo::Recipe]
23
+ def load_from_file(path)
24
+ unless File.file?(path)
25
+ raise RecipeNotFound, "No recipe file found at #{path}"
26
+ end
27
+
28
+ hash = YAML.load_file(path)
29
+ new(hash, path)
30
+ end
31
+ end
32
+
33
+ # @param hash [Hash] the raw hash from the underlying yaml file
34
+ def initialize(hash, hippofile_path = nil)
35
+ @hash = hash
36
+ @hippofile_path = hippofile_path
37
+ end
38
+
39
+ attr_reader :path
40
+
41
+ # Return the root directory where the Hippofile is located
42
+ #
43
+ def root
44
+ File.dirname(@hippofile_path)
45
+ end
46
+
47
+ # Return the repository that this manifest should be working with
48
+ #
49
+ # @return [Hippo::Repository]
50
+ def repository
51
+ return unless @hash['repository']
52
+
53
+ @repository ||= Repository.new(@hash['repository'])
54
+ end
55
+
56
+ # Return the app name
57
+ #
58
+ # @return [String, nil]
59
+ def name
60
+ @hash['name']
61
+ end
62
+
63
+ # Return kubernetes configuration
64
+ #
65
+ # @return [Hippo::Kubernetes]
66
+ def kubernetes
67
+ @kubernetes ||= Kubernetes.new(self, @hash['kubernetes'] || {})
68
+ end
69
+
70
+ # Return the stages for this recipe
71
+ #
72
+ # @return [Hash<Hippo::Stage>]
73
+ def stages
74
+ @stages ||= begin
75
+ yamls = load_yaml_from_directory(File.join(root, 'stages'))
76
+ yamls.each_with_object({}) do |yaml, hash|
77
+ stage = Stage.new(yaml)
78
+ hash[stage.name] = stage
79
+ end
80
+ end
81
+ end
82
+
83
+ # Return the builds for this recipe
84
+ #
85
+ # @return [Hash<Hippo::BuildSpec>]
86
+ def build_specs
87
+ @build_specs ||= @hash['builds'].each_with_object({}) do |(key, options), hash|
88
+ hash[key] = BuildSpec.new(self, key, options)
89
+ end
90
+ end
91
+
92
+ # Return configuration
93
+ #
94
+ # @return []
95
+ def console
96
+ @hash['console']
97
+ end
98
+
99
+ # Return the template variables that should be exposed
100
+ #
101
+ # @return [Hash]
102
+ def template_vars
103
+ {
104
+ 'repository' => repository ? repository.template_vars : nil,
105
+ 'builds' => build_specs.each_with_object({}) { |(_, bs), h| h[bs.name] = bs.template_vars }
106
+ }
107
+ end
108
+
109
+ # Parse a string through the template parser
110
+ #
111
+ # @param string [String]
112
+ # @return [String]
113
+ def parse(stage, commit, string)
114
+ template = Liquid::Template.parse(string)
115
+ template_variables = template_vars
116
+ template_variables['stage'] = stage.template_vars
117
+ if commit
118
+ template_variables['commit'] = {
119
+ 'ref' => commit.objectish,
120
+ 'message' => commit.message
121
+ }
122
+ end
123
+ template.render(template_variables)
124
+ end
125
+ end
126
+ end