cyclid 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +174 -0
- data/README.md +54 -0
- data/app/cyclid.rb +61 -0
- data/app/cyclid/config.rb +38 -0
- data/app/cyclid/controllers.rb +123 -0
- data/app/cyclid/controllers/auth.rb +34 -0
- data/app/cyclid/controllers/auth/token.rb +78 -0
- data/app/cyclid/controllers/health.rb +96 -0
- data/app/cyclid/controllers/organizations.rb +104 -0
- data/app/cyclid/controllers/organizations/collection.rb +134 -0
- data/app/cyclid/controllers/organizations/config.rb +128 -0
- data/app/cyclid/controllers/organizations/document.rb +135 -0
- data/app/cyclid/controllers/organizations/job.rb +266 -0
- data/app/cyclid/controllers/organizations/members.rb +145 -0
- data/app/cyclid/controllers/organizations/stages.rb +251 -0
- data/app/cyclid/controllers/users.rb +47 -0
- data/app/cyclid/controllers/users/collection.rb +131 -0
- data/app/cyclid/controllers/users/document.rb +133 -0
- data/app/cyclid/health_helpers.rb +40 -0
- data/app/cyclid/job.rb +3 -0
- data/app/cyclid/job/helpers.rb +67 -0
- data/app/cyclid/job/job.rb +164 -0
- data/app/cyclid/job/runner.rb +275 -0
- data/app/cyclid/job/stage.rb +67 -0
- data/app/cyclid/log_buffer.rb +104 -0
- data/app/cyclid/models.rb +3 -0
- data/app/cyclid/models/job_record.rb +25 -0
- data/app/cyclid/models/organization.rb +64 -0
- data/app/cyclid/models/plugin_config.rb +25 -0
- data/app/cyclid/models/stage.rb +42 -0
- data/app/cyclid/models/step.rb +29 -0
- data/app/cyclid/models/user.rb +60 -0
- data/app/cyclid/models/user_permissions.rb +28 -0
- data/app/cyclid/monkey_patches.rb +37 -0
- data/app/cyclid/plugin_registry.rb +75 -0
- data/app/cyclid/plugins.rb +125 -0
- data/app/cyclid/plugins/action.rb +48 -0
- data/app/cyclid/plugins/action/command.rb +89 -0
- data/app/cyclid/plugins/action/email.rb +207 -0
- data/app/cyclid/plugins/action/email/html.erb +58 -0
- data/app/cyclid/plugins/action/email/text.erb +13 -0
- data/app/cyclid/plugins/action/script.rb +90 -0
- data/app/cyclid/plugins/action/slack.rb +129 -0
- data/app/cyclid/plugins/action/slack/note.erb +5 -0
- data/app/cyclid/plugins/api.rb +195 -0
- data/app/cyclid/plugins/api/github.rb +111 -0
- data/app/cyclid/plugins/api/github/callback.rb +66 -0
- data/app/cyclid/plugins/api/github/methods.rb +201 -0
- data/app/cyclid/plugins/api/github/status.rb +67 -0
- data/app/cyclid/plugins/builder.rb +80 -0
- data/app/cyclid/plugins/builder/mist.rb +107 -0
- data/app/cyclid/plugins/dispatcher.rb +89 -0
- data/app/cyclid/plugins/dispatcher/local.rb +167 -0
- data/app/cyclid/plugins/provisioner.rb +40 -0
- data/app/cyclid/plugins/provisioner/debian.rb +90 -0
- data/app/cyclid/plugins/provisioner/ubuntu.rb +98 -0
- data/app/cyclid/plugins/source.rb +39 -0
- data/app/cyclid/plugins/source/git.rb +64 -0
- data/app/cyclid/plugins/transport.rb +63 -0
- data/app/cyclid/plugins/transport/ssh.rb +155 -0
- data/app/cyclid/sinatra/api_helpers.rb +66 -0
- data/app/cyclid/sinatra/auth_helpers.rb +127 -0
- data/app/cyclid/sinatra/warden/strategies/api_token.rb +62 -0
- data/app/cyclid/sinatra/warden/strategies/basic.rb +58 -0
- data/app/cyclid/sinatra/warden/strategies/hmac.rb +76 -0
- data/app/db.rb +51 -0
- data/bin/cyclid-db-init +107 -0
- data/db/schema.rb +92 -0
- data/lib/cyclid/app.rb +4 -0
- metadata +407 -0
@@ -0,0 +1,133 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Copyright 2016 Liqwyd Ltd.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
# Top level module for all of the core Cyclid code.
|
17
|
+
module Cyclid
|
18
|
+
# Module for the Cyclid API
|
19
|
+
module API
|
20
|
+
# Module for all User related API endpoints
|
21
|
+
module Users
|
22
|
+
# API endpoints for a single Organization document
|
23
|
+
# @api REST
|
24
|
+
module Document
|
25
|
+
# @!group Users
|
26
|
+
|
27
|
+
# @!method get_users_user
|
28
|
+
# @overload GET /users/:username
|
29
|
+
# @macro rest
|
30
|
+
# @param [String] username Username of the user.
|
31
|
+
# Get a specific user.
|
32
|
+
# @return The requested user.
|
33
|
+
# @return [404] The user does not exist
|
34
|
+
|
35
|
+
# @!method put_users_user(body)
|
36
|
+
# @overload PUT /users/:username
|
37
|
+
# @macro rest
|
38
|
+
# @param [String] username Username of the user.
|
39
|
+
# Modify a specific user.
|
40
|
+
# @param [JSON] body User information
|
41
|
+
# @option body [String] name Users real name
|
42
|
+
# @option body [String] email Users new email address
|
43
|
+
# @option body [String] password New Bcrypt2 encrypted password
|
44
|
+
# @option body [String] new_password New password in plain text, which will be
|
45
|
+
# encrypted before being stored in the databaase.
|
46
|
+
# @option body [String] secret New HMAC signing secret. This should be a suitably
|
47
|
+
# long random string.
|
48
|
+
# @return [200] User was modified successfully
|
49
|
+
# @return [400] The user definition is invalid
|
50
|
+
# @return [404] The user does not exist
|
51
|
+
|
52
|
+
# @!method delete_users_user
|
53
|
+
# @overload DELETE /users/:username
|
54
|
+
# @macro rest
|
55
|
+
# @param [String] username Username of the user.
|
56
|
+
# Delete a specific user.
|
57
|
+
# @return [200] User was deleted successfully
|
58
|
+
# @return [404] The user does not exist
|
59
|
+
|
60
|
+
# @!endgroup
|
61
|
+
|
62
|
+
# Sinatra callback
|
63
|
+
# @private
|
64
|
+
def self.registered(app)
|
65
|
+
include Errors::HTTPErrors
|
66
|
+
|
67
|
+
# Get a specific user.
|
68
|
+
app.get do
|
69
|
+
authorized_as!(params[:username], Operations::READ)
|
70
|
+
|
71
|
+
user = User.find_by(username: params[:username])
|
72
|
+
halt_with_json_response(404, INVALID_USER, 'user does not exist') \
|
73
|
+
if user.nil?
|
74
|
+
|
75
|
+
Cyclid.logger.debug user.organizations
|
76
|
+
|
77
|
+
# Convert to a Hash and inject the User data
|
78
|
+
user_hash = user.serializable_hash
|
79
|
+
user_hash['organizations'] = user.organizations.map(&:name)
|
80
|
+
|
81
|
+
user_hash = sanitize_user(user_hash)
|
82
|
+
|
83
|
+
return user_hash.to_json
|
84
|
+
end
|
85
|
+
|
86
|
+
# Modify a specific user.
|
87
|
+
app.put do
|
88
|
+
authorized_as!(params[:username], Operations::WRITE)
|
89
|
+
|
90
|
+
payload = parse_request_body
|
91
|
+
Cyclid.logger.debug payload
|
92
|
+
|
93
|
+
user = User.find_by(username: params[:username])
|
94
|
+
halt_with_json_response(404, INVALID_USER, 'user does not exist') \
|
95
|
+
if user.nil?
|
96
|
+
|
97
|
+
begin
|
98
|
+
user.name = payload['name'] if payload.key? 'name'
|
99
|
+
user.email = payload['email'] if payload.key? 'email'
|
100
|
+
user.password = payload['password'] if payload.key? 'password'
|
101
|
+
user.secret = payload['secret'] if payload.key? 'secret'
|
102
|
+
user.new_password = payload['new_password'] if payload.key? 'new_password'
|
103
|
+
user.save!
|
104
|
+
rescue ActiveRecord::ActiveRecordError => ex
|
105
|
+
Cyclid.logger.debug ex.message
|
106
|
+
halt_with_json_response(400, INVALID_JSON, ex.message)
|
107
|
+
end
|
108
|
+
|
109
|
+
return json_response(NO_ERROR, "user #{payload['username']} modified")
|
110
|
+
end
|
111
|
+
|
112
|
+
# Delete a specific user.
|
113
|
+
app.delete do
|
114
|
+
authorized_as!(params[:username], Operations::ADMIN)
|
115
|
+
|
116
|
+
user = User.find_by(username: params[:username])
|
117
|
+
halt_with_json_response(404, INVALID_USER, 'user does not exist') \
|
118
|
+
if user.nil?
|
119
|
+
|
120
|
+
begin
|
121
|
+
user.delete
|
122
|
+
rescue ActiveRecord::ActiveRecordError => ex
|
123
|
+
Cyclid.logger.debug ex.message
|
124
|
+
halt_with_json_response(400, INVALID_JSON, ex.message)
|
125
|
+
end
|
126
|
+
|
127
|
+
return json_response(NO_ERROR, "user #{params['username']} deleted")
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Copyright 2016 Liqwyd Ltd.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
require 'sinatra-health-check'
|
17
|
+
|
18
|
+
# Top level module for all of the core Cyclid code.
|
19
|
+
module Cyclid
|
20
|
+
# Module for the Cyclid API
|
21
|
+
module API
|
22
|
+
module Health
|
23
|
+
# Helper methods to isolate the plugins from the implementation details
|
24
|
+
# of the healthcheck framework
|
25
|
+
module Helpers
|
26
|
+
# Health statuses
|
27
|
+
STATUSES = {
|
28
|
+
ok: SinatraHealthCheck::Status::SEVERITIES[:ok],
|
29
|
+
warning: SinatraHealthCheck::Status::SEVERITIES[:warning],
|
30
|
+
error: SinatraHealthCheck::Status::SEVERITIES[:error]
|
31
|
+
}.freeze
|
32
|
+
|
33
|
+
# Produce a SinatraHealthCheck object from the given status & message
|
34
|
+
def health_status(status, message)
|
35
|
+
SinatraHealthCheck::Status.new(status, message)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/app/cyclid/job.rb
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Copyright 2016 Liqwyd Ltd.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
# Top level module for the core Cyclid code.
|
17
|
+
module Cyclid
|
18
|
+
# Module for the Cyclid API
|
19
|
+
module API
|
20
|
+
# Module for Cyclid Job related classes
|
21
|
+
module Job
|
22
|
+
# Useful methods for dealing with Jobs
|
23
|
+
module Helpers
|
24
|
+
# Create & dispatch a Job from the job definition
|
25
|
+
def job_from_definition(definition, callback = nil, context = {})
|
26
|
+
# This function will only ever be called from a Sinatra context
|
27
|
+
org = Organization.find_by(name: params[:name])
|
28
|
+
halt_with_json_response(404, INVALID_ORG, 'organization does not exist') \
|
29
|
+
if org.nil?
|
30
|
+
|
31
|
+
# Create a new JobRecord
|
32
|
+
job_record = JobRecord.new
|
33
|
+
job_record.started = Time.now.to_s
|
34
|
+
job_record.status = Constants::JobStatus::NEW
|
35
|
+
job_record.save!
|
36
|
+
|
37
|
+
org.job_records << job_record
|
38
|
+
|
39
|
+
# The user may, or may not, be set: if the job has come via. the :organization/jobs
|
40
|
+
# endpoint it'll be set (as that's authenticated), if it's come from an API extension the
|
41
|
+
# user mat not be set (as it may be unauthenticated, or not using the same authentication
|
42
|
+
# as Cyclid)
|
43
|
+
user = current_user
|
44
|
+
current_user.job_records << job_record if user
|
45
|
+
|
46
|
+
begin
|
47
|
+
job = ::Cyclid::API::Job::JobView.new(definition, context, org)
|
48
|
+
Cyclid.logger.debug job.to_hash
|
49
|
+
|
50
|
+
job_id = Cyclid.dispatcher.dispatch(job, job_record, callback)
|
51
|
+
rescue StandardError => ex
|
52
|
+
Cyclid.logger.error "job dispatch failed: #{ex}"
|
53
|
+
|
54
|
+
# We couldn't dispatch the job; record the failure
|
55
|
+
job_record.status = Constants::JobStatus::FAILED
|
56
|
+
job_record.ended = Time.now.to_s
|
57
|
+
job_record.save!
|
58
|
+
|
59
|
+
raise
|
60
|
+
end
|
61
|
+
|
62
|
+
return job_id
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,164 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Copyright 2016 Liqwyd Ltd.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
# Top level module for the core Cyclid code.
|
17
|
+
module Cyclid
|
18
|
+
# Module for the Cyclid API
|
19
|
+
module API
|
20
|
+
# Module for Cyclid Job related classes
|
21
|
+
module Job
|
22
|
+
# Non-ActiveRecord class which holds a complete Job, complete with
|
23
|
+
# serialised stages and the resolved sequence.
|
24
|
+
class JobView
|
25
|
+
attr_reader :name, :version
|
26
|
+
|
27
|
+
def initialize(job, context, org)
|
28
|
+
# Job is a hash (converted from JSON or YAML)
|
29
|
+
job.symbolize_keys!
|
30
|
+
|
31
|
+
@name = job[:name]
|
32
|
+
@version = job[:version] || '1.0.0'
|
33
|
+
|
34
|
+
@context = context
|
35
|
+
@organization = org.name
|
36
|
+
@environment = job[:environment]
|
37
|
+
@sources = job[:sources] || []
|
38
|
+
@secrets = setec_astronomy(org, (job[:secrets] || {}))
|
39
|
+
|
40
|
+
# Build a single unified list of StageViews
|
41
|
+
@stages, @sequence = build_stage_collection(job, org)
|
42
|
+
end
|
43
|
+
|
44
|
+
# Return everything, serialized into a hash
|
45
|
+
def to_hash
|
46
|
+
hash = {}
|
47
|
+
hash[:name] = @name
|
48
|
+
hash[:version] = @version
|
49
|
+
hash[:context] = @context
|
50
|
+
hash[:organization] = @organization
|
51
|
+
hash[:environment] = @environment
|
52
|
+
hash[:sources] = @sources
|
53
|
+
hash[:secrets] = @secrets
|
54
|
+
hash[:stages] = @stages.each_with_object({}) do |(name, stage), h|
|
55
|
+
h[name.to_sym] = Oj.dump(stage)
|
56
|
+
end
|
57
|
+
hash[:sequence] = @sequence
|
58
|
+
|
59
|
+
return hash
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
# Too Many Secrets
|
65
|
+
def setec_astronomy(org, secrets)
|
66
|
+
# Create the RSA private key
|
67
|
+
private_key = OpenSSL::PKey::RSA.new(org.rsa_private_key)
|
68
|
+
|
69
|
+
secrets.hmap do |key, secret|
|
70
|
+
{ key => private_key.private_decrypt(Base64.decode64(secret)) }
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
# Create the ad-hoc StageViews & combine them with the Stages defined
|
75
|
+
# in the Job, returning an array of StageViews & a list of the Stages
|
76
|
+
# in the order to be run.
|
77
|
+
def build_stage_collection(job, org)
|
78
|
+
# Create a JobStage for each ad-hoc stage defined in the job and
|
79
|
+
# add it to the list of stages for this job
|
80
|
+
stages = {}
|
81
|
+
sequence = []
|
82
|
+
begin
|
83
|
+
job[:stages].each do |stage|
|
84
|
+
stage_view = StageView.new(stage)
|
85
|
+
stages[stage_view.name.to_sym] = stage_view
|
86
|
+
end if job.key? :stages
|
87
|
+
rescue StandardError => ex
|
88
|
+
# XXX Probably something wrong with the definition; re-raise it? Or
|
89
|
+
# maybe we get rid of this block and catch it further up (in the
|
90
|
+
# controller?)
|
91
|
+
Cyclid.logger.info "ad-hoc stage creation failed: #{ex}"
|
92
|
+
raise
|
93
|
+
end
|
94
|
+
|
95
|
+
# For each stage in the job, it's either already in the list of
|
96
|
+
# stages because we created on as an ad-hoc stage, or we need to load
|
97
|
+
# it from the database, create a JobStage from it, and add it to the
|
98
|
+
# list of stages
|
99
|
+
job_sequence = job[:sequence]
|
100
|
+
job_sequence.each do |job_stage|
|
101
|
+
job_stage.symbolize_keys!
|
102
|
+
|
103
|
+
raise ArgumentError, 'invalid stage definition' \
|
104
|
+
unless job_stage.key? :stage
|
105
|
+
|
106
|
+
# Store the job in the sequence so that we can run the stages in
|
107
|
+
# the correct order
|
108
|
+
name = job_stage[:stage]
|
109
|
+
sequence << name
|
110
|
+
|
111
|
+
# Try to find the stage
|
112
|
+
if stages.key? name.to_sym
|
113
|
+
# Ad-hoc stage defined in the job
|
114
|
+
stage_view = stages[name.to_sym]
|
115
|
+
else
|
116
|
+
# Try to find a matching pre-defined stage
|
117
|
+
stage = if job_stage.key? :version
|
118
|
+
org.stages.find_by(name: name, version: job_stage[:version])
|
119
|
+
else
|
120
|
+
# If no version given, get the latest
|
121
|
+
org.stages.where(name: name).last
|
122
|
+
end
|
123
|
+
|
124
|
+
raise ArgumentError, "stage #{name}:#{version} not found" \
|
125
|
+
if stage.nil?
|
126
|
+
|
127
|
+
stage_view = StageView.new(stage)
|
128
|
+
end
|
129
|
+
|
130
|
+
# Merge in the options specified in this job stage. If the
|
131
|
+
# on_success or on_failure stages are not already in the sequence,
|
132
|
+
# append them to the end.
|
133
|
+
stage_success = { stage: job_stage[:on_success] }
|
134
|
+
job_sequence << stage_success \
|
135
|
+
unless job_stage[:on_success].nil? or \
|
136
|
+
stage?(job_sequence, job_stage[:on_success])
|
137
|
+
stage_view.on_success = job_stage[:on_success]
|
138
|
+
|
139
|
+
stage_failure = { stage: job_stage[:on_failure] }
|
140
|
+
job_sequence << stage_failure \
|
141
|
+
unless job_stage[:on_failure].nil? or \
|
142
|
+
stage?(job_sequence, job_stage[:on_failure])
|
143
|
+
stage_view.on_failure = job_stage[:on_failure]
|
144
|
+
|
145
|
+
# Store the modified StageView
|
146
|
+
stages[stage_view.name.to_sym] = stage_view
|
147
|
+
end
|
148
|
+
|
149
|
+
return [stages, sequence]
|
150
|
+
end
|
151
|
+
|
152
|
+
# Search for a stage in the sequence, by name
|
153
|
+
def stage?(sequence, name)
|
154
|
+
found = false
|
155
|
+
sequence.each do |stage|
|
156
|
+
found = stage[:stage] == name || stage['stage'] == name
|
157
|
+
break if found
|
158
|
+
end
|
159
|
+
return found
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
@@ -0,0 +1,275 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Copyright 2016 Liqwyd Ltd.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
# Top level module for the core Cyclid code.
|
17
|
+
module Cyclid
|
18
|
+
# Module for the Cyclid API
|
19
|
+
module API
|
20
|
+
# Module for Cyclid Job related classes
|
21
|
+
module Job
|
22
|
+
# Run a job
|
23
|
+
class Runner
|
24
|
+
include Constants::JobStatus
|
25
|
+
|
26
|
+
def initialize(job_id, job_definition, notifier)
|
27
|
+
# The notifier for updating the job status & writing to the log
|
28
|
+
# buffer
|
29
|
+
@notifier = notifier
|
30
|
+
|
31
|
+
# Un-serialize the job
|
32
|
+
begin
|
33
|
+
@job = Oj.load(job_definition, symbol_keys: true)
|
34
|
+
|
35
|
+
environment = @job[:environment]
|
36
|
+
secrets = @job[:secrets]
|
37
|
+
rescue StandardError => ex
|
38
|
+
Cyclid.logger.error "couldn't un-serialize job for job ID #{job_id}: #{ex}"
|
39
|
+
raise 'job failed'
|
40
|
+
end
|
41
|
+
|
42
|
+
# Create an initial job context (more will be added as the job runs)
|
43
|
+
@ctx = @job[:context]
|
44
|
+
|
45
|
+
@ctx[:job_id] = job_id
|
46
|
+
@ctx[:job_name] = @job[:name]
|
47
|
+
@ctx[:job_version] = @job[:version]
|
48
|
+
@ctx[:organization] = @job[:organization]
|
49
|
+
@ctx.merge! environment
|
50
|
+
@ctx.merge! secrets
|
51
|
+
|
52
|
+
begin
|
53
|
+
# We're off!
|
54
|
+
@notifier.status = WAITING
|
55
|
+
|
56
|
+
# Create a Builder
|
57
|
+
@builder = create_builder
|
58
|
+
|
59
|
+
# Obtain a host to run the job on
|
60
|
+
@notifier.write "#{Time.now} : Obtaining build host...\n"
|
61
|
+
@build_host = request_build_host(@builder, environment)
|
62
|
+
|
63
|
+
# We have a build host
|
64
|
+
@notifier.status = STARTED
|
65
|
+
|
66
|
+
# Add some build host details to the build context
|
67
|
+
@ctx.merge! @build_host.context_info
|
68
|
+
|
69
|
+
# Connect a transport to the build host; the notifier is a proxy
|
70
|
+
# to the log buffer
|
71
|
+
@transport = create_transport(@build_host, @notifier)
|
72
|
+
|
73
|
+
# Prepare the host
|
74
|
+
provisioner = create_provisioner(@build_host)
|
75
|
+
|
76
|
+
@notifier.write "#{Time.now} : Preparing build host...\n#{'=' * 79}\n"
|
77
|
+
provisioner.prepare(@transport, @build_host, environment)
|
78
|
+
|
79
|
+
# Check out sources
|
80
|
+
if @job[:sources].any?
|
81
|
+
@notifier.write "#{'=' * 79}\n#{Time.now} : Checking out source...\n"
|
82
|
+
checkout_sources(@transport, @ctx, @job[:sources])
|
83
|
+
end
|
84
|
+
rescue StandardError => ex
|
85
|
+
Cyclid.logger.error "job runner failed: #{ex}"
|
86
|
+
|
87
|
+
@notifier.status = FAILED
|
88
|
+
@notifier.ended = Time.now.to_s
|
89
|
+
|
90
|
+
begin
|
91
|
+
@builder.release(@transport, @build_host) if @build_host
|
92
|
+
@transport.close if @transport
|
93
|
+
rescue ::Net::SSH::Disconnect # rubocop:disable Lint/HandleExceptions
|
94
|
+
# Ignored
|
95
|
+
end
|
96
|
+
|
97
|
+
raise # XXX Raise an internal exception
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# Run the stages.
|
102
|
+
#
|
103
|
+
# Start with the first stage, and execute all of the steps until
|
104
|
+
# either one fails, or there are no more steps. The follow the
|
105
|
+
# on_success & on_failure handlers to the next stage. If no
|
106
|
+
# handler is defined, stop.
|
107
|
+
def run
|
108
|
+
status = STARTED
|
109
|
+
|
110
|
+
@notifier.write "#{'=' * 79}\n#{Time.now} : Job started. " \
|
111
|
+
"Context: #{@ctx.stringify_keys}\n"
|
112
|
+
|
113
|
+
# Run the Job stage actions
|
114
|
+
stages = @job[:stages] || []
|
115
|
+
sequence = (@job[:sequence] || []).first
|
116
|
+
|
117
|
+
# Run each stage in the sequence until there are none left
|
118
|
+
until sequence.nil?
|
119
|
+
# Find the stage
|
120
|
+
raise 'stage not found' unless stages.key? sequence.to_sym
|
121
|
+
|
122
|
+
# Un-serialize the stage into a StageView
|
123
|
+
stage_definition = stages[sequence.to_sym]
|
124
|
+
stage = Oj.load(stage_definition, symbol_keys: true)
|
125
|
+
|
126
|
+
@notifier.write "#{'-' * 79}\n#{Time.now} : " \
|
127
|
+
"Running stage #{stage.name} v#{stage.version}\n"
|
128
|
+
|
129
|
+
# Run the stage
|
130
|
+
success, rc = run_stage(stage)
|
131
|
+
|
132
|
+
Cyclid.logger.info "stage #{(success ? 'succeeded' : 'failed')} and returned #{rc}"
|
133
|
+
|
134
|
+
# Decide which stage to run next depending on the outcome of this
|
135
|
+
# one
|
136
|
+
if success
|
137
|
+
sequence = stage.on_success
|
138
|
+
else
|
139
|
+
sequence = stage.on_failure
|
140
|
+
|
141
|
+
# Remember the failure while the failure handlers run
|
142
|
+
status = FAILING
|
143
|
+
@notifier.status = status
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
# Either all of the stages succeeded, and thus the job suceeded, or
|
148
|
+
# (at least one of) the stages failed, and thus the job failed
|
149
|
+
if status == FAILING
|
150
|
+
@notifier.status = FAILED
|
151
|
+
@notifier.ended = Time.now
|
152
|
+
success = false
|
153
|
+
else
|
154
|
+
@notifier.status = SUCCEEDED
|
155
|
+
@notifier.ended = Time.now
|
156
|
+
success = true
|
157
|
+
end
|
158
|
+
|
159
|
+
# We no longer require the build host & transport
|
160
|
+
begin
|
161
|
+
@builder.release(@transport, @build_host)
|
162
|
+
@transport.close
|
163
|
+
rescue ::Net::SSH::Disconnect # rubocop:disable Lint/HandleExceptions
|
164
|
+
# Ignored
|
165
|
+
end
|
166
|
+
|
167
|
+
return success
|
168
|
+
end
|
169
|
+
|
170
|
+
private
|
171
|
+
|
172
|
+
# Create a suitable Builder
|
173
|
+
def create_builder
|
174
|
+
# Each worker creates a new instance
|
175
|
+
builder = Cyclid.builder.new
|
176
|
+
raise "couldn't create a builder" \
|
177
|
+
unless builder
|
178
|
+
|
179
|
+
return builder
|
180
|
+
end
|
181
|
+
|
182
|
+
# Acquire a build host from the builder
|
183
|
+
def request_build_host(builder, environment)
|
184
|
+
# Request a BuildHost
|
185
|
+
build_host = builder.get(environment)
|
186
|
+
raise "couldn't obtain a build host" unless build_host
|
187
|
+
|
188
|
+
return build_host
|
189
|
+
end
|
190
|
+
|
191
|
+
# Find a transport that can be used with the build host, create one and
|
192
|
+
# connect them together
|
193
|
+
def create_transport(build_host, log_buffer)
|
194
|
+
# Create a Transport & connect it to the build host
|
195
|
+
host, username, password, key = build_host.connect_info
|
196
|
+
Cyclid.logger.debug "create_transport: host: #{host} " \
|
197
|
+
"username: #{username} " \
|
198
|
+
"password: #{password} " \
|
199
|
+
"key: #{key}"
|
200
|
+
|
201
|
+
# Try to match a transport that the host supports, to a transport we know how
|
202
|
+
# to create; transports should be listed in the order they're preferred.
|
203
|
+
transport_plugin = nil
|
204
|
+
build_host.transports.each do |t|
|
205
|
+
transport_plugin = Cyclid.plugins.find(t, Cyclid::API::Plugins::Transport)
|
206
|
+
end
|
207
|
+
|
208
|
+
raise "couldn't find a valid transport from #{build_host.transports}" \
|
209
|
+
unless transport_plugin
|
210
|
+
|
211
|
+
# Connect the transport to the build host
|
212
|
+
transport = transport_plugin.new(host: host,
|
213
|
+
user: username,
|
214
|
+
password: password,
|
215
|
+
key: key,
|
216
|
+
log: log_buffer)
|
217
|
+
raise 'failed to connect the transport' unless transport
|
218
|
+
|
219
|
+
return transport
|
220
|
+
end
|
221
|
+
|
222
|
+
# Find a provisioner that can be used with the build host and create
|
223
|
+
# one
|
224
|
+
def create_provisioner(build_host)
|
225
|
+
distro = build_host[:distro]
|
226
|
+
|
227
|
+
provisioner_plugin = Cyclid.plugins.find(distro, Cyclid::API::Plugins::Provisioner)
|
228
|
+
raise "couldn't find a valid provisioner for #{distro}" \
|
229
|
+
unless provisioner_plugin
|
230
|
+
|
231
|
+
provisioner = provisioner_plugin.new
|
232
|
+
raise 'failed to create provisioner' unless provisioner
|
233
|
+
|
234
|
+
return provisioner
|
235
|
+
end
|
236
|
+
|
237
|
+
# Find and create a suitable source plugin instance for each source and have it check out
|
238
|
+
# the given source using the transport.
|
239
|
+
def checkout_sources(transport, ctx, sources)
|
240
|
+
sources.each do |job_source|
|
241
|
+
raise 'no type given in source definition' unless job_source.key? :type
|
242
|
+
|
243
|
+
source = Cyclid.plugins.find(job_source[:type], Cyclid::API::Plugins::Source)
|
244
|
+
raise "can't find a plugin for #{job_source[:type]} source" if source.nil?
|
245
|
+
|
246
|
+
success = source.new.checkout(transport, ctx, job_source)
|
247
|
+
raise 'failed to check out source' unless success
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
# Perform each action defined in the steps of the given stage, until
|
252
|
+
# either an action fails or we run out of steps
|
253
|
+
def run_stage(stage)
|
254
|
+
stage.steps.each do |step|
|
255
|
+
begin
|
256
|
+
# Un-serialize the Action for this step
|
257
|
+
action = Oj.load(step[:action], symbol_keys: true)
|
258
|
+
rescue StandardError
|
259
|
+
Cyclid.logger.error "couldn't un-serialize action for job ID #{job_id}"
|
260
|
+
raise 'job failed'
|
261
|
+
end
|
262
|
+
|
263
|
+
# Run the action
|
264
|
+
action.prepare(transport: @transport, ctx: @ctx)
|
265
|
+
success, rc = action.perform(@notifier)
|
266
|
+
|
267
|
+
return [false, rc] unless success
|
268
|
+
end
|
269
|
+
|
270
|
+
return [true, 0]
|
271
|
+
end
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|