cyclid 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +174 -0
  3. data/README.md +54 -0
  4. data/app/cyclid.rb +61 -0
  5. data/app/cyclid/config.rb +38 -0
  6. data/app/cyclid/controllers.rb +123 -0
  7. data/app/cyclid/controllers/auth.rb +34 -0
  8. data/app/cyclid/controllers/auth/token.rb +78 -0
  9. data/app/cyclid/controllers/health.rb +96 -0
  10. data/app/cyclid/controllers/organizations.rb +104 -0
  11. data/app/cyclid/controllers/organizations/collection.rb +134 -0
  12. data/app/cyclid/controllers/organizations/config.rb +128 -0
  13. data/app/cyclid/controllers/organizations/document.rb +135 -0
  14. data/app/cyclid/controllers/organizations/job.rb +266 -0
  15. data/app/cyclid/controllers/organizations/members.rb +145 -0
  16. data/app/cyclid/controllers/organizations/stages.rb +251 -0
  17. data/app/cyclid/controllers/users.rb +47 -0
  18. data/app/cyclid/controllers/users/collection.rb +131 -0
  19. data/app/cyclid/controllers/users/document.rb +133 -0
  20. data/app/cyclid/health_helpers.rb +40 -0
  21. data/app/cyclid/job.rb +3 -0
  22. data/app/cyclid/job/helpers.rb +67 -0
  23. data/app/cyclid/job/job.rb +164 -0
  24. data/app/cyclid/job/runner.rb +275 -0
  25. data/app/cyclid/job/stage.rb +67 -0
  26. data/app/cyclid/log_buffer.rb +104 -0
  27. data/app/cyclid/models.rb +3 -0
  28. data/app/cyclid/models/job_record.rb +25 -0
  29. data/app/cyclid/models/organization.rb +64 -0
  30. data/app/cyclid/models/plugin_config.rb +25 -0
  31. data/app/cyclid/models/stage.rb +42 -0
  32. data/app/cyclid/models/step.rb +29 -0
  33. data/app/cyclid/models/user.rb +60 -0
  34. data/app/cyclid/models/user_permissions.rb +28 -0
  35. data/app/cyclid/monkey_patches.rb +37 -0
  36. data/app/cyclid/plugin_registry.rb +75 -0
  37. data/app/cyclid/plugins.rb +125 -0
  38. data/app/cyclid/plugins/action.rb +48 -0
  39. data/app/cyclid/plugins/action/command.rb +89 -0
  40. data/app/cyclid/plugins/action/email.rb +207 -0
  41. data/app/cyclid/plugins/action/email/html.erb +58 -0
  42. data/app/cyclid/plugins/action/email/text.erb +13 -0
  43. data/app/cyclid/plugins/action/script.rb +90 -0
  44. data/app/cyclid/plugins/action/slack.rb +129 -0
  45. data/app/cyclid/plugins/action/slack/note.erb +5 -0
  46. data/app/cyclid/plugins/api.rb +195 -0
  47. data/app/cyclid/plugins/api/github.rb +111 -0
  48. data/app/cyclid/plugins/api/github/callback.rb +66 -0
  49. data/app/cyclid/plugins/api/github/methods.rb +201 -0
  50. data/app/cyclid/plugins/api/github/status.rb +67 -0
  51. data/app/cyclid/plugins/builder.rb +80 -0
  52. data/app/cyclid/plugins/builder/mist.rb +107 -0
  53. data/app/cyclid/plugins/dispatcher.rb +89 -0
  54. data/app/cyclid/plugins/dispatcher/local.rb +167 -0
  55. data/app/cyclid/plugins/provisioner.rb +40 -0
  56. data/app/cyclid/plugins/provisioner/debian.rb +90 -0
  57. data/app/cyclid/plugins/provisioner/ubuntu.rb +98 -0
  58. data/app/cyclid/plugins/source.rb +39 -0
  59. data/app/cyclid/plugins/source/git.rb +64 -0
  60. data/app/cyclid/plugins/transport.rb +63 -0
  61. data/app/cyclid/plugins/transport/ssh.rb +155 -0
  62. data/app/cyclid/sinatra/api_helpers.rb +66 -0
  63. data/app/cyclid/sinatra/auth_helpers.rb +127 -0
  64. data/app/cyclid/sinatra/warden/strategies/api_token.rb +62 -0
  65. data/app/cyclid/sinatra/warden/strategies/basic.rb +58 -0
  66. data/app/cyclid/sinatra/warden/strategies/hmac.rb +76 -0
  67. data/app/db.rb +51 -0
  68. data/bin/cyclid-db-init +107 -0
  69. data/db/schema.rb +92 -0
  70. data/lib/cyclid/app.rb +4 -0
  71. metadata +407 -0
@@ -0,0 +1,111 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require_rel 'github/methods'
17
+ require_rel 'github/status'
18
+ require_rel 'github/callback'
19
+
20
+ # Top level module for the core Cyclid code.
21
+ module Cyclid
22
+ # Module for the Cyclid API
23
+ module API
24
+ # Module for Cyclid Plugins
25
+ module Plugins
26
+ # API extension for Github hooks
27
+ class Github < Api
28
+ # Return an instance of the Github API controller
29
+ def self.controller
30
+ return ApiExtension::Controller.new(ApiExtension::GithubMethods)
31
+ end
32
+
33
+ class << self
34
+ # Merge the given config into the current config & validate
35
+ def update_config(config, new)
36
+ Cyclid.logger.debug "config=#{config} new=#{new}"
37
+
38
+ if new.key? 'repository_tokens'
39
+ Cyclid.logger.debug 'updating repository tokens'
40
+
41
+ new_tokens = new['repository_tokens']
42
+ current_tokens = config['repository_tokens']
43
+
44
+ raise 'repository_tokens must be an array' \
45
+ unless new_tokens.is_a? Array
46
+
47
+ # Merge the current list of tokens with the new list of tokens;
48
+ # we have to do this in a 'roundabout fashion:
49
+ #
50
+ # 1. Convert both into a hash, with the url as the key and
51
+ # the original hash itself as the value. E.g.
52
+ # {url: 'example.com', token: 'abcdef'} becomes
53
+ # {'exmaple.com': {url: 'example.com', token: 'abcdef'}}
54
+ # 2. Merge the new hash into the current hash; this will
55
+ # over-write any existing entries.
56
+ # 3. Obtain the values of the the resulting merged object, which
57
+ # is an array of the original hashes.
58
+ #
59
+ # Thanks, Stackoverflow!
60
+ new_hash = Hash[new_tokens.map{ |h| [h['url'], h] }]
61
+ current_hash = Hash[current_tokens.map{ |h| [h['url'], h] }]
62
+
63
+ merged = current_hash.merge(new_hash).values
64
+
65
+ # Delete any entries where the token value is nil
66
+ merged.delete_if do |entry|
67
+ entry['token'].nil?
68
+ end
69
+
70
+ config['repository_tokens'] = merged
71
+ end
72
+
73
+ if new.key? 'hmac_secret'
74
+ Cyclid.logger.debug 'updating HMAC secret'
75
+ config['hmac_secret'] = new['hmac_secret']
76
+ end
77
+
78
+ return config
79
+ end
80
+
81
+ # Default configuration
82
+ def default_config
83
+ config = {}
84
+ config['repository_tokens'] = []
85
+ config['hmac_secret'] = nil
86
+
87
+ return config
88
+ end
89
+
90
+ # Github plugin configuration schema
91
+ def config_schema
92
+ schema = []
93
+ schema << { name: 'repository_tokens',
94
+ type: 'hash-list',
95
+ description: 'Repository OAuth tokens',
96
+ default: [] }
97
+ schema << { name: 'hmac_secret',
98
+ type: 'string',
99
+ description: 'Github HMAC signing secret',
100
+ default: nil }
101
+
102
+ return schema
103
+ end
104
+ end
105
+
106
+ # Register this plugin
107
+ register_plugin 'github'
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Top level module for the core Cyclid code.
17
+ module Cyclid
18
+ # Module for the Cyclid API
19
+ module API
20
+ # Module for Cyclid Plugins
21
+ module Plugins
22
+ # Container for the Sinatra related controllers modules
23
+ module ApiExtension
24
+ # Notifier callback for Github. Updates the external Github Pull
25
+ # Request status as the job progresses.
26
+ class GithubCallback < Plugins::Notifier::Callback
27
+ def initialize(statuses, auth_token)
28
+ @statuses = statuses
29
+ @auth_token = auth_token
30
+ end
31
+
32
+ # Job status has changed
33
+ def status_changed(job_id, status)
34
+ case status
35
+ when Constants::JobStatus::WAITING
36
+ state = 'pending'
37
+ message = "Queued job ##{job_id}."
38
+ when Constants::JobStatus::STARTED
39
+ state = 'pending'
40
+ message = "Job ##{job_id} started."
41
+ when Constants::JobStatus::FAILING
42
+ state = 'failure'
43
+ message = "Job ##{job_id} failed. Waiting for job to complete."
44
+ else
45
+ return false
46
+ end
47
+
48
+ GithubStatus.set_status(@statuses, @auth_token, state, message)
49
+ end
50
+
51
+ # Job has completed
52
+ def completion(job_id, status)
53
+ if status == true
54
+ state = 'success'
55
+ message = "Job ##{job_id} completed successfuly."
56
+ else
57
+ state = 'failure'
58
+ message = "Job ##{job_id} failed."
59
+ end
60
+ GithubStatus.set_status(@statuses, @auth_token, state, message)
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,201 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Top level module for the core Cyclid code.
17
+ module Cyclid
18
+ # Module for the Cyclid API
19
+ module API
20
+ # Module for Cyclid Plugins
21
+ module Plugins
22
+ # Container for the Sinatra related controllers modules
23
+ module ApiExtension
24
+ # Github plugin method callbacks
25
+ module GithubMethods
26
+ include Methods
27
+
28
+ # Return a reference to the plugin that is associated with this
29
+ # controller; used by the lower level code.
30
+ def controller_plugin
31
+ Cyclid.plugins.find('github', Cyclid::API::Plugins::Api)
32
+ end
33
+
34
+ # HTTP POST callback
35
+ def post(data, headers, config)
36
+ return_failure(400, 'no event specified') \
37
+ unless headers.include? 'X-Github-Event'
38
+
39
+ return_failure(400, 'no delivery ID specified') \
40
+ unless headers.include? 'X-Github-Delivery'
41
+
42
+ event = headers['X-Github-Event']
43
+ # Not used yet but will be when we add HMAC support
44
+ # signature = headers['X-Hub-Signature'] || nil
45
+
46
+ Cyclid.logger.debug "Github: event is #{event}"
47
+
48
+ case event
49
+ when 'pull_request'
50
+ result = gh_pull_request(data, config)
51
+ when 'ping'
52
+ result = true
53
+ when 'status'
54
+ result = true
55
+ else
56
+ return_failure(400, "event type '#{event}' is not supported")
57
+ end
58
+
59
+ return result
60
+ end
61
+
62
+ # Handle a Github Pull Request event
63
+ def gh_pull_request(data, config)
64
+ action = data['action'] || nil
65
+ pr = data['pull_request'] || nil
66
+
67
+ Cyclid.logger.debug "action=#{action}"
68
+ return true unless action == 'opened' \
69
+ or action == 'reopened' \
70
+ or action == 'synchronize'
71
+
72
+ # Get the list of files in the root of the repository in the
73
+ # Pull Request branch
74
+ html_url = URI(pr['base']['repo']['html_url'])
75
+ pr_sha = pr['head']['sha']
76
+ ref = pr['head']['ref']
77
+
78
+ Cyclid.logger.debug "sha=#{pr_sha} ref=#{ref}"
79
+
80
+ # Get some useful endpoints & interpolate the SHA for this PR
81
+ url = pr['head']['repo']['statuses_url']
82
+ statuses = url.gsub('{sha}', pr_sha)
83
+
84
+ url = pr['head']['repo']['trees_url']
85
+ trees = url.gsub('{/sha}', "/#{pr_sha}")
86
+
87
+ # Get an OAuth token, if one is set for this repo
88
+ Cyclid.logger.debug "attempting to find auth token for #{html_url}"
89
+ auth_token = nil
90
+ config['repository_tokens'].each do |entry|
91
+ entry_url = URI(entry['url'])
92
+ auth_token = entry['token'] if entry_url.host == html_url.host && \
93
+ entry_url.path == html_url.path
94
+ end
95
+
96
+ # XXX We probably don't want to be logging auth tokens in plain text
97
+ Cyclid.logger.debug "auth token=#{auth_token}"
98
+
99
+ # Set the PR to 'pending'
100
+ GithubStatus.set_status(statuses, auth_token, 'pending', 'Preparing build')
101
+
102
+ # Get the Pull Request
103
+ begin
104
+ trees_url = URI(trees)
105
+ Cyclid.logger.debug "Getting root for #{trees_url}"
106
+
107
+ request = Net::HTTP::Get.new(trees_url)
108
+ request.add_field('Authorization', "token #{auth_token}") \
109
+ unless auth_token.nil?
110
+
111
+ http = Net::HTTP.new(trees_url.hostname, trees_url.port)
112
+ http.use_ssl = (trees_url.scheme == 'https')
113
+ response = http.request(request)
114
+
115
+ Cyclid.logger.debug response.inspect
116
+ raise "couldn't get repository root" \
117
+ unless response.code == '200'
118
+
119
+ root = Oj.load response.body
120
+ rescue StandardError => ex
121
+ Cyclid.logger.error "failed to retrieve Pull Request root: #{ex}"
122
+ return_failure(500, 'could not retrieve Pull Request root')
123
+ end
124
+
125
+ # See if a .cyclid.yml or .cyclid.json file exists in the project
126
+ # root
127
+ job_url = nil
128
+ job_type = nil
129
+ root['tree'].each do |file|
130
+ match = file['path'].match(/\A\.cyclid\.(json|yml)\z/)
131
+ next unless match
132
+
133
+ job_url = URI(file['url'])
134
+ job_type = match[1]
135
+ end
136
+
137
+ Cyclid.logger.debug "job_url=#{job_url}"
138
+
139
+ if job_url.nil?
140
+ GithubStatus.set_status(statuses, auth_token, 'error', 'No Cyclid job file found')
141
+ return_failure(400, 'not a Cyclid repository')
142
+ end
143
+
144
+ # Pull down the job file
145
+ begin
146
+ Cyclid.logger.info "Retrieving PR job from #{job_url}"
147
+
148
+ request = Net::HTTP::Get.new(job_url)
149
+ request.add_field('Authorization', "token #{auth_token}") \
150
+ unless auth_token.nil?
151
+
152
+ http = Net::HTTP.new(job_url.hostname, job_url.port)
153
+ http.use_ssl = (job_url.scheme == 'https')
154
+ response = http.request(request)
155
+ raise "couldn't get Cyclid job" unless response.code == '200'
156
+
157
+ job_blob = Oj.load response.body
158
+ case job_type
159
+ when 'json'
160
+ job_definition = Oj.load(Base64.decode64(job_blob['content']))
161
+ when 'yml'
162
+ job_definition = YAML.load(Base64.decode64(job_blob['content']))
163
+ end
164
+
165
+ # Insert this repository & branch into the sources
166
+ #
167
+ # XXX Could this cause collisions between the existing sources in
168
+ # the job definition? Not entirely sure what the workflow will
169
+ # look like.
170
+ job_sources = job_definition['sources'] || []
171
+ job_sources << { 'type' => 'git',
172
+ 'url' => html_url.to_s,
173
+ 'branch' => ref,
174
+ 'token' => auth_token }
175
+ job_definition['sources'] = job_sources
176
+
177
+ Cyclid.logger.debug "sources=#{job_definition['sources']}"
178
+ rescue StandardError => ex
179
+ GithubStatus.set_status(statuses,
180
+ auth_token,
181
+ 'error',
182
+ "Couldn't retrieve Cyclid job file")
183
+ Cyclid.logger.error "failed to retrieve Github Pull Request job: #{ex}"
184
+ raise
185
+ end
186
+
187
+ Cyclid.logger.debug "job_definition=#{job_definition}"
188
+
189
+ begin
190
+ callback = GithubCallback.new(statuses, auth_token)
191
+ job_from_definition(job_definition, callback)
192
+ rescue StandardError => ex
193
+ GithubStatus.set_status(statuses, auth_token, 'failure', ex)
194
+ return_failure(500, 'job failed')
195
+ end
196
+ end
197
+ end
198
+ end
199
+ end
200
+ end
201
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require 'net/http'
17
+
18
+ # Top level module for the core Cyclid code.
19
+ module Cyclid
20
+ # Module for the Cyclid API
21
+ module API
22
+ # Module for Cyclid Plugins
23
+ module Plugins
24
+ # Container for the Sinatra related controllers modules
25
+ module ApiExtension
26
+ # Wrapper for a static method to push a status update to Github
27
+ module GithubStatus
28
+ # Call the Github statuses API to update the status
29
+ def self.set_status(statuses, auth_token, state, description)
30
+ # Update the PR status
31
+
32
+ statuses_url = URI(statuses)
33
+ status = { state: state,
34
+ target_url: 'http://cyclid.io',
35
+ description: description,
36
+ context: 'continuous-integration/cyclid' }
37
+
38
+ # Post the status to the statuses endpoint
39
+ request = Net::HTTP::Post.new(statuses_url)
40
+ request.content_type = 'application/json'
41
+ request.add_field 'Authorization', "token #{auth_token}" \
42
+ unless auth_token.nil?
43
+ request.body = status.to_json
44
+
45
+ http = Net::HTTP.new(statuses_url.hostname, statuses_url.port)
46
+ http.use_ssl = (statuses_url.scheme == 'https')
47
+ response = http.request(request)
48
+
49
+ case response
50
+ when Net::HTTPSuccess, Net::HTTPRedirection
51
+ Cyclid.logger.info "updated PR status to #{state}"
52
+ when Net::HTTPNotFound
53
+ Cyclid.logger.error 'update PR status failed; possibly an auth failure'
54
+ raise
55
+ else
56
+ Cyclid.logger.error "update PR status failed: #{response}"
57
+ raise
58
+ end
59
+ rescue StandardError => ex
60
+ Cyclid.logger.error "couldn't set status for PR: #{ex}"
61
+ raise
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Top level module for the core Cyclid code.
17
+ module Cyclid
18
+ # Module for the Cyclid API
19
+ module API
20
+ # Module for Cyclid Plugins
21
+ module Plugins
22
+ # Base class for BuildHost
23
+ class BuildHost < Hash
24
+ def initialize(args)
25
+ args.each do |key, value|
26
+ self[key.to_sym] = value
27
+ end
28
+ end
29
+
30
+ # Return the information needed (hostname/IP, username, password & key
31
+ # if there are any) to create a Transport to this host, in a normalized form.
32
+ def connect_info
33
+ [self[:host], self[:username], self[:password], self[:key]]
34
+ end
35
+
36
+ # Return a list of acceptable Transports that can be used to connect to this
37
+ # host.
38
+ def transports
39
+ # XXX Maybe create some constants for "well known" Transports such as 'ssh'
40
+ []
41
+ end
42
+
43
+ # Return free-form data about this host that may be useful to the build
44
+ # process and can be merged into the Job context. This may be a subset of the
45
+ # data for this BuildHost, or the full set.
46
+ def context_info
47
+ dup
48
+ end
49
+ end
50
+
51
+ # Base class for Builders
52
+ class Builder < Base
53
+ # Create a build host, probably on a remote system, and return information
54
+ # about it in a BuildHost object that encapsulates the information about it.
55
+ def initialize(*args)
56
+ end
57
+
58
+ # Return the 'human' name for the plugin type
59
+ def self.human_name
60
+ 'builder'
61
+ end
62
+
63
+ # Get or create a build host that can be used by a job. Args will be things
64
+ # like the OS & version required, taken from the 'environment' section of the
65
+ # job definition.
66
+ #
67
+ # The Builder can call out to external service E.g. AWS, DO, RAX etc. or
68
+ # return an existing instance from a pool
69
+ def get(*args)
70
+ end
71
+
72
+ # Shut down/release/destroy (if appropriate) the build host
73
+ def release(_transport, _buildhost)
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
79
+
80
+ require_rel 'builder/*.rb'