cyclid 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +174 -0
  3. data/README.md +54 -0
  4. data/app/cyclid.rb +61 -0
  5. data/app/cyclid/config.rb +38 -0
  6. data/app/cyclid/controllers.rb +123 -0
  7. data/app/cyclid/controllers/auth.rb +34 -0
  8. data/app/cyclid/controllers/auth/token.rb +78 -0
  9. data/app/cyclid/controllers/health.rb +96 -0
  10. data/app/cyclid/controllers/organizations.rb +104 -0
  11. data/app/cyclid/controllers/organizations/collection.rb +134 -0
  12. data/app/cyclid/controllers/organizations/config.rb +128 -0
  13. data/app/cyclid/controllers/organizations/document.rb +135 -0
  14. data/app/cyclid/controllers/organizations/job.rb +266 -0
  15. data/app/cyclid/controllers/organizations/members.rb +145 -0
  16. data/app/cyclid/controllers/organizations/stages.rb +251 -0
  17. data/app/cyclid/controllers/users.rb +47 -0
  18. data/app/cyclid/controllers/users/collection.rb +131 -0
  19. data/app/cyclid/controllers/users/document.rb +133 -0
  20. data/app/cyclid/health_helpers.rb +40 -0
  21. data/app/cyclid/job.rb +3 -0
  22. data/app/cyclid/job/helpers.rb +67 -0
  23. data/app/cyclid/job/job.rb +164 -0
  24. data/app/cyclid/job/runner.rb +275 -0
  25. data/app/cyclid/job/stage.rb +67 -0
  26. data/app/cyclid/log_buffer.rb +104 -0
  27. data/app/cyclid/models.rb +3 -0
  28. data/app/cyclid/models/job_record.rb +25 -0
  29. data/app/cyclid/models/organization.rb +64 -0
  30. data/app/cyclid/models/plugin_config.rb +25 -0
  31. data/app/cyclid/models/stage.rb +42 -0
  32. data/app/cyclid/models/step.rb +29 -0
  33. data/app/cyclid/models/user.rb +60 -0
  34. data/app/cyclid/models/user_permissions.rb +28 -0
  35. data/app/cyclid/monkey_patches.rb +37 -0
  36. data/app/cyclid/plugin_registry.rb +75 -0
  37. data/app/cyclid/plugins.rb +125 -0
  38. data/app/cyclid/plugins/action.rb +48 -0
  39. data/app/cyclid/plugins/action/command.rb +89 -0
  40. data/app/cyclid/plugins/action/email.rb +207 -0
  41. data/app/cyclid/plugins/action/email/html.erb +58 -0
  42. data/app/cyclid/plugins/action/email/text.erb +13 -0
  43. data/app/cyclid/plugins/action/script.rb +90 -0
  44. data/app/cyclid/plugins/action/slack.rb +129 -0
  45. data/app/cyclid/plugins/action/slack/note.erb +5 -0
  46. data/app/cyclid/plugins/api.rb +195 -0
  47. data/app/cyclid/plugins/api/github.rb +111 -0
  48. data/app/cyclid/plugins/api/github/callback.rb +66 -0
  49. data/app/cyclid/plugins/api/github/methods.rb +201 -0
  50. data/app/cyclid/plugins/api/github/status.rb +67 -0
  51. data/app/cyclid/plugins/builder.rb +80 -0
  52. data/app/cyclid/plugins/builder/mist.rb +107 -0
  53. data/app/cyclid/plugins/dispatcher.rb +89 -0
  54. data/app/cyclid/plugins/dispatcher/local.rb +167 -0
  55. data/app/cyclid/plugins/provisioner.rb +40 -0
  56. data/app/cyclid/plugins/provisioner/debian.rb +90 -0
  57. data/app/cyclid/plugins/provisioner/ubuntu.rb +98 -0
  58. data/app/cyclid/plugins/source.rb +39 -0
  59. data/app/cyclid/plugins/source/git.rb +64 -0
  60. data/app/cyclid/plugins/transport.rb +63 -0
  61. data/app/cyclid/plugins/transport/ssh.rb +155 -0
  62. data/app/cyclid/sinatra/api_helpers.rb +66 -0
  63. data/app/cyclid/sinatra/auth_helpers.rb +127 -0
  64. data/app/cyclid/sinatra/warden/strategies/api_token.rb +62 -0
  65. data/app/cyclid/sinatra/warden/strategies/basic.rb +58 -0
  66. data/app/cyclid/sinatra/warden/strategies/hmac.rb +76 -0
  67. data/app/db.rb +51 -0
  68. data/bin/cyclid-db-init +107 -0
  69. data/db/schema.rb +92 -0
  70. data/lib/cyclid/app.rb +4 -0
  71. metadata +407 -0
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Top level module for the core Cyclid code.
17
+ module Cyclid
18
+ # Module for the Cyclid API
19
+ module API
20
+ # Module for Cyclid Plugins
21
+ module Plugins
22
+ # Ubuntu provisioner
23
+ class Ubuntu < Provisioner
24
+ # Prepare an Ubuntu based build host
25
+ def prepare(transport, buildhost, env = {})
26
+ transport.export_env('DEBIAN_FRONTEND' => 'noninteractive')
27
+
28
+ if env.key? :repos
29
+ env[:repos].each do |repo|
30
+ next unless repo.key? :url
31
+
32
+ # Check if it's a PPA, or a traditional repository
33
+ url = repo[:url]
34
+ match = url.match(/\A(ppa|http|https):.*\Z/)
35
+ next unless match
36
+
37
+ case match[1]
38
+ when 'ppa'
39
+ add_ppa_repository(transport, url)
40
+ when 'http', 'https'
41
+ add_http_repository(transport, url, repo, buildhost)
42
+ end
43
+ end
44
+
45
+ success = transport.exec 'sudo apt-get update'
46
+ raise 'failed to update repositories' unless success
47
+ end
48
+
49
+ env[:packages].each do |package|
50
+ success = transport.exec \
51
+ "sudo -E apt-get install -y #{package}"
52
+ raise "failed to install package #{package}" unless success
53
+ end if env.key? :packages
54
+ rescue StandardError => ex
55
+ Cyclid.logger.error "failed to provision #{buildhost[:name]}: #{ex}"
56
+ raise
57
+ end
58
+
59
+ private
60
+
61
+ def add_ppa_repository(transport, url)
62
+ success = transport.exec "sudo apt-add-repository -y #{url}"
63
+ raise "failed to add repository #{url}" unless success
64
+ end
65
+
66
+ def add_http_repository(transport, url, repo, buildhost)
67
+ raise 'an HTTP repository must provide a list of components' \
68
+ unless repo.key? :components
69
+
70
+ # Create a sources.list.d fragment
71
+ release = buildhost[:release]
72
+ components = repo[:components]
73
+ fragment = "deb #{url} #{release} #{components}"
74
+
75
+ success = transport.exec \
76
+ "echo '#{fragment}' | sudo tee -a /etc/apt/sources.list.d/cyclid.list"
77
+ raise "failed to add repository #{url}" unless success
78
+
79
+ if repo.key? :key_id
80
+ # Import the signing key
81
+ key_id = repo[:key_id]
82
+
83
+ success = transport.exec \
84
+ "gpg --keyserver keyserver.ubuntu.com --recv-keys #{key_id}"
85
+ raise "failed to import key #{key_id}" unless success
86
+
87
+ success = transport.exec \
88
+ "gpg -a --export #{key_id} | sudo apt-key add -"
89
+ raise "failed to add repository key #{key_id}" unless success
90
+ end
91
+ end
92
+
93
+ # Register this plugin
94
+ register_plugin 'ubuntu'
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Top level module for the core Cyclid code.
17
+ module Cyclid
18
+ # Module for the Cyclid API
19
+ module API
20
+ # Module for Cyclid Plugins
21
+ module Plugins
22
+ # Base class for Source plugins
23
+ class Source < Base
24
+ # Return the 'human' name for the plugin type
25
+ def self.human_name
26
+ 'source'
27
+ end
28
+
29
+ # Process the source to produce a copy of the remote code in a
30
+ # directory in the working directory
31
+ def checkout(_transport, _ctx, _source = {})
32
+ false
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+
39
+ require_rel 'source/*.rb'
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Top level module for the core Cyclid code.
17
+ module Cyclid
18
+ # Module for the Cyclid API
19
+ module API
20
+ # Module for Cyclid Plugins
21
+ module Plugins
22
+ # Git source plugin
23
+ class Git < Source
24
+ # Run commands via. the transport to check out a given Git remote
25
+ # repository
26
+ def checkout(transport, ctx, source = {})
27
+ source.symbolize_keys!
28
+
29
+ raise 'invalid git source definition' \
30
+ unless source.key? :url
31
+
32
+ # Add any context data (which could include secrets)
33
+ source = source.interpolate(ctx)
34
+
35
+ url = URI(source[:url])
36
+
37
+ # If the source includes an OAuth token, add it to the URL as the
38
+ # username
39
+ url.user = source[:token] if source.key? :token
40
+
41
+ success = transport.exec "git clone #{url}"
42
+ return false unless success
43
+
44
+ # If a branch was given, check it out
45
+ if source.key? :branch
46
+ branch = source[:branch]
47
+
48
+ match = url.path.match(%r{^.*\/(\w*)})
49
+ source_dir = "#{ctx[:workspace]}/#{match[1]}"
50
+
51
+ success = transport.exec("git fetch origin #{branch}:#{branch}", source_dir)
52
+ success = transport.exec("git checkout #{branch}", source_dir) \
53
+ unless success == false
54
+ end
55
+
56
+ return success
57
+ end
58
+
59
+ # Register this plugin
60
+ register_plugin 'git'
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Top level module for the core Cyclid code.
17
+ module Cyclid
18
+ # Module for the Cyclid API
19
+ module API
20
+ # Module for Cyclid Plugins
21
+ module Plugins
22
+ # Base class for Transport plugins
23
+ class Transport < Base
24
+ def initialize(args = {})
25
+ end
26
+
27
+ # Return the 'human' name for the plugin type
28
+ def self.human_name
29
+ 'transport'
30
+ end
31
+
32
+ # If possible, export each of the variables in env as a shell
33
+ # environment variables. The default is simply to remember the
34
+ # environment variables, which will be exported each time when a
35
+ # command is run.
36
+ def export_env(env = {})
37
+ @env = env
38
+ end
39
+
40
+ # Run a command on the remote host.
41
+ def exec(_cmd, _path = nil)
42
+ false
43
+ end
44
+
45
+ # Copy data from a local IO object to a remote file.
46
+ def upload(_io, _path)
47
+ false
48
+ end
49
+
50
+ # Copy a data from remote file to a local IO object
51
+ def download(_io, _path)
52
+ false
53
+ end
54
+
55
+ # Disconnect the transport
56
+ def close
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+
63
+ require_rel 'transport/*.rb'
@@ -0,0 +1,155 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require 'net/ssh'
17
+ require 'net/scp'
18
+ require 'cyclid/log_buffer'
19
+
20
+ # Top level module for the core Cyclid code.
21
+ module Cyclid
22
+ # Module for the Cyclid API
23
+ module API
24
+ # Module for Cyclid Plugins
25
+ module Plugins
26
+ # SSH based transport
27
+ class Ssh < Transport
28
+ attr_reader :exit_code, :exit_signal
29
+
30
+ def initialize(args = {})
31
+ args.symbolize_keys!
32
+
33
+ # Hostname, username & a log target are required
34
+ return false unless args.include? :host and \
35
+ args.include? :user and \
36
+ args.include? :log
37
+
38
+ password = args[:password] if args.include? :password
39
+ keys = [args[:key]] if args.include? :key
40
+
41
+ @log = args[:log]
42
+
43
+ # Create an SSH channel
44
+ Cyclid.logger.debug 'waiting for SSH...'
45
+
46
+ start = Time.now
47
+ loop do
48
+ begin
49
+ @session = Net::SSH.start(args[:host],
50
+ args[:user],
51
+ password: password,
52
+ keys: keys,
53
+ timeout: 5)
54
+ break unless @session.nil?
55
+ rescue Net::SSH::AuthenticationFailed
56
+ Cyclid.logger.debug 'SSH authentication failed'
57
+ end
58
+
59
+ sleep 5
60
+
61
+ raise 'timed out waiting for SSH' \
62
+ if (Time.now - start) >= 30
63
+ end
64
+ end
65
+
66
+ # Execute a command via. SSH
67
+ def exec(cmd, path = nil)
68
+ command = build_command(cmd, path, @env)
69
+ Cyclid.logger.debug "command=#{command}"
70
+
71
+ @session.open_channel do |channel|
72
+ channel.on_open_failed do |_ch, _code, desc|
73
+ # XXX raise
74
+ abort "failed to open channel: #{desc}"
75
+ end
76
+
77
+ # STDOUT
78
+ channel.on_data do |_ch, data|
79
+ # Send to Log Buffer
80
+ @log.write data
81
+ end
82
+
83
+ # STDERR
84
+ channel.on_extended_data do |_ch, _type, data|
85
+ # Send to Log Buffer
86
+ @log.write data
87
+ end
88
+
89
+ # Capture return value from commands
90
+ channel.on_request 'exit-status' do |_ch, data|
91
+ @exit_code = data.read_long
92
+ end
93
+
94
+ # Capture a command exiting with a signal
95
+ channel.on_request 'exit-signal' do |_ch, data|
96
+ @exit_signal = data.read_long
97
+ end
98
+
99
+ channel.exec command do |_ch, _success|
100
+ end
101
+ end
102
+
103
+ # Run the SSH even loop; this blocks until the command has completed
104
+ @session.loop
105
+
106
+ @exit_code.zero? && @exit_signal.nil? ? true : false
107
+ end
108
+
109
+ # Copy data from a local IO object to a remote file via. SCP
110
+ def upload(io, path)
111
+ channel = @session.scp.upload io, path
112
+ channel.wait
113
+ end
114
+
115
+ # Copy a data from remote file to a local IO object
116
+ def download(io, path)
117
+ channel = @session.scp.download path, io
118
+ channel.wait
119
+ end
120
+
121
+ # Close the SSH connection
122
+ def close
123
+ logout
124
+
125
+ @session.close
126
+ end
127
+
128
+ private
129
+
130
+ def logout
131
+ exec 'exit'
132
+ end
133
+
134
+ def build_command(cmd, path = nil, env = {})
135
+ command = []
136
+ if env
137
+ vars = env.map do |k, value|
138
+ key = k.upcase
139
+ key.gsub!(/\s/, '_')
140
+ "export #{key}=\"#{value}\""
141
+ end
142
+ command << vars.join(';')
143
+ end
144
+
145
+ command << "cd #{path}" if path
146
+ command << cmd
147
+ command.join(';')
148
+ end
149
+
150
+ # Register this plugin
151
+ register_plugin 'ssh'
152
+ end
153
+ end
154
+ end
155
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+ # Copyright 2016 Liqwyd Ltd.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require 'oj'
17
+ require 'yaml'
18
+
19
+ # Top level module for the core Cyclid code.
20
+ module Cyclid
21
+ # Module for the Cyclid API
22
+ module API
23
+ # Sinatra helpers
24
+ module APIHelpers
25
+ # Safely parse & validate the request body
26
+ def parse_request_body
27
+ # Parse the the request
28
+ begin
29
+ request.body.rewind
30
+
31
+ if request.content_type == 'application/json' or \
32
+ request.content_type == 'text/json'
33
+
34
+ data = Oj.load request.body.read
35
+ elsif request.content_type == 'application/x-yaml' or \
36
+ request.content_type == 'text/x-yaml'
37
+
38
+ data = YAML.load request.body.read
39
+ else
40
+ halt_with_json_response(415, \
41
+ Errors::HTTPErrors::INVALID_JSON, \
42
+ "unsupported content type #{request.content_type}")
43
+ end
44
+ rescue Oj::ParseError, YAML::Exception => ex
45
+ Cyclid.logger.debug ex.message
46
+ halt_with_json_response(400, Errors::HTTPErrors::INVALID_JSON, ex.message)
47
+ end
48
+
49
+ # Sanity check the request
50
+ halt_with_json_response(400, \
51
+ Errors::HTTPErrors::INVALID_JSON, \
52
+ 'request body can not be empty') if data.nil?
53
+ halt_with_json_response(400, \
54
+ Errors::HTTPErrors::INVALID_JSON, \
55
+ 'request body is invalid') unless data.is_a?(Hash)
56
+
57
+ return data
58
+ end
59
+
60
+ # Return a RESTful JSON response
61
+ def json_response(id, description)
62
+ Oj.dump('id' => id, 'description' => description)
63
+ end
64
+ end
65
+ end
66
+ end