satorix 0.0.1 → 1.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +4 -1
  3. data/.gitlab-ci.yml +45 -0
  4. data/.rspec +2 -1
  5. data/.rubocop.yml +11 -0
  6. data/.ruby-version +1 -0
  7. data/Gemfile +2 -0
  8. data/Gemfile.lock +25 -0
  9. data/Procfile +1 -0
  10. data/README.md +93 -1
  11. data/Rakefile +8 -4
  12. data/bin/console +3 -3
  13. data/bin/satorix +8 -0
  14. data/lib/satorix/CI/deploy/flynn/environment_variables.rb +123 -0
  15. data/lib/satorix/CI/deploy/flynn/resources.rb +79 -0
  16. data/lib/satorix/CI/deploy/flynn/routes.rb +267 -0
  17. data/lib/satorix/CI/deploy/flynn/scale.rb +52 -0
  18. data/lib/satorix/CI/deploy/flynn.rb +132 -0
  19. data/lib/satorix/CI/shared/buildpack_manager/buildpack.rb +159 -0
  20. data/lib/satorix/CI/shared/buildpack_manager.rb +220 -0
  21. data/lib/satorix/CI/shared/ruby/gem_manager.rb +80 -0
  22. data/lib/satorix/CI/shared/yarn_manager.rb +25 -0
  23. data/lib/satorix/CI/test/python/django_test.rb +38 -0
  24. data/lib/satorix/CI/test/python/safety.rb +30 -0
  25. data/lib/satorix/CI/test/ruby/brakeman.rb +35 -0
  26. data/lib/satorix/CI/test/ruby/bundler_audit.rb +35 -0
  27. data/lib/satorix/CI/test/ruby/cucumber.rb +29 -0
  28. data/lib/satorix/CI/test/ruby/rails_test.rb +29 -0
  29. data/lib/satorix/CI/test/ruby/rspec.rb +29 -0
  30. data/lib/satorix/CI/test/ruby/rubocop.rb +98 -0
  31. data/lib/satorix/CI/test/shared/database.rb +74 -0
  32. data/lib/satorix/shared/console.rb +157 -0
  33. data/lib/satorix/version.rb +1 -1
  34. data/lib/satorix.rb +343 -2
  35. data/satorix/CI/deploy/ie_gem_server.rb +80 -0
  36. data/satorix/CI/deploy/rubygems.rb +81 -0
  37. data/satorix/custom.rb +21 -0
  38. data/satorix.gemspec +13 -11
  39. metadata +57 -29
  40. data/.travis.yml +0 -5
@@ -0,0 +1,74 @@
1
+ module Satorix
2
+ module CI
3
+ module Test
4
+ module Shared
5
+ module Database
6
+
7
+
8
+ include Satorix::Shared::Console
9
+
10
+
11
+ extend self
12
+
13
+
14
+ def host
15
+ ENV['DB_HOST'].to_s
16
+ end
17
+
18
+
19
+ def name
20
+ name_key.empty? ? '' : ENV[name_key].to_s
21
+ end
22
+
23
+
24
+ def name_key
25
+ {
26
+ mariadb: 'MYSQL_DATABASE',
27
+ mysql: 'MYSQL_DATABASE',
28
+ postgres: 'POSTGRES_DB'
29
+ }[host.to_sym].to_s
30
+ end
31
+
32
+
33
+ def password
34
+ password_key.empty? ? '' : ENV[password_key].to_s
35
+ end
36
+
37
+
38
+ def password_key
39
+ {
40
+ mariadb: 'MYSQL_ROOT_PASSWORD',
41
+ mysql: 'MYSQL_ROOT_PASSWORD',
42
+ postgres: 'POSTGRES_PASSWORD'
43
+ }[host.to_sym].to_s
44
+ end
45
+
46
+
47
+ def url
48
+ unset = [host, user, password, name].select(&:empty?)
49
+ if unset.empty?
50
+ "#{ host }://#{ user }:#{ password }@#{ host }/#{ name }"
51
+ else
52
+ log 'No database has been configured.'
53
+ log 'If you would like to use a database for this job, ensure that your gitlab-config.yml is configured.'
54
+ log 'Most databases require you to specify a type, host, database, username, and password.'
55
+ nil
56
+ end
57
+ end
58
+
59
+
60
+ def user
61
+ (ENV[user_key] || 'root').to_s
62
+ end
63
+
64
+
65
+ def user_key
66
+ "#{ host.upcase }_USER"
67
+ end
68
+
69
+
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,157 @@
1
+ module Satorix
2
+ module Shared
3
+ module Console
4
+
5
+
6
+ require 'benchmark'
7
+ require 'English' # http://ruby-doc.org/stdlib-2.0.0/libdoc/English/rdoc/English.html
8
+ require 'shellwords'
9
+
10
+
11
+ extend self
12
+
13
+
14
+ def colors
15
+ Hash.new(39).merge(red: '0;31', light_red: '1;31',
16
+ green: '0;32', light_green: '1;32',
17
+ yellow: '0;33', light_yellow: '1;33',
18
+ blue: '0;34', light_blue: '1;34',
19
+ magenta: '0;35', light_magenta: '1;35',
20
+ cyan: '0;36', light_cyan: '1;36',
21
+ white: '0;37', light_white: '1;37')
22
+ end
23
+
24
+
25
+ def colorize(text, color = nil)
26
+ "\033[#{ colors[color] }m#{ text }\033[0m"
27
+ end
28
+
29
+
30
+ def log(text, color = nil)
31
+ puts color ? colorize(text, color) : text
32
+ end
33
+
34
+
35
+ def log_command(text)
36
+ log text, :cyan
37
+ end
38
+
39
+
40
+ def log_duration(text)
41
+ log text, :light_cyan
42
+ end
43
+
44
+
45
+ def log_header(text)
46
+ log("\n#{ text }", :light_green)
47
+ end
48
+
49
+
50
+ def log_error(text)
51
+ log text, :light_red
52
+ end
53
+
54
+
55
+ def log_error_and_abort(text)
56
+ log_error text
57
+ abort text
58
+ end
59
+
60
+
61
+ def log_bench(message)
62
+ log_header message
63
+ result = nil
64
+ log_duration "Time elapsed: #{ humanize_time Benchmark.realtime { result = yield } }."
65
+ result
66
+ end
67
+
68
+
69
+ def humanize_time(seconds)
70
+ return 'less than 1 second' if seconds < 1
71
+ [[60, :second], [60, :minute], [24, :hour], [1000, :day]].map do |count, name|
72
+ if seconds > 0
73
+ seconds, n = seconds.divmod(count)
74
+ n = n.to_i
75
+ "#{ n } #{ name }#{ 's' if n > 1 }" if n > 0
76
+ end
77
+ end.compact.reverse.join(' ')
78
+ end
79
+
80
+
81
+ # This method is used *EVERYWHERE*.
82
+ # Seemingly small changes can have far-ranging and extreme consequences.
83
+ # Modify only with extreme caution.
84
+ #
85
+ # Note: Many applications (like echo and the Flynn CLI) append a newline to their output.
86
+ # If you are using the command result, it may be desirable to chomp the trailing
87
+ # newline. It is left to the implementing call to handle this, as it proved
88
+ # cumbersome to handle in this method in a way that worked perfectly in all cases.
89
+ def run_command(command, filtered_text: [], quiet: false)
90
+ command = command.shelljoin if command.is_a?(Array)
91
+ logged_command = logged_command(command, filtered_text)
92
+ log_command(logged_command) unless quiet
93
+
94
+ ''.tap do |output|
95
+ IO.popen(bash(command)) do |io|
96
+ until io.eof?
97
+ line = io.gets
98
+ puts line unless quiet || line.empty?
99
+ $stdout.flush
100
+ output.concat line
101
+ end
102
+ end
103
+
104
+ handle_run_command_error(logged_command, quiet) unless $CHILD_STATUS.success?
105
+ end
106
+ end
107
+
108
+
109
+ # http://stackoverflow.com/questions/1197224/source-shell-script-into-environment-within-a-ruby-script
110
+ # TODO : reduce / consolidate?
111
+ # Find variables changed as a result of sourcing the given file, and update in ENV.
112
+ def source_env_from(file)
113
+ bash_source(file).each { |k, v| ENV[k] = v }
114
+ end
115
+
116
+
117
+ private ##########################################################################################################
118
+
119
+
120
+ def logged_command(command, filtered_text)
121
+ filtered_text = [filtered_text].flatten.delete_if { |text| text.nil? || text.strip.empty? }
122
+ command.gsub(Regexp.union(filtered_text), '[MASKED]')
123
+ end
124
+
125
+
126
+ def bash(command)
127
+ "bash -c #{ command.shellescape }"
128
+ end
129
+
130
+
131
+ # http://stackoverflow.com/questions/1197224/source-shell-script-into-environment-within-a-ruby-script
132
+ # TODO : reduce / consolidate?
133
+ # Read in the bash environment, after an optional command. Returns Array of key/value pairs.
134
+ def bash_env(cmd = nil)
135
+ env_cmd = bash("#{ cmd + ';' if cmd } printenv")
136
+ env = `#{ env_cmd }`
137
+ env.split(/\n/).map { |l| l.split(/=/, 2) }
138
+ end
139
+
140
+
141
+ # http://stackoverflow.com/questions/1197224/source-shell-script-into-environment-within-a-ruby-script
142
+ # TODO : reduce / consolidate?
143
+ # Source a given file, and compare environment before and after. Returns Hash of any keys that have changed.
144
+ def bash_source(file)
145
+ Hash[bash_env("source #{ File.realpath file }") - bash_env]
146
+ end
147
+
148
+
149
+ def handle_run_command_error(logged_command, quiet)
150
+ error_message = "\nAn error has occurred while running the following command:\n#{ logged_command }\n"
151
+ quiet ? abort : log_error_and_abort(error_message)
152
+ end
153
+
154
+
155
+ end
156
+ end
157
+ end
@@ -1,3 +1,3 @@
1
1
  module Satorix
2
- VERSION = '0.0.1'
2
+ VERSION = '1.5.3'.freeze
3
3
  end
data/lib/satorix.rb CHANGED
@@ -1,5 +1,346 @@
1
- require "satorix/version"
1
+ require 'airbrake-ruby'
2
+ require 'English' # http://ruby-doc.org/stdlib-2.0.0/libdoc/English/rdoc/English.html
3
+ require 'satorix/version'
4
+ require 'etc'
2
5
 
3
6
  module Satorix
4
- # Your code goes here...
7
+
8
+
9
+ module Shared
10
+ autoload :Console, 'satorix/shared/console'
11
+ end
12
+
13
+
14
+ module CI
15
+
16
+ module Deploy
17
+ autoload :Flynn, 'satorix/CI/deploy/flynn'
18
+ end
19
+
20
+ module Shared
21
+ autoload :BuildpackManager, 'satorix/CI/shared/buildpack_manager'
22
+ module BuildpackManager
23
+ autoload :Buildpack, 'satorix/CI/shared/buildpack_manager/buildpack'
24
+ end
25
+ autoload :YarnManager, 'satorix/CI/shared/yarn_manager'
26
+
27
+ module Ruby
28
+ autoload :GemManager, 'satorix/CI/shared/ruby/gem_manager'
29
+ end
30
+ end
31
+
32
+ module Test
33
+ module Python
34
+ autoload :DjangoTest, 'satorix/CI/test/python/django_test'
35
+ autoload :Safety, 'satorix/CI/test/python/safety'
36
+ end
37
+
38
+ module Ruby
39
+ autoload :Brakeman, 'satorix/CI/test/ruby/brakeman'
40
+ autoload :BundlerAudit, 'satorix/CI/test/ruby/bundler_audit'
41
+ autoload :Cucumber, 'satorix/CI/test/ruby/cucumber'
42
+ autoload :RailsTest, 'satorix/CI/test/ruby/rails_test'
43
+ autoload :Rspec, 'satorix/CI/test/ruby/rspec'
44
+ autoload :Rubocop, 'satorix/CI/test/ruby/rubocop'
45
+ end
46
+
47
+ module Shared
48
+ autoload :Database, 'satorix/CI/test/shared/database'
49
+ end
50
+ end
51
+
52
+ end
53
+
54
+
55
+ include Satorix::Shared::Console
56
+
57
+
58
+ extend self
59
+
60
+
61
+ def go
62
+ airbrake_start
63
+ prepare_app_environment
64
+
65
+ begin
66
+ log_header "Executing #{ ci_job_name } script for #{ ci_commit_ref_name }..."
67
+
68
+ execute_as_user 'satorix' do
69
+ Dir.chdir(build_dir)
70
+ unless skip_buildpack?
71
+ Satorix::CI::Shared::BuildpackManager.go
72
+ Dir.chdir(app_dir)
73
+ end
74
+ Satorix::CI::Shared::Ruby::GemManager.go if ruby_gem?
75
+ Satorix::CI::Shared::YarnManager.go if yarn?
76
+
77
+ job_class.go
78
+ end
79
+
80
+ log_header "\nDone executing #{ ci_job_name } script for #{ ci_commit_ref_name }.\n"
81
+ rescue Exception => e
82
+ # TODO: add link to issue tracker
83
+ log 'If you feel this failure was in error, please check the issue tracker for more information.'
84
+ Airbrake.notify(e) if airbrake_configured?
85
+ raise e
86
+ end
87
+ end
88
+
89
+
90
+ def add_user(username)
91
+ unless user_exists?(username)
92
+ run_command ['useradd', '--user-group', '--comment', "'#{ username } user'", '--shell', '/bin/bash', '--home', app_dir, username], quiet: true
93
+ end
94
+ end
95
+
96
+
97
+ def user_exists?(username)
98
+ Etc.getpwnam(username)
99
+ rescue
100
+ end
101
+
102
+
103
+ def airbrake_configured?
104
+ [airbrake_project_id, airbrake_project_key].all? { |var| !var.empty? }
105
+ end
106
+
107
+
108
+ def airbrake_project_id
109
+ ENV['SATORIX_CI_AIRBRAKE_PROJECT_ID'].to_s
110
+ end
111
+
112
+
113
+ def airbrake_project_key
114
+ ENV['SATORIX_CI_AIRBRAKE_PROJECT_KEY'].to_s
115
+ end
116
+
117
+
118
+ def airbrake_start
119
+ return unless airbrake_configured?
120
+
121
+ Airbrake.configure do |c|
122
+ c.project_id = airbrake_project_id
123
+ c.project_key = airbrake_project_key
124
+ end
125
+
126
+ at_exit do
127
+ Airbrake.notify_sync($ERROR_INFO) if $ERROR_INFO # https://github.com/airbrake/airbrake-ruby#reporting-critical-exceptions
128
+ Airbrake.close # https://github.com/airbrake/airbrake-ruby#airbrakeclose
129
+ end
130
+ end
131
+
132
+
133
+ def custom_loader_relative_path
134
+ File.join 'satorix', 'custom.rb'
135
+ end
136
+
137
+
138
+ def custom_loader_full_path
139
+ File.join build_dir, custom_loader_relative_path
140
+ end
141
+
142
+
143
+ def load_custom
144
+ log_bench "Looking for custom job definitions in #{ custom_loader_relative_path }....." do
145
+ if File.exist? custom_loader_full_path
146
+ log "Loading custom job definitions from #{ custom_loader_relative_path }."
147
+ require custom_loader_full_path
148
+ Satorix::Custom.available_jobs
149
+ else
150
+ # TODO : create this documentation and link to it in the message below
151
+ log 'No custom jobs found.'
152
+ log "You can define custom jobs by adding a #{ custom_loader_relative_path } file to your app root."
153
+ log 'For more information, please refer to https://www.satorix.com/docs/articles/custom_satorix_jobs.'
154
+ {}
155
+ end
156
+ end
157
+ end
158
+
159
+
160
+ def default_jobs
161
+ { deploy: { deploy_with_flynn: Satorix::CI::Deploy::Flynn },
162
+ test: { brakeman: Satorix::CI::Test::Ruby::Brakeman,
163
+ bundler_audit: Satorix::CI::Test::Ruby::BundlerAudit,
164
+ cucumber: Satorix::CI::Test::Ruby::Cucumber,
165
+ django_test: Satorix::CI::Test::Python::DjangoTest,
166
+ rails_test: Satorix::CI::Test::Ruby::RailsTest,
167
+ rspec: Satorix::CI::Test::Ruby::Rspec,
168
+ rubocop: Satorix::CI::Test::Ruby::Rubocop,
169
+ safety: Satorix::CI::Test::Python::Safety } }
170
+ end
171
+
172
+
173
+ def available_jobs
174
+ @_available_jobs ||= load_custom.tap do |jobs|
175
+ default_jobs.each do |stage, job_definitions|
176
+ jobs[stage] ||= {}
177
+ job_definitions.each { |job, target| jobs[stage][job] = target }
178
+ end
179
+ end
180
+ end
181
+
182
+
183
+ def ci_job_name
184
+ ENV['CI_JOB_NAME']
185
+ end
186
+
187
+
188
+ def ci_job_stage
189
+ ENV['CI_JOB_STAGE']
190
+ end
191
+
192
+
193
+ def ci_commit_ref_name
194
+ ENV['CI_COMMIT_REF_NAME']
195
+ end
196
+
197
+
198
+ def ci_commit_sha
199
+ ENV['CI_COMMIT_SHA']
200
+ end
201
+
202
+
203
+ def current_branch
204
+ @_current_branch ||= ci_commit_ref_name.to_s.upcase
205
+ end
206
+
207
+
208
+ def job_class
209
+ available_jobs[ci_job_stage.to_sym] && available_jobs[ci_job_stage.to_sym][ci_job_name.to_sym] || begin
210
+ log_error_and_abort "The #{ ci_job_name } job does not exist for the #{ ci_job_stage } stage!"
211
+ end
212
+ end
213
+
214
+
215
+ # https://stackoverflow.com/questions/4548151/run-ruby-block-as-specific-os-user/
216
+ def execute_as_user(user, &block)
217
+ u = (user.is_a? Integer) ? Etc.getpwuid(user) : Etc.getpwnam(user)
218
+
219
+ ENV['HOME'] = Satorix.app_dir
220
+
221
+ reader, writer = IO.pipe
222
+
223
+ Process.fork do
224
+ # the child process won't need to read from the pipe
225
+ reader.close
226
+
227
+ # use primary group ID of target user
228
+ # This needs to be done first as we won't have
229
+ # permission to change our group after changing EUID
230
+ Process.gid = Process.egid = u.gid
231
+
232
+ # set real and effective UIDs to target user
233
+ Process.uid = Process.euid = u.uid
234
+
235
+ # get the result and write it to the IPC pipe
236
+ result = block.call(user)
237
+ Marshal.dump(result, writer)
238
+ writer.close
239
+
240
+ # prevent shutdown hooks from running
241
+ Process.exit!(true)
242
+ end
243
+
244
+ # back to reality... we won't be writing anything
245
+ writer.close
246
+
247
+ # block until there's data to read
248
+ result = Marshal.load(reader)
249
+
250
+ # done with that!
251
+ reader.close
252
+
253
+ # return block result
254
+ result
255
+ rescue EOFError
256
+ log_error 'This job has failed.'
257
+ abort
258
+ end
259
+
260
+
261
+ def paths
262
+ ci_project_dir = ENV['CI_PROJECT_DIR'].to_s
263
+ { buildpacks: '/tmp/buildpacks',
264
+ app_dir: '/tmp/app',
265
+ build_dir: ci_project_dir,
266
+ cache: File.join(ci_project_dir, 'tmp', 'satorix', 'cache'),
267
+ env: '/tmp/env' }
268
+ end
269
+
270
+
271
+ def app_dir
272
+ paths[:app_dir]
273
+ end
274
+
275
+
276
+ def bin_dir
277
+ File.join app_dir, 'bin'
278
+ end
279
+
280
+
281
+ def build_dir
282
+ paths[:build_dir]
283
+ end
284
+
285
+
286
+ def prepare_app_environment
287
+ log_bench 'Preparing app environment...' do
288
+ add_user('satorix')
289
+ setup_directories
290
+ end
291
+ end
292
+
293
+
294
+ def project_name
295
+ ENV['CI_PROJECT_NAME']
296
+ end
297
+
298
+
299
+ def setup_directories
300
+ paths.values.each { |path| FileUtils.mkdir_p path }
301
+ paths.values.each { |path| FileUtils.chown_R 'satorix', 'satorix', path }
302
+ FileUtils.ln_s app_dir, '/app'
303
+ end
304
+
305
+
306
+ def skip_buildpack?
307
+ job_class.respond_to?(:skip_buildpack) && job_class.skip_buildpack
308
+ end
309
+
310
+
311
+ def yarn?
312
+ File.exist? yarn_lock_file
313
+ end
314
+
315
+
316
+ def yarn_lock_file
317
+ File.join(app_dir, 'yarn.lock')
318
+ end
319
+
320
+
321
+ def rails_app?
322
+ # This used to follow the Heroku buildpack, with:
323
+ # File.exist?(File.join(build_dir, 'Gemfile'))
324
+ # It has been adjusted to accommodate non-rails apps that use ruby gems.
325
+ # The config/application.rb file seems to be present in all rails applications.
326
+ # It may be useful to add additional checks, for increased precision, in the future.
327
+ File.exist?(File.join(build_dir, 'config', 'application.rb'))
328
+ end
329
+
330
+
331
+ def django_app?
332
+ # Satorix requires django apps be in a public directory.
333
+ # The manage.py file seems to be present in all django applications.
334
+ # It may be useful to add additional checks, for increased precision, in the future.
335
+ File.exist?(File.join(build_dir, 'public', 'manage.py'))
336
+ end
337
+
338
+
339
+ def ruby_gem?
340
+ # Checks for the presence of a *.gemspec file in the project root.
341
+ # It may be useful to add additional checks, for increased precision, in the future.
342
+ Dir[File.join(build_dir, '*.gemspec')].any?
343
+ end
344
+
345
+
5
346
  end
@@ -0,0 +1,80 @@
1
+ module Satorix
2
+ module CI
3
+ module Deploy
4
+ module IeGemServer
5
+
6
+
7
+ require 'fileutils'
8
+
9
+
10
+ include Satorix::Shared::Console
11
+
12
+
13
+ extend self
14
+
15
+
16
+ def go
17
+ log_bench('Installing the geminabox gem...') { install_geminabox_gem }
18
+ log_bench('Preparing gem build directory...') { prepare_gem_build_directory }
19
+ log_bench('Building gem...') { build_gem }
20
+ built_gems.each { |gem| log_bench("Publishing #{ File.basename gem }...") { publish_gem gem } }
21
+ end
22
+
23
+
24
+ def build_gem
25
+ run_command 'rake build'
26
+ end
27
+
28
+
29
+ def built_gems
30
+ Dir.glob(File.join(gem_build_directory, '*.gem')).select { |e| File.file? e }
31
+ end
32
+
33
+
34
+ def gem_build_directory
35
+ File.join Satorix.app_dir, 'pkg'
36
+ end
37
+
38
+
39
+ def ie_gem_server_host
40
+ "https://#{ ie_gem_server_user_name }:#{ ie_gem_server_password }@gems.iexposure.com"
41
+ end
42
+
43
+
44
+ def install_geminabox_gem
45
+ run_command "gem install geminabox --source https://gems.iexposure.com --no-document --bindir #{ Satorix.bin_dir }"
46
+ end
47
+
48
+
49
+ def prepare_gem_build_directory
50
+ run_command "rm -rf #{ gem_build_directory }", quiet: true
51
+ FileUtils.mkdir_p gem_build_directory
52
+ end
53
+
54
+
55
+ def publish_gem(gem)
56
+ run_command "gem inabox #{ gem } --host #{ ie_gem_server_host }",
57
+ filtered_text: [ie_gem_server_user_name, ie_gem_server_password]
58
+ rescue RuntimeError
59
+ # To prevent the display of an ugly stacktrace.
60
+ abort "\nGem was not published!"
61
+ end
62
+
63
+
64
+ private ########################################################################################################
65
+
66
+
67
+ def ie_gem_server_password
68
+ ENV['IE_GEM_SERVER_PASSWORD']
69
+ end
70
+
71
+
72
+ def ie_gem_server_user_name
73
+ ENV['IE_GEM_SERVER_USER_NAME']
74
+ end
75
+
76
+
77
+ end
78
+ end
79
+ end
80
+ end