satorix 0.0.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +4 -1
  3. data/.gitlab-ci.yml +45 -0
  4. data/.rspec +2 -1
  5. data/.rubocop.yml +11 -0
  6. data/.ruby-version +1 -0
  7. data/Gemfile +2 -0
  8. data/Gemfile.lock +25 -0
  9. data/Procfile +1 -0
  10. data/README.md +93 -1
  11. data/Rakefile +8 -4
  12. data/bin/console +3 -3
  13. data/bin/satorix +8 -0
  14. data/lib/satorix.rb +338 -2
  15. data/lib/satorix/CI/deploy/flynn.rb +129 -0
  16. data/lib/satorix/CI/deploy/flynn/environment_variables.rb +121 -0
  17. data/lib/satorix/CI/deploy/flynn/resources.rb +77 -0
  18. data/lib/satorix/CI/deploy/flynn/routes.rb +266 -0
  19. data/lib/satorix/CI/deploy/flynn/scale.rb +52 -0
  20. data/lib/satorix/CI/shared/buildpack_manager.rb +213 -0
  21. data/lib/satorix/CI/shared/buildpack_manager/buildpack.rb +153 -0
  22. data/lib/satorix/CI/shared/ruby/gem_manager.rb +79 -0
  23. data/lib/satorix/CI/shared/yarn_manager.rb +22 -0
  24. data/lib/satorix/CI/test/python/django_test.rb +35 -0
  25. data/lib/satorix/CI/test/python/safety.rb +27 -0
  26. data/lib/satorix/CI/test/ruby/brakeman.rb +32 -0
  27. data/lib/satorix/CI/test/ruby/bundler_audit.rb +32 -0
  28. data/lib/satorix/CI/test/ruby/cucumber.rb +26 -0
  29. data/lib/satorix/CI/test/ruby/rails_test.rb +26 -0
  30. data/lib/satorix/CI/test/ruby/rspec.rb +26 -0
  31. data/lib/satorix/CI/test/ruby/rubocop.rb +98 -0
  32. data/lib/satorix/CI/test/shared/database.rb +74 -0
  33. data/lib/satorix/shared/console.rb +180 -0
  34. data/lib/satorix/version.rb +1 -1
  35. data/satorix.gemspec +13 -11
  36. data/satorix/CI/deploy/ie_gem_server.rb +76 -0
  37. data/satorix/CI/deploy/rubygems.rb +77 -0
  38. data/satorix/custom.rb +21 -0
  39. metadata +57 -29
  40. data/.travis.yml +0 -5
@@ -0,0 +1,52 @@
1
+ module Satorix
2
+ module CI
3
+ module Deploy
4
+ module Flynn
5
+ module Scale
6
+
7
+ def adjust_scale
8
+ cached_scale_options_to_set = scale_options_to_set
9
+ log "No scale specified in #{ defined_scale_key }. Displaying current scale:" if cached_scale_options_to_set.empty?
10
+ fc_scale cached_scale_options_to_set
11
+ end
12
+
13
+
14
+ def current_scale
15
+ scale_string_to_hash fc_scale
16
+ end
17
+
18
+
19
+ def defined_scale
20
+ scale_string_to_hash ENV[defined_scale_key].to_s
21
+ end
22
+
23
+
24
+ def defined_scale_key
25
+ "FLYNN_#{ current_branch }_SCALE"
26
+ end
27
+
28
+
29
+ def fc_scale(scale_options_to_set = nil)
30
+ command = %w[flynn scale].concat(scale_options_to_set)
31
+
32
+ run_command(command)
33
+ end
34
+
35
+
36
+ def scale_options_to_set
37
+ [].tap do |scale|
38
+ defined_scale.each { |job, workers| scale << "#{ job }=#{ workers }" }
39
+ (current_scale.keys - defined_scale.keys).each { |job| scale << "#{ job }=0" } unless defined_scale.empty?
40
+ end
41
+ end
42
+
43
+
44
+ def scale_string_to_hash(scale_string)
45
+ {}.tap { |jobs| scale_string.split.map { |x| x.partition('=') }.each { |x| jobs[x.first.to_sym] = x.last } }
46
+ end
47
+
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,213 @@
1
+ module Satorix
2
+ module CI
3
+ module Shared
4
+
5
+ # This code was inspired by:
6
+ # herokuish - https://github.com/gliderlabs/herokuish
7
+ # flynn - https://github.com/flynn/flynn/blob/master/slugbuilder/builder/build.sh
8
+ module BuildpackManager
9
+
10
+ require 'fileutils'
11
+
12
+ include Satorix::Shared::Console
13
+
14
+ extend self
15
+
16
+
17
+ def go
18
+ abort_if_skip_buildpack
19
+ prepare_buildpack_environment
20
+ run_buildpacks
21
+ load_dot_release_file
22
+ switch_context_to_new_test_slug
23
+ end
24
+
25
+
26
+ def active_buildpack_list
27
+ custom_buildpack_list || stack_buildpack_list
28
+ end
29
+
30
+
31
+ def active_buildpacks
32
+ active_buildpack_list.split.map { |buildpack| Buildpack.new buildpack }
33
+ end
34
+
35
+
36
+ def cedar_14?
37
+ stack_version == 'cedar-14'
38
+ end
39
+
40
+
41
+ def copy_app_dir
42
+ FileUtils.cp_r File.join(Satorix.build_dir, '/.'), Satorix.app_dir, remove_destination: true
43
+ end
44
+
45
+
46
+ def custom_buildpack_list
47
+ IO.binread(files[:dot_buildpacks]).strip if custom_buildpacks?
48
+ end
49
+
50
+
51
+ def custom_buildpacks?
52
+ File.exist? files[:dot_buildpacks]
53
+ end
54
+
55
+
56
+ def default_buildpack_list
57
+ # When updating, pull from the most recent nightly release listed at https://releases.flynn.io/.
58
+ # Below was taken from https://github.com/flynn/flynn/blob/v20190814.0/slugbuilder/builder/buildpacks.txt
59
+ <<-THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN.strip
60
+ https://github.com/heroku/heroku-buildpack-multi.git#ed950773
61
+ https://github.com/cloudfoundry/staticfile-buildpack.git#e482e8f2
62
+ https://github.com/heroku/heroku-buildpack-ruby.git#4ca71a9d
63
+ https://github.com/heroku/heroku-buildpack-nodejs.git#a4fb9419
64
+ https://github.com/heroku/heroku-buildpack-clojure.git#5858bad3
65
+ https://github.com/heroku/heroku-buildpack-python.git#9dcabe24
66
+ https://github.com/heroku/heroku-buildpack-java.git#354d2a79
67
+ https://github.com/heroku/heroku-buildpack-gradle.git#b89c8c38
68
+ https://github.com/heroku/heroku-buildpack-scala.git#41c296d4
69
+ https://github.com/heroku/heroku-buildpack-php.git#62a691bf
70
+ https://github.com/heroku/heroku-buildpack-go.git#6f80fd9c
71
+ THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN
72
+ end
73
+
74
+
75
+ def default_cedar_14_buildpack_list
76
+ # When updating, pull from the most recent stable release listed at https://releases.flynn.io/.
77
+ # Below was taken from https://github.com/flynn/flynn/blob/v20170321.0/slugbuilder/builder/buildpacks.txt
78
+ <<-THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN.strip
79
+ https://github.com/heroku/heroku-buildpack-multi.git#ed950773
80
+ https://github.com/cloudfoundry/staticfile-buildpack.git#206728f9
81
+ https://github.com/heroku/heroku-buildpack-ruby.git#6988832f
82
+ https://github.com/heroku/heroku-buildpack-nodejs.git#9b8a98d8
83
+ https://github.com/heroku/heroku-buildpack-clojure.git#8768a8ff
84
+ https://github.com/heroku/heroku-buildpack-python.git#cafd4182
85
+ https://github.com/heroku/heroku-buildpack-java.git#8c34efe9
86
+ https://github.com/heroku/heroku-buildpack-gradle.git#13fa1fe7
87
+ https://github.com/heroku/heroku-buildpack-scala.git#dd0dd806
88
+ https://github.com/heroku/heroku-buildpack-play.git#cc5e6166
89
+ https://github.com/heroku/heroku-buildpack-php.git#e0499a7f
90
+ https://github.com/heroku/heroku-buildpack-go.git#bd1acfe5
91
+ THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN
92
+ end
93
+
94
+
95
+ def files
96
+ build_dir = Satorix.build_dir
97
+ { dot_buildpacks: File.join(build_dir, '.buildpacks'),
98
+ dot_env: File.join(build_dir, '.env'),
99
+ dot_release: File.join(build_dir, '.release') }
100
+ end
101
+
102
+
103
+ def load_dot_release_file
104
+ # TODO : Currently not required. Maybe should be implemented for maximum compatibility?
105
+ log_error_and_abort "Unhandled file '#{ files[:dot_release] }'." if File.exists? files[:dot_release]
106
+ end
107
+
108
+
109
+ def lsb_release
110
+ run_command(%w[lsb_release --release --short], quiet: true).chomp
111
+ end
112
+
113
+
114
+ def prepare_buildpack_environment
115
+ log_bench 'Preparing buildpack environment...' do
116
+ set_buildpack_expectations
117
+ process_dot_env_file
118
+ end
119
+ end
120
+
121
+
122
+ def process_dot_env_file
123
+ # TODO : Currently not required. Maybe should be implemented for maximum compatibility?
124
+ log_error_and_abort "Unhandled file '#{ files[:dot_env] }'" if File.exists? files[:dot_env]
125
+ end
126
+
127
+
128
+ def load_env
129
+ # TODO : Currently not required. Maybe should be implemented for maximum compatibility?
130
+ log_error_and_abort 'Unhandled load_env' unless (Dir.entries(Satorix.paths[:env]) - %w(. ..)).empty?
131
+ end
132
+
133
+
134
+ def load_profile
135
+ profile_locations.each do |profile_location|
136
+ Dir.glob(profile_location) do |shell_file|
137
+ # Uncomment below to debug profile loading
138
+ # run_command(['cat', shell_file])
139
+ source_env_from shell_file
140
+ end
141
+ end
142
+ end
143
+
144
+
145
+ def profile_locations
146
+ [File.join('/etc/profile.d', '*.sh'),
147
+ File.join(Satorix.app_dir, '.profile.d', '*.sh')]
148
+ end
149
+
150
+
151
+ def run_buildpacks
152
+ buildpacks = []
153
+
154
+ log_bench('Detecting application type...') do
155
+ active_buildpacks.each(&:ensure_correctness)
156
+ buildpacks = active_buildpacks.keep_if(&:detected?)
157
+
158
+ if buildpacks.empty?
159
+ log_error_and_abort 'Unable to select a buildpack!'
160
+ elsif buildpacks.length > 1 && !custom_buildpacks?
161
+ log_error("\n\nWarning: Multiple default buildpacks reported the ability to handle this app."\
162
+ " The first buildpack in the list above will be used.\n\n")
163
+ buildpacks = buildpacks.first(1)
164
+ end
165
+ end
166
+
167
+ buildpacks.each(&:compile)
168
+ end
169
+
170
+
171
+ def set_buildpack_expectations
172
+ ENV['APP_DIR'] = Satorix.app_dir
173
+ ENV['REQUEST_ID'] = "commit-#{ Satorix.ci_commit_sha }"
174
+ ENV['STACK'] = stack_version
175
+ ENV['DATABASE_URL'] = Satorix::CI::Test::Shared::Database.url
176
+ ENV['CURL_CONNECT_TIMEOUT'] = '30'
177
+ ENV['CURL_TIMEOUT'] = '600'
178
+ end
179
+
180
+
181
+ def abort_if_skip_buildpack
182
+ if ENV['SKIP_BUILDPACK']
183
+ log_error_and_abort("\n\nDEPRECATED: Skipping buildpack is no longer supported, please update your .gitlab-ci.yml.\n\n")
184
+ end
185
+ end
186
+
187
+
188
+ def stack_version
189
+ if lsb_release == '14.04'
190
+ return 'cedar-14'
191
+ else
192
+ return 'heroku-18'
193
+ end
194
+ end
195
+
196
+
197
+ def stack_buildpack_list
198
+ cedar_14? ? default_cedar_14_buildpack_list : default_buildpack_list
199
+ end
200
+
201
+
202
+ def switch_context_to_new_test_slug
203
+ log_bench 'Switching context to the newly built application...' do
204
+ copy_app_dir
205
+ load_env
206
+ load_profile
207
+ end
208
+ end
209
+
210
+ end
211
+ end
212
+ end
213
+ end
@@ -0,0 +1,153 @@
1
+ module Satorix
2
+ module CI
3
+ module Shared
4
+ module BuildpackManager
5
+
6
+ class Buildpack
7
+
8
+ require 'fileutils'
9
+ require 'uri'
10
+
11
+ include Satorix::Shared::Console
12
+
13
+ attr_accessor :commit_sha_short,
14
+ :url
15
+
16
+
17
+ def go
18
+ log_error_and_abort 'Buildpack.go should not be called directly - use BuildpackManager.go.'
19
+ end
20
+
21
+
22
+ def initialize(initialization_url)
23
+ self.url, self.commit_sha_short = initialization_url.to_s.strip.split('#')
24
+ end
25
+
26
+
27
+ def compile
28
+ log_bench "Building application using the #{ name } buildpack..." do
29
+ run_command([compile_binary_path,
30
+ Satorix.build_dir,
31
+ Satorix.paths[:cache],
32
+ Satorix.paths[:env]])
33
+ end
34
+ end
35
+
36
+
37
+ def compile_binary_path
38
+ bin_path = File.join(path, 'bin')
39
+ test_compile_path = File.join(bin_path, 'test-compile')
40
+ compile_path = File.join(bin_path, 'compile')
41
+ File.exist?(test_compile_path) ? test_compile_path : compile_path
42
+ end
43
+
44
+
45
+ def detected?
46
+ if BuildpackManager.custom_buildpacks?
47
+ log "Custom Buildpack: #{ name }."
48
+ else
49
+ command = [File.join(path, 'bin', 'detect'), Satorix.build_dir]
50
+ buildpack_name = run_command(command, quiet: true).chomp
51
+ log "Detected Framework: #{ buildpack_name }."
52
+ end
53
+ true
54
+ rescue SystemExit
55
+ # By design, buildpacks return a non-zero exit code
56
+ # (which raises a SystemExit exception) when not detected.
57
+ # This is not particularly exceptional, so we just return false.
58
+ false
59
+ end
60
+
61
+
62
+ def name
63
+ File.basename(URI.parse(url).path, ".git")
64
+ end
65
+
66
+
67
+ def path
68
+ File.join Satorix.paths[:buildpacks], name
69
+ end
70
+
71
+
72
+ def ensure_correctness
73
+ unless correct_version?
74
+ delete!
75
+ checkout
76
+ end
77
+ end
78
+
79
+
80
+ def delete!
81
+ FileUtils.rm_rf path
82
+ end
83
+
84
+
85
+ def exist?
86
+ Dir.exist? path
87
+ end
88
+
89
+
90
+ def checkout
91
+ log "Downloading Buildpack: #{ url }."
92
+ commit_sha_short ? checkout_specific_version : checkout_newest_version
93
+ end
94
+
95
+
96
+ def checkout_specific_version
97
+ run_command(['git', 'clone', '--quiet', '--no-checkout', url, path], quiet: true)
98
+ Dir.chdir(path) { run_command(['git', 'checkout', '--quiet', commit_sha_short], quiet: true) }
99
+ Dir.chdir(path) { run_command(%w[git submodule update --init --recursive], quiet: true) }
100
+ end
101
+
102
+
103
+ def checkout_newest_version
104
+ run_command(['git', 'clone', '--quiet', '--recursive', '--depth', '1', url, path], quiet: true)
105
+ end
106
+
107
+
108
+ def correct_version?
109
+ exist? &&
110
+ commit_sha_short &&
111
+ commit_sha_short == current_commit_sha_short_on_disk
112
+ end
113
+
114
+
115
+ def commit_sha_length
116
+ commit_sha_short ? commit_sha_short.to_s.length : 7
117
+ end
118
+
119
+
120
+ def satorix_distrib_sha_version
121
+ IO.binread(satorix_distrib_sha_path).strip if satorix_distrib_sha?
122
+ end
123
+
124
+
125
+ def satorix_distrib_sha?
126
+ File.exist? satorix_distrib_sha_path
127
+ end
128
+
129
+
130
+ def satorix_distrib_sha_path
131
+ File.join(path, '.distrib-sha')
132
+ end
133
+
134
+
135
+ def git_sha_short
136
+ Dir.chdir(path) do
137
+ # Also works: `git show -s --format=%h`.strip
138
+ run_command(['git', 'rev-parse', "--short=#{ commit_sha_length }", 'HEAD'], quiet: true).strip
139
+ end
140
+ end
141
+
142
+
143
+ def current_commit_sha_short_on_disk
144
+ return '' unless exist?
145
+ satorix_distrib_sha_version || git_sha_short
146
+ end
147
+
148
+ end
149
+
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,79 @@
1
+ module Satorix
2
+ module CI
3
+ module Shared
4
+ module Ruby
5
+ module GemManager
6
+
7
+ include Satorix::Shared::Console
8
+
9
+ extend self
10
+
11
+
12
+ def go(quiet = true)
13
+ if Satorix.ci_job_stage == 'deploy'
14
+ prepare_gem_for_deployment(quiet)
15
+ else
16
+ prepare_gem_environment_for_test(quiet)
17
+ end
18
+ end
19
+
20
+
21
+ def prepare_gem_for_deployment(quiet = true)
22
+ log_bench 'Preparing gem for deployment...' do
23
+ # Development dependencies should not be included in the Gemfile.lock
24
+ # otherwise they will become dependencies of the implementing application
25
+ remove_development_dependencies_from_gemspec(quiet)
26
+ regenerate_gemfile_lock(quiet)
27
+ end
28
+ end
29
+
30
+
31
+ def prepare_gem_environment_for_test(quiet = true)
32
+ log_bench 'Preparing gem environment...' do
33
+ restore_missing_gems(quiet)
34
+ end
35
+ end
36
+
37
+
38
+ def restore_missing_gems(quiet = true)
39
+ # The buildpack incorrectly removes gems that are included using add_development_dependency.
40
+ # These gems must remain available for testing.
41
+ run_command(%w[bundle install --with development], quiet: quiet)
42
+ end
43
+
44
+
45
+ def remove_development_dependencies_from_gemspec(quiet = true)
46
+ Dir[File.join(Satorix.app_dir, '*.gemspec')].each do |file|
47
+ log File.read(file) unless quiet
48
+
49
+ backup_file_name = "#{ file }.satorix.old"
50
+ FileUtils.mv(file, backup_file_name)
51
+
52
+ File.open(file, 'w') do |modified_file|
53
+ File.foreach(backup_file_name) do |line|
54
+ modified_file.puts line unless line =~ /\.add_development_dependency/
55
+ end
56
+ end
57
+
58
+ File.delete(backup_file_name)
59
+
60
+ log File.read(file) unless quiet
61
+ end
62
+
63
+ end
64
+
65
+
66
+ def regenerate_gemfile_lock(quiet = true)
67
+ run_command(%w[bundle config --local frozen false], quiet: quiet)
68
+ run_command(%w[bundle install], quiet: quiet)
69
+ log "\n\n#{ File.read(File.join(Satorix.app_dir, 'Gemfile.lock')) }" unless quiet
70
+ end
71
+
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+
79
+