satorix 0.0.1 → 1.6.0

data/lib/satorix/CI/deploy/flynn/scale.rb

@@ -0,0 +1,52 @@
+module Satorix
+  module CI
+    module Deploy
+      module Flynn
+        module Scale
+
+          def adjust_scale
+            cached_scale_options_to_set = scale_options_to_set
+            log "No scale specified in #{ defined_scale_key }. Displaying current scale:" if cached_scale_options_to_set.empty?
+            fc_scale cached_scale_options_to_set
+          end
+
+
+          def current_scale
+            scale_string_to_hash fc_scale
+          end
+
+
+          def defined_scale
+            scale_string_to_hash ENV[defined_scale_key].to_s
+          end
+
+
+          def defined_scale_key
+            "FLYNN_#{ current_branch }_SCALE"
+          end
+
+
+          def fc_scale(scale_options_to_set = nil)
+            command = %w[flynn scale].concat(scale_options_to_set)
+
+            run_command(command)
+          end
+
+
+          def scale_options_to_set
+            [].tap do |scale|
+              defined_scale.each { |job, workers| scale << "#{ job }=#{ workers }" }
+              (current_scale.keys - defined_scale.keys).each { |job| scale << "#{ job }=0" } unless defined_scale.empty?
+            end
+          end
+
+
+          def scale_string_to_hash(scale_string)
+            {}.tap { |jobs| scale_string.split.map { |x| x.partition('=') }.each { |x| jobs[x.first.to_sym] = x.last } }
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/satorix/CI/shared/buildpack_manager.rb

@@ -0,0 +1,213 @@
+module Satorix
+  module CI
+    module Shared
+
+      # This code was inspired by:
+      # herokuish - https://github.com/gliderlabs/herokuish
+      # flynn - https://github.com/flynn/flynn/blob/master/slugbuilder/builder/build.sh
+      module BuildpackManager
+
+        require 'fileutils'
+
+        include Satorix::Shared::Console
+
+        extend self
+
+
+        def go
+          abort_if_skip_buildpack
+          prepare_buildpack_environment
+          run_buildpacks
+          load_dot_release_file
+          switch_context_to_new_test_slug
+        end
+
+
+        def active_buildpack_list
+          custom_buildpack_list || stack_buildpack_list
+        end
+
+
+        def active_buildpacks
+          active_buildpack_list.split.map { |buildpack| Buildpack.new buildpack }
+        end
+
+
+        def cedar_14?
+          stack_version == 'cedar-14'
+        end
+
+
+        def copy_app_dir
+          FileUtils.cp_r File.join(Satorix.build_dir, '/.'), Satorix.app_dir, remove_destination: true
+        end
+
+
+        def custom_buildpack_list
+          IO.binread(files[:dot_buildpacks]).strip if custom_buildpacks?
+        end
+
+
+        def custom_buildpacks?
+          File.exist? files[:dot_buildpacks]
+        end
+
+
+        def default_buildpack_list
+          # When updating, pull from the most recent nightly release listed at https://releases.flynn.io/.
+          # Below was taken from https://github.com/flynn/flynn/blob/v20190814.0/slugbuilder/builder/buildpacks.txt
+          <<-THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN.strip
+              https://github.com/heroku/heroku-buildpack-multi.git#ed950773
+              https://github.com/cloudfoundry/staticfile-buildpack.git#e482e8f2
+              https://github.com/heroku/heroku-buildpack-ruby.git#4ca71a9d
+              https://github.com/heroku/heroku-buildpack-nodejs.git#a4fb9419
+              https://github.com/heroku/heroku-buildpack-clojure.git#5858bad3
+              https://github.com/heroku/heroku-buildpack-python.git#9dcabe24
+              https://github.com/heroku/heroku-buildpack-java.git#354d2a79
+              https://github.com/heroku/heroku-buildpack-gradle.git#b89c8c38
+              https://github.com/heroku/heroku-buildpack-scala.git#41c296d4
+              https://github.com/heroku/heroku-buildpack-php.git#62a691bf
+              https://github.com/heroku/heroku-buildpack-go.git#6f80fd9c
+          THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN
+        end
+
+
+        def default_cedar_14_buildpack_list
+          # When updating, pull from the most recent stable release listed at https://releases.flynn.io/.
+          # Below was taken from https://github.com/flynn/flynn/blob/v20170321.0/slugbuilder/builder/buildpacks.txt
+          <<-THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN.strip
+              https://github.com/heroku/heroku-buildpack-multi.git#ed950773
+              https://github.com/cloudfoundry/staticfile-buildpack.git#206728f9
+              https://github.com/heroku/heroku-buildpack-ruby.git#6988832f
+              https://github.com/heroku/heroku-buildpack-nodejs.git#9b8a98d8
+              https://github.com/heroku/heroku-buildpack-clojure.git#8768a8ff
+              https://github.com/heroku/heroku-buildpack-python.git#cafd4182
+              https://github.com/heroku/heroku-buildpack-java.git#8c34efe9
+              https://github.com/heroku/heroku-buildpack-gradle.git#13fa1fe7
+              https://github.com/heroku/heroku-buildpack-scala.git#dd0dd806
+              https://github.com/heroku/heroku-buildpack-play.git#cc5e6166
+              https://github.com/heroku/heroku-buildpack-php.git#e0499a7f
+              https://github.com/heroku/heroku-buildpack-go.git#bd1acfe5
+          THESE_ARE_THE_BUILDPACKS_OFFICIALLY_SUPPORTED_BY_FLYNN
+        end
+
+
+        def files
+          build_dir = Satorix.build_dir
+          { dot_buildpacks: File.join(build_dir, '.buildpacks'),
+            dot_env: File.join(build_dir, '.env'),
+            dot_release: File.join(build_dir, '.release') }
+        end
+
+
+        def load_dot_release_file
+          # TODO : Currently not required. Maybe should be implemented for maximum compatibility?
+          log_error_and_abort "Unhandled file '#{ files[:dot_release] }'." if File.exists? files[:dot_release]
+        end
+
+
+        def lsb_release
+          run_command(%w[lsb_release --release --short], quiet: true).chomp
+        end
+
+
+        def prepare_buildpack_environment
+          log_bench 'Preparing buildpack environment...' do
+            set_buildpack_expectations
+            process_dot_env_file
+          end
+        end
+
+
+        def process_dot_env_file
+          # TODO : Currently not required. Maybe should be implemented for maximum compatibility?
+          log_error_and_abort "Unhandled file '#{ files[:dot_env] }'" if File.exists? files[:dot_env]
+        end
+
+
+        def load_env
+          # TODO : Currently not required. Maybe should be implemented for maximum compatibility?
+          log_error_and_abort 'Unhandled load_env' unless (Dir.entries(Satorix.paths[:env]) - %w(. ..)).empty?
+        end
+
+
+        def load_profile
+          profile_locations.each do |profile_location|
+            Dir.glob(profile_location) do |shell_file|
+              # Uncomment below to debug profile loading
+              # run_command(['cat', shell_file])
+              source_env_from shell_file
+            end
+          end
+        end
+
+
+        def profile_locations
+          [File.join('/etc/profile.d', '*.sh'),
+           File.join(Satorix.app_dir, '.profile.d', '*.sh')]
+        end
+
+
+        def run_buildpacks
+          buildpacks = []
+
+          log_bench('Detecting application type...') do
+            active_buildpacks.each(&:ensure_correctness)
+            buildpacks = active_buildpacks.keep_if(&:detected?)
+
+            if buildpacks.empty?
+              log_error_and_abort 'Unable to select a buildpack!'
+            elsif buildpacks.length > 1 && !custom_buildpacks?
+              log_error("\n\nWarning: Multiple default buildpacks reported the ability to handle this app."\
+                              " The first buildpack in the list above will be used.\n\n")
+              buildpacks = buildpacks.first(1)
+            end
+          end
+
+          buildpacks.each(&:compile)
+        end
+
+
+        def set_buildpack_expectations
+          ENV['APP_DIR'] = Satorix.app_dir
+          ENV['REQUEST_ID'] = "commit-#{ Satorix.ci_commit_sha }"
+          ENV['STACK'] = stack_version
+          ENV['DATABASE_URL'] = Satorix::CI::Test::Shared::Database.url
+          ENV['CURL_CONNECT_TIMEOUT'] = '30'
+          ENV['CURL_TIMEOUT'] = '600'
+        end
+
+
+        def abort_if_skip_buildpack
+          if ENV['SKIP_BUILDPACK']
+            log_error_and_abort("\n\nDEPRECATED: Skipping buildpack is no longer supported, please update your .gitlab-ci.yml.\n\n")
+          end
+        end
+
+
+        def stack_version
+          if lsb_release == '14.04'
+            return 'cedar-14'
+          else
+            return 'heroku-18'
+          end
+        end
+
+
+        def stack_buildpack_list
+          cedar_14? ? default_cedar_14_buildpack_list : default_buildpack_list
+        end
+
+
+        def switch_context_to_new_test_slug
+          log_bench 'Switching context to the newly built application...' do
+            copy_app_dir
+            load_env
+            load_profile
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/satorix/CI/shared/buildpack_manager/buildpack.rb

@@ -0,0 +1,153 @@
+module Satorix
+  module CI
+    module Shared
+      module BuildpackManager
+
+        class Buildpack
+
+          require 'fileutils'
+          require 'uri'
+
+          include Satorix::Shared::Console
+
+          attr_accessor :commit_sha_short,
+                        :url
+
+
+          def go
+            log_error_and_abort 'Buildpack.go should not be called directly - use BuildpackManager.go.'
+          end
+
+
+          def initialize(initialization_url)
+            self.url, self.commit_sha_short = initialization_url.to_s.strip.split('#')
+          end
+
+
+          def compile
+            log_bench "Building application using the #{ name } buildpack..." do
+              run_command([compile_binary_path,
+                           Satorix.build_dir,
+                           Satorix.paths[:cache],
+                           Satorix.paths[:env]])
+            end
+          end
+
+
+          def compile_binary_path
+            bin_path = File.join(path, 'bin')
+            test_compile_path = File.join(bin_path, 'test-compile')
+            compile_path = File.join(bin_path, 'compile')
+            File.exist?(test_compile_path) ? test_compile_path : compile_path
+          end
+
+
+          def detected?
+            if BuildpackManager.custom_buildpacks?
+              log "Custom Buildpack: #{ name }."
+            else
+              command = [File.join(path, 'bin', 'detect'), Satorix.build_dir]
+              buildpack_name = run_command(command, quiet: true).chomp
+              log "Detected Framework: #{ buildpack_name }."
+            end
+            true
+          rescue SystemExit
+            # By design, buildpacks return a non-zero exit code
+            # (which raises a SystemExit exception) when not detected.
+            # This is not particularly exceptional, so we just return false.
+            false
+          end
+
+
+          def name
+            File.basename(URI.parse(url).path, ".git")
+          end
+
+
+          def path
+            File.join Satorix.paths[:buildpacks], name
+          end
+
+
+          def ensure_correctness
+            unless correct_version?
+              delete!
+              checkout
+            end
+          end
+
+
+          def delete!
+            FileUtils.rm_rf path
+          end
+
+
+          def exist?
+            Dir.exist? path
+          end
+
+
+          def checkout
+            log "Downloading Buildpack: #{ url }."
+            commit_sha_short ? checkout_specific_version : checkout_newest_version
+          end
+
+
+          def checkout_specific_version
+            run_command(['git', 'clone', '--quiet', '--no-checkout', url, path], quiet: true)
+            Dir.chdir(path) { run_command(['git', 'checkout', '--quiet', commit_sha_short], quiet: true) }
+            Dir.chdir(path) { run_command(%w[git submodule update --init --recursive], quiet: true) }
+          end
+
+
+          def checkout_newest_version
+            run_command(['git', 'clone', '--quiet', '--recursive', '--depth', '1', url, path], quiet: true)
+          end
+
+
+          def correct_version?
+            exist? &&
+              commit_sha_short &&
+              commit_sha_short == current_commit_sha_short_on_disk
+          end
+
+
+          def commit_sha_length
+            commit_sha_short ? commit_sha_short.to_s.length : 7
+          end
+
+
+          def satorix_distrib_sha_version
+            IO.binread(satorix_distrib_sha_path).strip if satorix_distrib_sha?
+          end
+
+
+          def satorix_distrib_sha?
+            File.exist? satorix_distrib_sha_path
+          end
+
+
+          def satorix_distrib_sha_path
+            File.join(path, '.distrib-sha')
+          end
+
+
+          def git_sha_short
+            Dir.chdir(path) do
+              # Also works: `git show -s --format=%h`.strip
+              run_command(['git', 'rev-parse', "--short=#{ commit_sha_length }", 'HEAD'], quiet: true).strip
+            end
+          end
+
+
+          def current_commit_sha_short_on_disk
+            return '' unless exist?
+            satorix_distrib_sha_version || git_sha_short
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/satorix/CI/shared/ruby/gem_manager.rb

@@ -0,0 +1,79 @@
+module Satorix
+  module CI
+    module Shared
+      module Ruby
+        module GemManager
+
+          include Satorix::Shared::Console
+
+          extend self
+
+
+          def go(quiet = true)
+            if Satorix.ci_job_stage == 'deploy'
+              prepare_gem_for_deployment(quiet)
+            else
+              prepare_gem_environment_for_test(quiet)
+            end
+          end
+
+
+          def prepare_gem_for_deployment(quiet = true)
+            log_bench 'Preparing gem for deployment...' do
+              # Development dependencies should not be included in the Gemfile.lock
+              # otherwise they will become dependencies of the implementing application
+              remove_development_dependencies_from_gemspec(quiet)
+              regenerate_gemfile_lock(quiet)
+            end
+          end
+
+
+          def prepare_gem_environment_for_test(quiet = true)
+            log_bench 'Preparing gem environment...' do
+              restore_missing_gems(quiet)
+            end
+          end
+
+
+          def restore_missing_gems(quiet = true)
+            # The buildpack incorrectly removes gems that are included using add_development_dependency.
+            # These gems must remain available for testing.
+            run_command(%w[bundle install --with development], quiet: quiet)
+          end
+
+
+          def remove_development_dependencies_from_gemspec(quiet = true)
+            Dir[File.join(Satorix.app_dir, '*.gemspec')].each do |file|
+              log File.read(file) unless quiet
+
+              backup_file_name = "#{ file }.satorix.old"
+              FileUtils.mv(file, backup_file_name)
+
+              File.open(file, 'w') do |modified_file|
+                File.foreach(backup_file_name) do |line|
+                  modified_file.puts line unless line =~ /\.add_development_dependency/
+                end
+              end
+
+              File.delete(backup_file_name)
+
+              log File.read(file) unless quiet
+            end
+
+          end
+
+
+          def regenerate_gemfile_lock(quiet = true)
+            run_command(%w[bundle config --local frozen false], quiet: quiet)
+            run_command(%w[bundle install], quiet: quiet)
+            log "\n\n#{ File.read(File.join(Satorix.app_dir, 'Gemfile.lock')) }" unless quiet
+          end
+
+        end
+      end
+    end
+  end
+end
+
+
+