satorix 0.0.1 → 1.6.0

data/lib/satorix/CI/deploy/flynn.rb

@@ -0,0 +1,129 @@
+module Satorix
+  module CI
+    module Deploy
+      module Flynn
+
+        include Satorix
+        include Satorix::Shared::Console
+
+        require_relative 'flynn/environment_variables'
+        include EnvironmentVariables
+
+        require_relative 'flynn/resources'
+        include Resources
+
+        require_relative 'flynn/routes'
+        include Routes
+
+        require_relative 'flynn/scale'
+        include Scale
+
+        extend self
+
+
+        def go
+          log_bench('Adding a local reference to the remote Flynn cluster...') { add_cluster }
+          log_bench('Creating Flynn project...') { create_project }
+          log_bench('Adding Flynn remote to CI local git...') { add_remote }
+          log_bench('Adjusting Flynn environment variables...') { adjust_env_vars }
+          log_bench('Configuring Flynn inactive slug release count...') { configure_inactive_slug_releases }
+          log_bench('Setting resources...') { set_resources }
+          log_bench('Deploying to Flynn...') { deploy }
+          log_bench('Running Database migrations...') { run_database_migrations } if run_database_migrations?
+          log_bench('Scaling application processes...') { adjust_scale }
+          log_bench('Setting up routes...') { setup_routes }
+        end
+
+
+        def skip_buildpack
+          true
+        end
+
+
+        def add_cluster
+          run_command(['flynn', 'cluster', 'add', '--force', '--default', "--tls-pin=#{ tls_pin }", cluster_name, domain_for_cluster, key], filtered_text: [tls_pin, key])
+        end
+
+
+        def add_remote
+          run_command(['flynn', '-a', project_name, 'remote', 'add', cluster_name, '-y'])
+        end
+
+
+        def cluster_name
+          domain_for_cluster
+        end
+
+
+        def configure_inactive_slug_releases
+          run_command(%w[flynn meta set gc.max_inactive_slug_releases=2])
+        end
+
+
+        def create_project
+          if project_exists?
+            log("Skipping - Flynn project '#{ project_name }' already exists.")
+          else
+            run_command(['flynn', 'create', "--remote=#{ cluster_name }", '-y', 'project_name'])
+          end
+        end
+
+
+        def deploy
+          run_command(['git', 'push', cluster_name, 'HEAD:refs/heads/master'])
+        end
+
+
+        def domain_for_cluster
+          ENV["FLYNN_#{ current_branch }_DOMAIN"]
+        end
+
+
+        def domain_for_web_host
+          "#{ current_branch.downcase }.#{ hosting_namespace }"
+        end
+
+
+        def hosting_namespace
+          if ENV['SATORIX_HOSTING_NAMESPACE'].to_s !~ only_whitespace
+            ENV['SATORIX_HOSTING_NAMESPACE']
+          else
+            log_error_and_abort("Satorix configuration error: Missing SATORIX_HOSTING_NAMESPACE.\n\nPlease contact support.\n")
+          end
+
+        end
+
+
+        def key
+          ENV["FLYNN_#{ current_branch }_KEY"]
+        end
+
+
+        def project_exists?
+          run_command(%w[flynn apps], quiet: true).split(/\R/).map { |a| a.split.last }.include? project_name
+        end
+
+
+        def run_database_migrations?
+          [rails_app?, django_app?].any?
+        end
+
+
+        def run_database_migrations
+          if desired_resource_provider_names.nil? || desired_resource_provider_names.empty?
+            log 'Skipping migrations, no database has been defined. Please see the resources section of this log for more information.'
+          else
+            run_command(%w[flynn run --enable-log bundle exec rake db:migrate]) if rails_app?
+            run_command(%w[flynn run --enable-log python public/manage.py migrate]) if django_app?
+          end
+        end
+
+
+        def tls_pin
+          ENV["FLYNN_#{ current_branch }_TLSPIN"]
+        end
+
+      end
+    end
+  end
+end

data/lib/satorix/CI/deploy/flynn/environment_variables.rb

@@ -0,0 +1,121 @@
+require 'yaml'
+
+module Satorix
+  module CI
+    module Deploy
+      module Flynn
+        module EnvironmentVariables
+
+          def adjust_env_vars
+            ensure_required_env_vars_are_defined unless ENV["SATORIX_#{ current_branch }_ENFORCE_ENV_VAR_DEFINITION"] == 'false'
+            env_unset
+            env_set
+          end
+
+
+          def aeev_key_exists?
+            !(run_command(%w[flynn env], quiet: true) =~ /#{ exported_aeevs_key }=/mi).nil?
+          end
+
+
+          def aeev_prefix
+            # AEEV - Application Exportable Environment Variable
+            "AEEV_#{ current_branch }_"
+          end
+
+
+          def ci_provided_env_vars
+            {}.tap do |vars|
+              ENV.each do |key, value|
+                vars[key.sub(aeev_prefix, '')] = value.sub('***EMPTY_STRING***', '') if key.start_with?(aeev_prefix)
+              end
+            end
+          end
+
+
+          def current_flynn_keys
+            aeev_key_exists? ? run_command(['flynn', 'env', 'get', exported_aeevs_key], quiet: true).split : []
+          end
+
+
+          def desired_env_vars
+            ci_provided_env_vars.tap do |vars|
+              vars[exported_aeevs_key] = (vars.keys << exported_aeevs_key).sort.join("\n")
+            end
+          end
+
+
+          def ensure_required_env_vars_are_defined(configuration_file: 'config/secrets.yml')
+            # TODO : handle new Rails secrets methods, like encrypted secrets
+            if File.file?(configuration_file)
+              secrets = YAML.load_file(configuration_file)
+
+              all_secrets = {}
+              all_secrets.merge!(secrets['shared']) if secrets['shared']
+              all_secrets.merge!(secrets['production']) if secrets['production']
+
+              required = all_secrets.to_yaml.scan(/ENV\[['"](?<var>[A-Z0-9_.]+)['"]\]/).flatten.uniq
+              set = ci_provided_env_vars.keys
+
+              missing = required - set
+
+              if missing.empty?
+                log "All required environment variables from #{ configuration_file } have been defined."
+              else
+                log_error "Environment variables specified in #{ configuration_file } were not defined for the #{ current_branch.downcase } branch."
+                log_error "\nPlease define the following variables in your dashboard:\n#{ missing.join("\n") }\n"
+                log_error_and_abort 'Missing required environment variables.'
+              end
+            else
+              log "No #{ configuration_file } exists, skipping environment variable enforcement."
+            end
+          end
+
+
+          def env_set
+            if env_vars_to_set.empty?
+              log 'No new environment variables to set.'
+            else
+              keys_and_values = env_vars_to_set.map { |k, v| "#{ k }=#{ v }" }
+              run_command(['flynn', 'env', 'set', keys_and_values].flatten, filtered_text: env_vars_to_set.values)
+            end
+          end
+
+
+          def env_unset
+            if env_vars_to_unset.empty?
+              log 'No existing environment variables to unset.'
+            else
+              run_command(['flynn', 'env', 'unset', env_vars_to_unset])
+            end
+          end
+
+
+          def env_vars
+            @_env_vars ||= begin
+                             {}.tap do |vars|
+                               current_flynn_keys.each { |key| vars[key] = run_command(['flynn', 'env', 'get', key], quiet: true).chomp }
+                             end
+                           end
+          end
+
+
+          def env_vars_to_set
+            @_env_vars_to_set ||= desired_env_vars.reject { |k, v| env_vars.key?(k) && env_vars[k] == v }
+          end
+
+
+          def env_vars_to_unset
+            @_env_vars_to_unset ||= env_vars.keys - desired_env_vars.keys
+          end
+
+
+          def exported_aeevs_key
+            'AEEV_KEYS'
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/satorix/CI/deploy/flynn/resources.rb

@@ -0,0 +1,77 @@
+module Satorix
+  module CI
+    module Deploy
+      module Flynn
+        module Resources
+
+          def add_resources
+            if resources_to_add.empty?
+              log 'No resources to add.'
+            else
+              resources_to_add.each { |resource| run_command(['flynn', 'resource', 'add', resource]) }
+            end
+          end
+
+
+          def available_resources
+            %w(postgres mysql mongodb redis)
+          end
+
+
+          def current_resource_provider_names
+            resource.split("\n").drop(1).map(&:split).map(&:last)
+          end
+
+
+          def desired_resource_provider_names
+            names = ENV[resource_provider_key].to_s.split
+            disallowed = names - available_resources
+            unless disallowed.empty?
+              log_error_and_abort("Invalid resource#{ 's' if disallowed.length > 1 }: #{ disallowed.join(' ') }")
+            end
+            names
+          end
+
+
+          def remove_resources
+            if resources_to_remove.empty?
+              log 'No resources to remove.'
+            else
+              log "The following previously allocated resources are no longer defined in #{ resource_provider_key }:"
+              log resources_to_remove.join(' ')
+              log ''
+              log 'To remove them, use the flynn resource command: https://flynn.io/docs/cli#resource'
+            end
+          end
+
+
+          def resource
+            run_command(%w[flynn resource], quiet: true).chomp
+          end
+
+
+          def resource_provider_key
+            "FLYNN_#{ current_branch }_RESOURCES"
+          end
+
+
+          def resources_to_add
+            desired_resource_provider_names - current_resource_provider_names
+          end
+
+
+          def resources_to_remove
+            current_resource_provider_names - desired_resource_provider_names
+          end
+
+
+          def set_resources
+            add_resources
+            remove_resources
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/satorix/CI/deploy/flynn/routes.rb

@@ -0,0 +1,266 @@
+module Satorix
+  module CI
+    module Deploy
+      module Flynn
+        module Routes
+
+          def find_or_create_route(domain)
+            route_ids = route_ids(domain)
+            if route_ids.empty?
+              log "Adding route for #{ domain }..."
+              route_ids << fc_route_add(domain)
+              log "Route for #{ domain } added with the ID of #{ route_ids.first }."
+            else
+              multiple = route_ids.length > 1
+              log "Route#{ 's' if multiple } already exist#{ 's' unless multiple } for #{ domain }."
+            end
+
+            route_ids
+          end
+
+
+          def configure_routes
+            defined_and_internal_routes.each do |ddev_id, domain|
+              route_ids = find_or_create_route(domain)
+              route_ids.each do |route_id|
+                add_tls_to_route(route_id: route_id, ddev_id: ddev_id)
+              end
+            end
+          end
+
+
+          def add_tls_to_route(route_id:, ddev_id:)
+            domain = defined_and_internal_routes[ddev_id]
+            if use_tls?(ddev_id)
+              if use_lets_encrypt?(ddev_id)
+                log "Using Let's Encrypt for #{ domain }."
+                log_error_and_abort "Let's Encrypt support is not implemented, yet!"
+              else
+                log "Using #{ user_defined_tls?(ddev_id) ? 'custom' : 'default' } certificate details for #{ domain } (#{ service_for_route_id(route_id) } service)."
+                File.open('crt', 'w') { |f| f.write(crt_for_ddev_id(ddev_id)) }
+                File.open('key', 'w') { |f| f.write(key_for_ddev_id(ddev_id)) }
+                fc_route_update(route_id)
+              end
+            else
+              log "Skipping TLS configuration for #{ domain }."
+              log "Environment variables #{ env_var_crt_prefix }#{ ddev_id } and #{ env_var_key_prefix }#{ ddev_id } have not been specified."
+              log 'For more information, please refer to https://www.satorix.com/docs/user/projects#certificates'
+            end
+            log ''
+          end
+
+
+          def canonical_domain
+            ENV["AEEV_#{ current_branch }_SATORIX_CANONICAL_URI_HOST"]
+          end
+
+
+          def canonical_domain_information
+            "\nThe above routes will all be redirected to the default URL:\n\n\t#{ canonical_uri }\n\n" if canonical_uri?
+          end
+
+
+          def canonical_domain_protocol
+            ENV["AEEV_#{ current_branch }_SATORIX_CANONICAL_URI_PROTOCOL"]
+          end
+
+
+          def canonical_uri
+            "#{ canonical_domain_protocol }://#{ canonical_domain }" if canonical_uri?
+          end
+
+
+          def canonical_uri?
+            [canonical_domain, canonical_domain_protocol].all? { |x| x.to_s !~ only_whitespace }
+          end
+
+
+          def crt_for_ddev_id(ddev_id)
+            ENV["#{ env_var_crt_prefix }#{ user_defined_tls?(ddev_id) ? ddev_id : 'DEFAULT' }"]
+          end
+
+
+          def custom_crt_for_ddev_id?(ddev_id)
+            ENV["#{ env_var_crt_prefix }#{ ddev_id }"].to_s !~ only_whitespace
+          end
+
+
+          def custom_key_for_ddev_id?(ddev_id)
+            ENV["#{ env_var_key_prefix }#{ ddev_id }"].to_s !~ only_whitespace
+          end
+
+
+          def defined_and_internal_routes
+            defined_routes.merge flynn_internal_routes
+          end
+
+
+          def defined_routes
+            {}.tap do |routes|
+              invalid_keys = flynn_internal_routes.keys
+              ENV.each do |key, value|
+                next unless key.start_with?(env_var_domain_prefix)
+                renamed_key = key.sub(env_var_domain_prefix, '')
+                message = "The user-defined route #{ key } conflicts with an internal route."
+                log_error_and_abort message if invalid_keys.include?(renamed_key)
+                routes[renamed_key] = value
+              end
+            end
+          end
+
+
+          def display_routing_information
+            log 'All application routes...'
+            log ''
+            log urlified_routes_for_display.map { |r| "\t#{ r }" }
+            log canonical_domain_information if canonical_uri?
+          end
+
+
+          def env_var_crt_prefix
+            "CRT_#{ current_branch }_"
+          end
+
+
+          def env_var_domain_prefix
+            "DDEV_#{ current_branch }_"
+          end
+
+
+          def env_var_key_prefix
+            "KEY_#{ current_branch }_"
+          end
+
+
+          def fc_route
+            run_command(['flynn', '-a', project_name, 'route'], quiet: true).chomp
+          end
+
+
+          def fc_route_add(domain)
+            run_command(['flynn', 'route', 'add', 'http', domain, '--sticky']).chomp
+          end
+
+
+          def fc_route_remove(route_id)
+            run_command(['flynn', 'route', 'remove', route_id])
+          end
+
+
+          def fc_route_update(route_id)
+            run_command(['flynn', 'route', 'update', route_id, '--tls-cert=crt', '--tls-key=key', '--sticky'], quiet: true)
+          end
+
+
+          def flynn_internal_routes
+            { 'FLYNN_INTERNAL' => "#{ project_name }.#{ domain_for_web_host }" }
+          end
+
+
+          def key_for_ddev_id(ddev_id)
+            ENV["#{ env_var_key_prefix }#{ user_defined_tls?(ddev_id) ? ddev_id : 'DEFAULT' }"]
+          end
+
+
+          def only_whitespace
+            /\A\s*\z/m
+          end
+
+
+          def remove_undefined_routes
+            routes_to_remove.each do |route|
+              log_header "Removing undefined route #{ route[routes_legend['ROUTE']] }..."
+              fc_route_remove route[routes_legend['ID']]
+            end
+          end
+
+
+          def route_ids(route)
+            [].tap do |route_ids|
+              routes_all_of('ROUTE').map { |r| r.sub(/https??:/i, '') }.each_with_index do |r, i|
+                route_ids << routes_all_of('ID')[i] if r == route
+              end
+            end
+          end
+
+
+          def routes
+            routes_with_legend.drop(1)
+          end
+
+
+          def routes_all_of(field)
+            routes.map { |a| a[routes_legend[field]] }
+          end
+
+
+          def routes_legend
+            {}.tap { |h| routes_with_legend.first.each_with_index { |route, index| h[route] = index } }
+          end
+
+
+          def routes_to_remove
+            routes.reject do |route|
+              defined_and_internal_routes.values.include? route[routes_legend['ROUTE']].partition(':').last
+            end
+          end
+
+
+          def routes_with_legend
+            fc_route.split("\n").map(&:split)
+          end
+
+
+          def service_for_route_id(route_id)
+            index = routes_all_of('ID').index(route_id)
+            routes_all_of('SERVICE')[index]
+          end
+
+
+          def setup_routes
+            configure_routes
+            remove_undefined_routes
+            display_routing_information
+          end
+
+
+          def urlified_routes
+            routes_all_of('ROUTE').map { |r| urlify_route r }
+          end
+
+
+          def urlified_routes_for_display
+            urlified_routes.map { |route| urlify_route_for_display route }.sort
+          end
+
+
+          def urlify_route(route)
+            route.sub(':', '://')
+          end
+
+
+          def urlify_route_for_display(route)
+            route =~ /^http:/ ? route : "#{ route.sub 'https', 'http' }, #{ route }"
+          end
+
+
+          def use_lets_encrypt?(ddev_id)
+            crt_for_ddev_id(ddev_id).to_s.gsub(/\W+/, '').casecmp('letsencrypt').zero? &&
+              key_for_ddev_id(ddev_id).to_s.gsub(/\W+/, '').casecmp('letsencrypt').zero?
+          end
+
+
+          def use_tls?(ddev_id)
+            [crt_for_ddev_id(ddev_id), key_for_ddev_id(ddev_id)].all? { |x| x.to_s !~ only_whitespace }
+          end
+
+
+          def user_defined_tls?(ddev_id)
+            custom_crt_for_ddev_id?(ddev_id) && custom_key_for_ddev_id?(ddev_id)
+          end
+
+        end
+      end
+    end
+  end
+end