razorrisk-cassini-utilities-cassid 0.8.11

data/lib/razor_risk/cassini/utilities/cassid/main.rb

@@ -0,0 +1,643 @@
+# encoding: utf-8
+
+
+# ##########################################################################
+#
+# Copyright (c) 2019 Razor Risk Technologies Pty Limited. All rights reserved.
+#
+# ##########################################################################
+
+# ##########################################################
+# requires
+
+require 'razor_risk/cassini/cli'
+require 'razor_risk/cassini/diagnostics/util_functions'
+require 'razor_risk/cassini/util/program_execution_util'
+require 'razor_risk/cassini/utilities/cassid/validations'
+require 'razor_risk/core/diagnostics/logger'
+
+require 'pantheios'
+require 'nokogiri'
+
+require 'base64'
+require 'yaml'
+
+
+# ##########################################################
+# includes
+
+include ::RazorRisk::Cassini::Utilities::CassiD::Validations
+include ::RazorRisk::Cassini::Util::ProgramExecutionUtil
+
+
+# ##########################################################
+# modules
+
+module RazorRisk
+module Cassini
+module Utilities
+module CassiD
+module Main
+
+
+# ##########################################################
+# includes
+
+include ::Pantheios
+include ::RazorRisk::Core::Diagnostics::Logger
+
+
+# ##########################################################
+# types
+
+class CassiDException < ::StandardError; end
+
+
+# ##########################################################
+# constants
+
+DEFAULT_LOG_DIRECTORY = File.join(Dir.pwd, 'logs')
+DEFAULT_LOG_THRESHOLD = [ :informational, :informational ]
+
+DEFAULT_CASSID_WAIT   = 10
+DEFAULT_SS_WAIT       = 4
+
+VALID_PORTS           = (1024...60000)
+
+ERROR_EXCEPTIONS      = [ ::ArgumentError, ::NameError, ::NoMethodError, ::TypeError ]
+
+
+# ##########################################################################
+# functions
+
+$children = []
+
+def self.combine_uri a0, a1
+
+    trace ParamNames[ :a0, :a1 ], a0, a1
+
+    a0 = a0[0...-1] if '/' == a0[-1]
+    a1 = a1[1..-1] if '/' == a1[0]
+
+    "#{a0}/#{a1}"
+end
+
+def self.kill_children
+
+    log :informational, "Killing child processes..."
+
+    $children.each do |c|
+        begin
+            Process.kill('KILL', c.pid)
+        rescue Errno::ESRCH
+        rescue
+            log :informational, "Failed to kill child process #{c.pid}"
+        else
+            log :informational, "Killed child process #{c.pid}"
+        end
+    end
+end
+
+def self.process_cli *args
+
+    options =   {
+        max_restarts: 0,
+        log_threshold: []
+    }
+
+    climate = LibCLImate::Climate.new do |cl|
+
+        cl.add_option(
+            '--microservice-multiplier',
+            alias: '-m',
+            help: "overrides the 'us_multiplier' configuration setting to the given number. Must be greater than 0"
+        ) do |o, a|
+
+            options[:us_multiplier] = Integer(o.value, nil: true)
+            unless options[:us_multiplier] and options[:us_multiplier] > 0
+                raise CassiDException.new("invalid value for microservice multiplier; use --help for usage")
+            end
+        end
+
+        cl.add_flag(
+            '--monitor-startup',
+            help: 'monitors child process statuses during initialisation'
+        ) do
+            options[:monitor_startup] = true
+        end
+
+        cl.add_flag(
+            '--pedantic',
+            help: 'conducts checks that the configuration is correct'
+        ) do
+            options[:pedantic] = true
+        end
+
+        cl.add_option(
+            '--max-restarts',
+            help: 'maximum number of times to restart a microservice'
+        ) do |o, a|
+            unless (val = Integer(o.value, nil: true)) < 0
+                options[:max_restarts] = val
+            else
+                raise CassiDException.new("max retries must be 0 or a positive integer; use --help for usage")
+            end
+        end
+
+        # Only added here as it may get passed through from cassid
+        cl.add_option('--exit-listener-port', alias: '-e',)
+
+        cl.add_option(
+            '--ruby-path',
+            alias: '-m',
+            help: 'absolute path to the ruby executable'
+        ) do |o, a|
+            options[:ruby_path] = o.value
+        end
+
+        cl.option_log_threshold(
+            options,
+            no_program_name: true,
+            no_benchmark: true,
+            limit: -2
+        )
+
+        cl.usage_values = '<config-yaml-file>'
+
+        cl.info_lines = [
+
+            'Startup Daemon for Cassini',
+            ::RazorRisk::Cassini::CLI.Copyright(2018),
+            :version,
+            '',
+            'Executes a Cassini instance based on a configuration file',
+            '',
+        ]
+    end
+
+    r = climate.run(args)
+
+    unless r.values[0]
+        raise CassiDException.new(
+            'configuration file not specified; use --help for usage'
+        )
+    end
+
+    config_path = File.expand_path(r.values[0])
+
+    [ config_path, options ]
+end
+
+def self.init_logging program_name, log_directory, log_threshold
+
+    ::Pantheios::Core.program_name = program_name if program_name
+    log_directory                  = log_directory || DEFAULT_LOG_DIRECTORY
+    log_threshold                  = log_threshold || DEFAULT_LOG_THRESHOLD
+
+    setup_diagnostic_logging(
+        ::Pantheios::Core.program_name,
+        log_directory,
+        log_threshold,
+        no_benchmark_log: true
+    )
+end
+
+def self.load_config config_path
+
+    trace ParamNames[ :config_path ], config_path
+
+    config = nil
+
+    begin
+        raise CassiDException.new(
+            "configuration file '#{config_path}' does not exist or is not a file"
+        ) unless File.file?(config_path)
+
+        yaml = ::YAML.load_file config_path
+
+        raise CassiDException.new(
+            "file '#{config_path}' does not contain a valid configuration"
+        ) unless valid_configuration?(yaml)
+
+        config = Configuration.new yaml
+    rescue *ERROR_EXCEPTIONS => x
+        log :violation, "unexpected exception (#{x.class}): '#{x.message}': #{x.backtrace}"
+        raise
+    rescue CassiDException
+        raise
+    rescue => x
+        log :debug, "exception (#{x.class}): '#{x}'"
+        raise CassiDException.new(
+            "configuration file '#{config_path}' is not YAML or could not be loaded"
+        )
+    end
+
+    config
+end
+
+def self.pedantic_check clarite_config, config
+
+    trace ParamNames[ :clarite_config, :config ], clarite_config, config
+
+    log :informational, 'Checking configuration ...'
+
+    failed = false
+
+    msg        = "Checking ClarITe configuration file '#{clarite_config}' is valid XML"
+    config_xml = nil
+    begin
+        config_xml = ::Nokogiri.XML(File.open(clarite_config)) do |c|
+            c.noblanks.strict
+        end
+        log :informational, "\t#{msg}: PASSED"
+    rescue
+        failed = true
+        log :critical, "\t#{msg}: FAILED"
+    end
+
+    msg = "Checking Razor alias or environment only"
+    unless config.razor.alias and config.razor.environment
+        log :informational, "\t#{msg}: PASSED"
+    else
+        failed = true
+        log :critical, "\t#{msg}: FAILED"
+    end
+
+    space       = config.razor.space
+    environment = config.razor.environment
+    if (environment or space) and config_xml
+
+        msg  = "Checking"
+        msg += " environment '#{environment}'" if environment
+        msg += " and" if environment and space
+        msg += " space '#{space}'" if space
+        msg += " is present within the ClarITe configuration file"
+
+        match = config_xml.xpath('//razor_server').any? do |razor_server|
+
+            server_space = razor_server.at_xpath('@space').text
+            server_env   = razor_server.at_xpath('@environment').text
+
+            (space ? server_space == space : true) &&
+                (environment ? server_env == environment : true)
+        end
+
+        if match
+            log :informational, "\t#{msg}: PASSED"
+        else
+            failed = true
+            log :critical, "\t#{msg}: FAILED"
+        end
+    end
+
+    raise CassiDException.new(
+        'failed one or more configuration checks'
+    ) if failed
+end
+
+def self.children_running?
+    $children.none? do |child|
+        !child.running?
+    end
+end
+
+def self.restart_children max_restarts
+    $children.each do |child|
+
+        unless child.running?
+            unless max_restarts < child.starts
+                child.start
+                log :notice, "Restarted #{child.name}..."
+            else
+                msg = "Microservice #{child.name} has failed too many times (#{child.starts})"
+                log :critical, msg
+                raise CassiDException.new msg
+            end
+        end
+    end
+end
+
+Microservice = Struct.new(:name, :command, :pid) do
+
+    def start
+        @starts ||= 0
+        @starts += 1
+        self.pid = Process.spawn(command.cmd)
+    end
+
+    def running?
+        begin
+            Process.waitpid(pid, Process::WNOHANG).nil?
+        rescue Errno::ECHILD
+            false
+        end
+    end
+
+    def starts
+        @starts
+    end
+end
+
+def self.init config_path, **options
+
+    trace ParamNames[ :config_path, :options ], config_path, options
+
+    begin
+
+        config = load_config config_path
+
+        # validate configuration elements:
+        #
+        # - root_dir is evaluated relative to the config file
+        #
+        # - clarite_config is evaluated relative to the root_dir, the current dir,
+        #   the config file
+
+        root_dir        =   validate_cassini_root_dir(config.cassini.root_dir, config_path) or raise CassiDException.new "value of 'cassini/root_dir' - '#{config.cassini.root_dir}' - is invalid, or cannot be determined through inference"
+        auth_mode       =   validate_auth_mode(config.cassini.auth_mode) or raise CassiDException.new "value of 'cassini/auth_mode' - '#{config.cassini.auth_mode}' - is not valid"
+        auth_test_mode  =   config.cassini.auth_test_mode
+        ces_svc_path    =   validate_service_path(config.cassini.ces['service_path'], root_dir, '.', config_path) or raise CassiDException.new "value of 'cassini/ces/service_path' - '#{config.cassini.ces['service_path']}' - missing or invalid, or cannot be determined through inference"
+        first_port      =   config.cassini.first_port and VALID_PORTS.include?(first_port) or raise CassiDException.new "value of 'cassini/first_port' - '#{config.cassini.first_port}' - is not a valid port number"
+        host            =   config.cassini.host or raise CassiDException.new "must supply 'cassini/host'"
+        host_ces        =   host
+        host_usvc       =   '0.0.0.0' == host ? 'localhost' : host
+        us_multiplier   =   options[:us_multiplier]
+        us_multiplier   ||= config.cassini.us_multiplier and (1..100).include?(config.cassini.us_multiplier) or raise CassiDException.new "value of 'cassini/us_multiplier' - '#{config.cassini.us_multiplier}' - is not a valid multiplier"
+
+        tls             =   validate_tls(config.cassini.tls, config_path) or raise CassiDException.new "value of 'cassini/tls' - '#{config.cassini.tls}' - is neither the path of an existing cert file that has an accompanying key file, nor is it an object containing keys 'cert' and 'key' that specify existing certificate and key files"
+        web_server      =   config.cassini.web_server
+        web_ui_server   =   config.cassini.web_ui_server
+
+        detach_children =   config.control.detach
+        ss_wait         =   config.control.ss_wait || DEFAULT_SS_WAIT
+        cassid_wait     =   config.control.cassid_wait || DEFAULT_CASSID_WAIT
+
+        log_thr_console =   (options[:log_threshold][0] || config.diagnostics.console['threshold']).to_sym
+        log_thr_main    =   (options[:log_threshold][1] || config.diagnostics.main['threshold']).to_sym
+        log_dir         =   options[:log_directory] || config.diagnostics.directory
+
+        clarite_config  =   validate_clarite_config(config.razor.clarite_config, root_dir, '.', config_path) or raise CassiDException.new "ClarITe configuration file '#{config.razor.clarite_config}' could not be found relative to root-directory, current directory or directory of configuration file"
+        executable      =   validate_executable(config.razor.executable, root_dir, config_path) or raise CassiDException.new "value of 'executable' - '#{config.razor.executable}' - could not be round relative to root-directory, current directory, or directory of configuration file"
+
+        init_logging(
+            options[:program_name],
+            log_dir,
+            [ log_thr_console, log_thr_main ],
+        )
+
+        if :jwt == auth_mode
+
+            ss = config.cassini.secret_server or raise CassiDException.new "'cassini/secret_server' is required when using JWT authentication"
+
+            # There are four possible elements:
+            #
+            # - 'host' (::String) [Optional] If not specified, the top-level
+            #   'host' is used
+            # - 'port' (::integer) The port on which to contact the
+            #   secret-server. [Optional] if 'secrets_path' and 'service_path'
+            #   are specified
+            # - 'secrets_path' (::String) [Optional] The secrets file path. If
+            #   specified, then 'service_path' must also be specified, and
+            #   together they indicate that the (Razor Risk) SecretServer
+            #   utility is to be launched
+            # - 'service_path' (::String) [Optional] The secrets server utility
+            #   path. If specified, then 'secrets_path' must also be specified,
+            #   and together they indicate that the (Razor Risk) SecretServer
+            #   utility is to be launched
+
+            ss_host         =   ss.host || host
+
+            if ss.port
+
+                ss_port = Integer(ss.port, nil: true) and VALID_PORTS.include?(ss.port) or raise CassiDException.new "value of 'cassini/secret_server/port' - '#{ss.port}' - is not a valid port number"
+            end
+
+            if ss.secrets_path
+
+                ss_secrets_path = validate_relative_path(ss.secrets_path, root_dir, '.', config_path) or raise CassiDException.new "secrets file path '#{ss.secrets_path}' could not be found relative to root-directory, current directory or directory of configuration file"
+            end
+
+            if ss.service_path
+
+                ss_service_path = validate_relative_path(ss.service_path, root_dir, '.', config_path) or raise CassiDException.new "service path '#{ss.service_path}' could not be found relative to root-directory, current directory or directory of configuration file"
+            end
+
+            raise CassiDException.new "A Login server is required when the authorisation mode is 'jwt'" unless config.cassini.microservices.dig('stock', 'login')
+
+        else
+
+            ss_host         =   nil
+            ss_port         =   nil
+            ss_secrets_path =   nil
+            ss_service_path =   nil
+        end
+
+        # ##########################################################################
+        # pedantic
+
+        pedantic_check(clarite_config, config) if options[:pedantic]
+
+        # ##########################################################################
+        # main
+
+        port = first_port
+
+        common_args =   [ auth_mode, executable, root_dir, clarite_config, host_usvc ]
+        ces_args    =   [ auth_mode, nil, root_dir, nil, host_ces ]
+
+        ms_options =  {
+            razor_environment: config.razor.environment,
+            razor_alias: config.razor.alias,
+            razor_space: config.razor.space,
+        }
+
+        ch_options  =   {
+            log_threshold: [ log_thr_console, log_thr_main ],
+            log_directory: log_dir,
+            web_server: web_server,
+            auth_test_mode: auth_test_mode,
+            ruby_path: options[:ruby_path],
+        }
+
+        ces_routes  =   {}
+
+        if :jwt == auth_mode
+
+            ss_port ||= port += 1
+
+            ssc = make_secretserver_command(root_dir, ss_secrets_path, ss_host, ss_port, **ch_options, **ms_options, program_name: 'ss', path: ss_service_path)
+
+            $children << Microservice.new('secret-server', ssc, nil)
+
+            ch_options[:secret_server] = "http://#{ss_host}:#{ss_port}/"
+
+            ch_options[:credentials_encoding_algorithm] = 'AES-256-CBC'
+            ch_options[:jwt_encoding_algorithm] = 'HS256'
+        end
+
+        # N x M microservices
+
+        config.cassini.microservices.each do |category, svcs|
+
+            svcs.each do |service, characteristics|
+
+                external_route  =   characteristics['external_route']
+                service_path    =   validate_service_path(characteristics['service_path'], root_dir, '.', config_path)
+                routes          =   characteristics['routes']
+
+                raise CassiDException.new "invalid configuration (#{characteristics})" if external_route.nil? || service_path.nil? || routes.nil?
+
+                raise CassiDException.new "invalid configuration: the external route '#{external_route}' has already been specified" if ces_routes.has_key?(external_route)
+
+                ex_route_spec   =   ces_routes[external_route] = {}
+
+                (0...us_multiplier).each do |n|
+
+                    port    +=  1
+                    usc     =   make_microservice_command service, *common_args, port, '', **ch_options, **ms_options, program_name: "#{service}-#{n}", path: service_path
+
+                    routes.each do |route|
+
+                        internal    =   route['internal']
+                        security    =   route['security'] || false
+
+                        uri         =   combine_uri usc.uri.to_s, internal
+
+                        verbs       =   route['verbs'] || []
+                        verb        =   route['verb']
+                        verbs       <<  verb if verb
+
+                        ex_route_spec[verbs]    =   [] unless ex_route_spec.has_key?(verbs)
+
+                        ex_route_spec[verbs]    <<  {
+
+                            route: uri,
+                            security: security,
+                        }
+                    end
+
+                    $children << Microservice.new("#{service}-#{n}", usc, nil)
+                end
+            end
+        end
+
+        config.cassini.external_services.each do |service, characteristics|
+
+            external_route = characteristics['external_route']
+            uri            = characteristics['uri']
+            routes         = characteristics['routes']
+
+            raise CassiDException.new(
+                "invalid configuration for #{service} (#{characteristics})"
+            ) if external_route.nil? || uri.nil?
+            raise CassiDException.new(
+                "invalid configuration for #{service}: the external route '#{external_route}' has already been specified"
+            ) if ces_routes.has_key?(external_route)
+
+            ex_route_spec = ces_routes[external_route] = {}
+            routes.each do |route|
+
+                internal  = route['internal']
+                security  = route['security'] || false
+
+                route_uri = combine_uri uri, internal
+
+                verbs     = route['verbs'] || []
+                verb      = route['verb']
+                verbs     << verb if verb
+
+                ex_route_spec[verbs] = [] unless ex_route_spec.has_key?(verbs)
+
+                ex_route_spec[verbs] << {
+                    route: route_uri,
+                    security: security,
+                }
+            end
+        end if config.cassini.external_services
+
+        # 1 x CES
+
+        yaml        =   ::YAML.dump ces_routes
+        routes_b64  =   Base64.encode64(yaml).split.join
+
+        ces_addnl   =   [
+
+            '--accept-string-routes',
+        ]
+
+        unless tls.empty?
+
+            ces_addnl   <<  [ '--tls-certificate-file', tls[0] ]
+            ces_addnl   <<  [ '--tls-public-key-file', tls[1] ]
+        end
+
+        ces_opts    =   {
+
+            routes_config: "base64:#{routes_b64}",
+            additional_arguments: ces_addnl,
+            uri_scheme: tls.empty? ? nil : 'https',
+        }
+
+        us_CES      =   make_microservice_command 'ces', *ces_args, first_port, nil, **ch_options, **ces_opts, path: ces_svc_path, web_ui_server: web_ui_server
+
+        $children << Microservice.new('ClientEdgeService', us_CES, nil)
+
+        # start the child processes
+
+        log :notice, 'starting child processes ...'
+
+        $children.each_with_index do |child, index0|
+
+            child.start
+
+            if 0 == index0 && :jwt == auth_mode
+
+                # wait for the secret server to start up
+                sleep(ss_wait)
+            end
+        end
+
+        log :notice, 'waiting for child processes to be ready ...'
+
+        sleep(cassid_wait)
+
+        if options[:monitor_startup]
+            unless children_running?
+                msg = 'One or more child process(es) has failed during startup'
+                log :critical, msg
+                raise CassiDException.new msg
+            end
+        end
+
+        # Output status
+
+        log :notice, "Cassini is now up and running, and able to receive calls at '#{us_CES.uri}'"
+
+        if detach_children
+            log :warning, "Detach mode has been depreciated in favour of service installation"
+        end
+
+    rescue *ERROR_EXCEPTIONS => x
+        log :violation, "unexpected exception (#{x.class}): '#{x.message}': #{x.backtrace}"
+        raise
+    rescue CassiDException
+        raise
+    rescue => x
+        log :alert, "exception (#{x.class}): '#{x.message}': #{x.backtrace}"
+        raise CassiDException.new 'Unexpected failure'
+    end
+end
+
+
+# ##########################################################
+# modules
+
+end # module Main
+end # module CassiD
+end # module Utilities
+end # module Cassini
+end # module RazorRisk
+
+# ############################## end of file ############################# #
+
+