oci-logging-analytics-kubernetes-discovery 1.0.0

data/lib/util/kube_client.rb

@@ -0,0 +1,141 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require 'resolv'
+require_relative '../util/logging'
+
+module Util
+  module KubeClient
+    extend Util::Logging
+
+    module_function
+
+    attr_accessor :current_kube_context, :core_client, :apps_client, :batch_client,
+                  :discover_v1_client, :kube_clients, :kube_config_hash,
+                  :kubernetes_url, :api_endpoint, :api_version,
+                  :client_cert, :client_key, :ca_file, :verify_ssl, :bearer_token_file, :secret_dir
+
+    def create_clients(kube_config_hash)
+      begin
+        initialize_config(kube_config_hash)
+        @core_client = create_client_by_object_type('', @api_version)
+        @apps_client = create_client_by_object_type('/apis/apps', @api_version)
+        @batch_client = create_client_by_object_type('/apis/batch', @api_version)
+        @discover_v1_client = create_client_by_object_type('/apis/discovery.k8s.io', @api_version)
+      rescue StandardError => e
+        logger.error("Error while creating kube clients. Error - #{e}")
+        raise e
+      end
+
+      set_clients(@core_client, @apps_client, @batch_client, @discover_v1_client)
+    end
+
+    def initialize_config(kube_config_hash)
+      @kube_config_hash = kube_config_hash
+
+      unless @kube_config_hash[:kube_config_location].nil?
+        begin
+          config = Kubeclient::Config.read @kube_config_hash[:kube_config_location]
+          @current_kube_context = config.context
+        rescue StandardError => e
+          logger.error("Error while initializing config params for kube client - #{e}")
+          raise e
+        end
+      end
+
+      @kubernetes_url = @kube_config_hash[:kubernetes_url]
+      @api_endpoint = @kube_config_hash[:api_endpoint]
+      @api_version = @kube_config_hash[:api_version]
+      @client_cert = @kube_config_hash[:client_cert]
+      @client_key = @kube_config_hash[:client_key]
+      @ca_file = @kube_config_hash[:ca_file]
+      @verify_ssl = @kube_config_hash[:verify_ssl]
+      @bearer_token_file = @kube_config_hash[:bearer_token_file]
+      @secret_dir = @kube_config_hash[:secret_dir]
+    end
+
+    def create_client_by_object_type(api_endpoint, api_version)
+      if !@current_kube_context.nil?
+        create_kube_client_config(api_endpoint, api_version)
+      else
+        create_kube_client_default(api_endpoint, api_version)
+      end
+    end
+
+    def create_kube_client_config(api_endpoint, api_version)
+      logger.debug("Endpoint: #{@current_kube_context.api_endpoint + api_endpoint} and api_version: #{api_version}")
+      begin
+        # TODO: Test with SSL options.
+        client = Kubeclient::Client.new(
+          @current_kube_context.api_endpoint + api_endpoint,
+          api_version,
+          ssl_options: @current_kube_context.ssl_options,
+          auth_options: @current_kube_context.auth_options
+        )
+      rescue StandardError => e
+        logger.error("Error while creating kube client based on config - #{e}")
+        raise e
+      end
+      client
+    end
+
+    def create_kube_client_default(api_endpoint, api_version)
+      # Reference: https://github.com/splunk/fluent-plugin-kubernetes-objects/blob/develop/lib/fluent/plugin/in_kubernetes_objects.rb#L139
+      kubernetes_url = nil
+      if @kubernetes_url.nil?
+        # Use Kubernetes default service account if we're in a pod.
+        env_host = ENV['KUBERNETES_SERVICE_HOST']
+        env_host = "[#{env_host}]" if env_host =~ Resolv::IPv6::Regex
+        env_port = ENV['KUBERNETES_SERVICE_PORT']
+        if env_host && env_port
+          # kubernetes_url = "https://#{env_host}:#{env_port}/#{@api_endpoint.delete_prefix('/')}#{api_endpoint}"
+          kubernetes_url = "https://#{env_host}:#{env_port}#{api_endpoint}"
+        end
+      else
+        kubernetes_url = @kubernetes_url + api_endpoint
+      end
+
+      # Use SSL certificate and bearer token from Kubernetes service account.
+      if Dir.exist?(@secret_dir)
+        secret_ca_file = File.join(@secret_dir, 'ca.crt')
+        secret_token_file = File.join(@secret_dir, 'token')
+
+        @ca_file = secret_ca_file if @ca_file.nil? && File.exist?(secret_ca_file)
+
+        @bearer_token_file = secret_token_file if @bearer_token_file.nil? && File.exist?(secret_token_file)
+      end
+
+      ssl_options = {
+        client_cert: @client_cert && OpenSSL::X509::Certificate.new(File.read(@client_cert)),
+        client_key: @client_key && OpenSSL::PKey::RSA.new(File.read(@client_key)),
+        ca_file: @ca_file,
+        verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+      }
+
+      auth_options = {}
+      auth_options[:bearer_token] = File.read(@bearer_token_file) if @bearer_token_file
+      logger.debug("kubernetes_url : #{kubernetes_url} and api_version: #{api_version}")
+      Kubeclient::Client.new(
+        kubernetes_url, @api_version,
+        ssl_options: ssl_options,
+        auth_options: auth_options
+      )
+    rescue StandardError => e
+      logger.error("Error while creating default kube client - #{e}")
+      raise e
+    end
+
+    def set_clients(core_client, apps_client, batch_client, discover_v1_client)
+      @kube_clients = {
+        core_client: core_client,
+        apps_client: apps_client,
+        batch_client: batch_client,
+        discover_v1_client: discover_v1_client
+      }
+    end
+
+    def get_clients
+      @kube_clients
+    end
+  end
+end

data/lib/util/kubectl_ops.rb

@@ -0,0 +1,229 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require_relative '../util/logging'
+
+# Enum
+require_relative '../enum/kubernetes_objects_enum'
+
+module Util
+  module KubectlOps
+    extend Util::Logging
+
+    module_function
+
+    attr_accessor :chunk_limit
+
+    CONFIG_MAP_RESOURCE_NAME = 'config_map'.freeze
+
+    # Status codes for which retry is attempted (To represent 5xx, use 5)
+    RETRYABLE_STATUS_CODE = [429, 5].freeze
+    MAX_ATTEMPTS = 5
+    INITIAL_DELAY = 1 # seconds
+    EXPONENT_CONSTANT = 2 # 'e' in exponential function (e^x) where x is number of retries
+
+    def set_chunk_limit(num)
+      @chunk_limit = num
+    end
+
+    def should_retry?(error, retry_count, object_type)
+      return false if error.error_code.nil? # for handling timeouts
+
+      if (RETRYABLE_STATUS_CODE.include? error.error_code) || RETRYABLE_STATUS_CODE.include?(error.error_code / 100)
+        if retry_count < MAX_ATTEMPTS
+          # If e = 2, and initial delay is 10, intervals will be (1, 2, 4, 8, 16)
+          sleep_interval = EXPONENT_CONSTANT.pow(retry_count) * INITIAL_DELAY
+          logger.warn("Error while making API Server call - '#{object_type}'. Retrying in #{sleep_interval} seconds. HTTP status code: #{error.error_code}")
+          sleep(sleep_interval)
+          return true
+        end
+      end
+      false
+    end
+
+    def kube_apiserver_request(method:, object_type:, client: nil, object_name: nil, namespace: nil, object: nil, patch: nil, opts: {})
+      response = nil
+      retry_count = 0
+      begin
+        case method
+        when :get_objects
+          if client.nil?
+            client = Util::KubeClient.get_clients[Enum::ObjectClientMapingEnum.const_get(object_type.upcase.to_s)]
+          end
+          logger.debug("Fetching '#{object_type}' details from all namespaces")
+          method_verb = 'GET'
+          opts.merge!({ as: :parsed_symbolized })
+          response = {}
+          loop do
+            entities = client.public_send("get_#{object_type}", opts)
+            continue = entities[:metadata][:continue]
+            if response.empty?
+              # when all objects are fetched via single API request
+              if continue.nil?
+                response = entities
+                break
+              end
+              response = entities.clone
+            else
+              response[:items] += entities[:items]
+            end
+            break if continue.nil?
+
+            logger.debug("remainingItemCount - #{entities[:metadata][:remainingItemCount]}")
+            opts.merge!({ continue: continue })
+          end
+          if response.nil? || response.empty?
+            logger.debug("Empty or null response recieved from Kubernetes API endpoint '#{client.rest_client.url}'.")
+            return { parsed_symbolized: [], raw: '' }
+          else
+            return { parsed_symbolized: response, raw: JSON.dump(response) }
+          end
+        when :get_object
+          method_verb = 'GET'
+          logger.debug("Fetching '#{object_type}/#{object_name}' details")
+          opts.merge!(as: :parsed_symbolized)
+          response = client.public_send("get_#{object_type}", object_name, namespace, opts)
+        when :create_object
+          method_verb = 'CREATE'
+          logger.debug("Creating '#{object_type}' - #{object}")
+          response = client.public_send("create_#{object_type}", object)
+        when :patch_object
+          method_verb = 'PATCH'
+          logger.debug("Patching '#{object_type}/#{object_name}' - #{patch}")
+          response = client.public_send("patch_#{object_type}", object_name, patch, namespace)
+        else
+          raise SyntaxError, "Unsupported method - #{method} passed."
+        end
+      rescue KubeException => e
+        if should_retry?(e, retry_count, object_type)
+          retry_count += 1
+          retry
+        end
+
+        if e.error_code.nil?
+          logger.error("Unexpected error occurred while calling #{method_verb} '#{object_type}'. Error details: #{e}")
+          raise e
+        elsif e.error_code == 302
+          logger.error("Max re-direct attempts while calling #{method_verb} '#{object_type}'. Error details: #{e}")
+          raise StandardError, "Max re-direct attempts while calling #{method_verb} '#{object_type}'."
+        elsif e.error_code == 401
+          # In such cases, client should be re-created in order to refresh token
+          logger.error("Not authorized to #{method_verb} '#{object_type}'. Error details: #{e}")
+          raise StandardError, "Not authorized to #{method_verb} '#{object_type}'."
+        elsif e.error_code == 403
+          logger.error("Forbidden to #{method_verb} '#{object_type}'. Error details: #{e}")
+          raise StandardError, "Forbidden to #{method_verb} '#{object_type}'."
+        elsif e.error_code == 404
+          logger.debug("Resource '#{object_type}' not found. Error details: #{e}")
+          raise e
+        elsif e.error_code == 410
+          # ref - https://kubernetes.io/docs/reference/using-api/api-concepts/#retrieving-large-results-sets-in-chunks
+          logger.error("Too many '#{object_type}' objects. Consider increasing CHUNK limit.")
+          raise StandardError, "Too many '#{object_type}' objects. Consider increasing CHUNK limit."
+        elsif e.error_code == 429
+          logger.error("Encountered throttling while calling #{method_verb} '#{object_type}'. Error details: #{e}")
+          raise StandardError, "Encountered throttling while #{method_verb} '#{object_type}'."
+        elsif (e.error_code / 100) == 5
+          logger.error("Unexpected error occurred while #{method_verb} '#{object_type}'. Error details: #{e}")
+          raise StandardError, "Unexpected error occurred while #{method_verb} '#{object_type}'."
+        else
+          logger.error("Error '#{e}' encountered while #{method_verb} '#{object_type}' from Kubernetes API endpoint '#{client.rest_client.url}'")
+          raise StandardError, "Error '#{e}' encountered while #{method_verb} '#{object_type}'."
+        end
+      rescue NoMethodError => e
+        logger.error('Unexpected Error - Method not found.')
+        raise e
+      rescue StandardError => e
+        logger.error("Error while #{method_verb} '#{object_type}'. Error: #{e}")
+
+        if should_retry?(e, retry_count, object_type)
+          retry_count += 1
+          retry
+        end
+
+        error_msg = "Maximum retry attempts ('#{MAX_ATTEMPTS}') exceeded. Aborting fetch for '#{object_type}'."
+        logger.error(error_msg)
+        raise error error_msg
+      end
+      response
+    end
+
+    def get_objects(object_type, client = nil, options = {})
+      options.merge!({ limit: @chunk_limit, continue: nil })
+      kube_apiserver_request(method: :get_objects,
+                             object_type: object_type,
+                             client: client,
+                             opts: options)
+    end
+
+    def get_object(object_type, object_name, namespace, client, options = {})
+      kube_apiserver_request(method: :get_object,
+                             object_type: object_type,
+                             client: client,
+                             object_name: object_name,
+                             namespace: namespace,
+                             opts: options)
+    end
+
+    def create_object(object_type, kubeclient_resource_object, client)
+      kube_apiserver_request(method: :create_object,
+                             object_type: object_type,
+                             client: client,
+                             object: kubeclient_resource_object)
+    end
+
+    def patch_object(object_type, object_name, patch_hash, namespace, client)
+      kube_apiserver_request(method: :patch_object,
+                             object_type: object_type,
+                             client: client,
+                             object_name: object_name,
+                             patch: patch_hash,
+                             namespace: namespace)
+    end
+
+    def get_configmap(cm_name, namespace)
+      object_type = CONFIG_MAP_RESOURCE_NAME
+      client = Util::KubeClient.get_clients[:core_client]
+      begin
+        response = get_object(object_type, cm_name, namespace, client)
+      rescue StandardError => e
+        return nil if e.error_code == 404
+
+        raise e
+      end
+      response
+    end
+
+    def create_configmap(cm_name, namespace, data_hash)
+      # Create resource object for config map
+      cm = Kubeclient::Resource.new
+      cm.metadata = {}
+      cm.metadata.name = cm_name
+      cm.metadata.namespace = namespace
+      cm.data = {}
+      cm.data = data_hash
+
+      object_type = CONFIG_MAP_RESOURCE_NAME
+      client = Util::KubeClient.get_clients[:core_client]
+
+      begin
+        create_object(object_type, cm, client)
+      rescue StandardError => e
+        logger.error("Unable to create '#{object_type}/#{cm_name}'")
+        raise e
+      end
+    end
+
+    def patch_configmap(cm_name, namespace, data_hash)
+      patch_hash = { data: data_hash }
+      object_type = CONFIG_MAP_RESOURCE_NAME
+      client = Util::KubeClient.get_clients[:core_client]
+      begin
+        patch_object(object_type, cm_name, patch_hash, namespace, client)
+      rescue StandardError => e
+        logger.error("Unable to patch '#{object_type}/#{cm_name}'")
+        raise e
+      end
+    end
+  end
+end

data/lib/util/log_analytics.rb

@@ -0,0 +1,154 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require_relative '../util/logging'
+require_relative '../util/oci_clients'
+
+# 'httplog' required *after* HTTP gem of choice
+# require 'httplog'
+
+module Util
+  module LogAnalytics
+    extend Util::Logging
+
+    # Client information for raw request signing
+    CLIENT_INFO = 'Oracle-ORD/Raw Request'.freeze
+    CLIENT_AGENT = 'Oracle-ORD/Raw Request (net/http)'.freeze
+
+    LOG_ANALYTICS_ENDPOINT_TEMPLATE = 'https://loganalytics.{region}.oci.{secondLevelDomain}'.freeze
+
+    # Status codes for which retry is attempted (To represent 5xx, use 5)
+    RETRYABLE_STATUS_CODE = [408, 429, 5].freeze
+    MAX_ATTEMPTS = 5
+    INITIAL_DELAY = 1 # seconds
+    EXPONENT_CONSTANT = 2 # 'e' in exponential function (e^x) where x is number of retries
+
+    module_function
+
+    def upload_discovery_data(namespace, payload, opts)
+      logger.info("Uploading payload to 'Log Analytics' with Discovery API")
+      begin
+        client = Util::OCIClients.get_clients[:la_client]
+        response = client.upload_discovery_data(namespace_name = namespace,
+                                                upload_discovery_data_details = payload,
+                                                opts)
+      rescue StandardError => e
+        logger.error("Error while uploading payload to 'Log Analytics': #{e}")
+        raise StandardError, "Unable to upload data to 'Log Analytics' client."
+      end
+      response
+    end
+
+    def upload_discovery_data_raw_request(namespace, payload, opts, auth_config)
+      logger.info('Uploading payload to \'Log Analytics\' with Discovery API (raw request)')
+
+      # Prepare config and signer
+      signer = nil
+      operation_signing_strategy = :exclude_body
+
+      case auth_config[:auth_type]
+      when Enum::AuthTypeEnum::CONFIG
+        config = auth_config[:oci_config]
+        signer = OCI::Signer.config_file_auth_builder(config)
+        region = config.region
+      when Enum::AuthTypeEnum::INSTANCE_PRINCIPAL
+        signer = auth_config[:instance_principals_signer]
+        region = signer.region
+      else
+        logger.warn("Unknown auth_type provided for Discovery API (raw request): #{auth_object[:auth_type]}")
+        raise StandardError 'Unknown auth_type for Discovery API (raw request)'
+      end
+
+      # Construct URL protocol, subdomain, domain and port
+
+      endpoint = if !auth_config[:endpoint].nil?
+                   auth_config[:endpoint] + '/20200601'
+                 else
+                   OCI::Regions.get_service_endpoint_for_template(region, LOG_ANALYTICS_ENDPOINT_TEMPLATE) + '/20200601'
+                 end
+
+      # Construct URL path
+      path = '/namespaces/{namespaceName}/actions/uploadDiscoveryData'.sub('{namespaceName}', namespace.to_s)
+
+      # Construct Query Params
+      query_params = '?discoveryDataType={discoveryDataType}&payloadType={payloadType}'
+                     .sub('{discoveryDataType}', opts[:discovery_data_type])
+                     .sub('{payloadType}', opts[:payload_type])
+
+      # Prepare URI
+      url = endpoint + path + query_params
+      uri = URI(url)
+
+      # Set user agent and OPC parameters for request
+      http_method = :post
+
+      request = Net::HTTP::Post.new(uri)
+      request['opc-client-info'] = CLIENT_INFO
+      request['opc-request-id'] = SecureRandom.uuid.delete!('-').upcase
+      request['User-Agent'] = CLIENT_AGENT
+
+      # Header Params
+      header_params = {}
+      header_params = opts[:additional_headers].clone if opts[:additional_headers].is_a?(Hash)
+      header_params[:accept] = 'application/json'
+      header_params[:expect] = '100-continue'
+      header_params[:'content-type'] = 'application/octet-stream'
+      header_params[:'opc-meta-properties'] = opts[:opc_meta_properties] if opts[:opc_meta_properties]
+      header_params[:'opc-retry-token'] = OCI::Retry.generate_opc_retry_token
+
+      # Set the payload
+      if payload.respond_to?(:read) && payload.respond_to?(:write)
+        request.body_stream = payload
+      else
+        request.body = payload
+      end
+
+      # Sign the request
+      signer.sign(http_method, url, header_params, payload, operation_signing_strategy)
+
+      # Add header parameters
+      header_params.each do |key, value|
+        request[key.to_s] = value
+      end
+
+      # Set HTTP parameters
+      http = Net::HTTP.new(uri.hostname, uri.port)
+      http.use_ssl = (uri.scheme == 'https')
+      http.open_timeout = 10
+      http.read_timeout = 180 # 180 seconds = 3 minutes
+      http.continue_timeout = 3 # expect 100 continue timeout
+
+      # Fetch response
+      retry_count = 0
+      response = nil
+
+      begin
+        http.start do
+          http.request(request) do |resp|
+            response = resp
+          end
+        end
+      rescue StandardError => e
+        code_from_response = !response.nil? ? response.code.to_i : 0
+
+        if (RETRYABLE_STATUS_CODE.include? code_from_response) || RETRYABLE_STATUS_CODE.include?(code_from_response / 100) || e.instance_of?(Net::ReadTimeout)
+          if retry_count < MAX_ATTEMPTS
+            # If e = 2, and initial delay is 10, intervals will be (1, 2, 4, 8, 16)
+            sleep_interval = EXPONENT_CONSTANT.pow(retry_count) * INITIAL_DELAY
+            logger.error("Error while uploading data to discovery API (raw request). Retrying in #{sleep_interval} seconds. HTTP status code: #{code_from_response}")
+            sleep(sleep_interval)
+            retry_count += 1
+            retry
+          end
+        end
+        raise OCI::Errors::NetworkError.new(e.message, code_from_response, request_made: request, response_received: response)
+      end
+
+      if response.is_a?(Net::HTTPSuccess)
+        return OCI::Response.new(response.code.to_i, response.header, nil)
+      end # Success safe check
+
+      raise "Response from Discovery API (raw request) suggests failure. Response code: + #{response.code}"
+    end
+  end
+end

data/lib/util/logging.rb

@@ -0,0 +1,96 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require 'logger'
+require 'yajl'
+
+module Util
+  module Logging
+    module_function
+
+    TRACE = Logger::DEBUG - 1
+    SEV_LABEL = {
+      -1 => 'TRACE',
+      0 => 'DEBUG',
+      1 => 'INFO',
+      2 => 'WARN',
+      3 => 'ERROR',
+      4 => 'FATAL'
+    }.freeze
+
+    def logger
+      log_level = LogConfigs.log_level
+      log_format = LogConfigs.log_format
+      datetime_format = '%Y-%m-%d %H:%M:%S %z'
+
+      case log_format
+      when 'text'
+        @logger = OCILogger.new(STDOUT,
+                                level: log_level,
+                                datetime_format: datetime_format,
+                                formatter: proc do |type, time, _level, msg|
+                                             r = "#{time} [#{type.downcase}] #{self} "
+                                             r << "#{msg}\n"
+                                             r
+                                           end)
+      when 'json'
+        @logger = OCILogger.new(STDOUT,
+                                level: log_level,
+                                datetime_format: datetime_format,
+                                formatter: proc do |type, time, _level, msg|
+                                             r = {
+                                               'time' => time,
+                                               'level' => type.downcase,
+                                               'class' => self,
+                                               'message' => msg
+                                             }
+                                             # JSON.pretty_generate(r, :indent => "\t") #
+                                             Yajl.dump(r).gsub('}', "}\n")
+                                             # r.to_json.gsub('}', "}\n")
+                                           end)
+      else
+        puts 'Logger type only text or json allowed'
+      end
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+
+    class OCILogger < Logger
+      def format_severity(severity)
+        SEV_LABEL[severity] || 'ANY'
+      end
+
+      def trace(message)
+        add(TRACE, message)
+      end
+    end
+
+    class LogConfigs
+      @@log_format
+      @@log_level
+
+      def initialize(l_format, l_level)
+        @@log_format = l_format
+        @@log_level = l_level
+      end
+
+      def self.log_format
+        @@log_format
+      end
+
+      def log_format
+        @@log_format
+      end
+
+      def self.log_level
+        @@log_level
+      end
+
+      def log_level
+        @@log_level
+      end
+    end
+  end
+end