oci-logging-analytics-kubernetes-discovery 1.0.0

data/lib/enum/object_client_mapping_enum.rb

@@ -0,0 +1,21 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Enum
+  module ObjectClientMapingEnum
+    NODES = :core_client
+    PODS = :core_client
+    SERVICES = :core_client
+    ENDPOINT_SLICES = :discover_v1_client
+    DEPLOYMENTS = :apps_client
+    REPLICA_SETS = :apps_client
+    DAEMON_SETS = :apps_client
+    CRON_JOBS = :batch_client
+    JOBS = :batch_client
+    STATEFUL_SETS = :apps_client
+    EVENTS = :core_client
+    NAMESPACES = :core_client
+    PERSISTENT_VOLUMES = :core_client
+    PERSISTENT_VOLUME_CLAIMS = :core_client
+  end
+end

data/lib/infra_resources.rb

@@ -0,0 +1,91 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+# Infra
+require_relative './discover/infrastructure'
+
+# Objects
+require_relative './dto/infra_objects_payload'
+
+# DTOs
+require_relative './dto/infra/cluster_entity_payload'
+require_relative './dto/infra/subnet_entity_payload'
+require_relative './dto/infra/vcn_entity_payload'
+require_relative './dto/infra/load_balancers_entity_payload'
+require_relative './dto/infra/node_pool_entity_payload'
+
+# Util
+require_relative './util/logging'
+
+module InfraResources
+  extend Util::Logging
+  extend Discover::Infrastructure
+
+  module_function
+
+  def get_infra_resources(auth_object, cluster_id)
+    # Fetch cluster details using cluster OCID provided as argument during program run
+    logger.debug('Fetching cluster details.')
+    ce_response = Discover::Infrastructure.fetch_cluster_details(auth_object, cluster_id)
+    cluster_entity_payload = Dto::Infra::ClusterEntityPayload.new(ce_response.name, ce_response.id)
+
+    # Fetch VCN associated with cluster
+    logger.debug('Fetching virtual cloud network details.')
+    vcn_response = Discover::Infrastructure.fetch_vcn_response(auth_object, ce_response.vcn_id)
+    vcn_entity_payload = Dto::Infra::VcnEntityPayload.new(vcn_response.display_name, vcn_response.id)
+
+    # Fetch subnets falling under same compartment as that of cluster using VCN OCID
+    logger.debug('Fetching subnet details.')
+    subnet_response = Discover::Infrastructure.fetch_subnet_list_response(auth_object, ce_response.compartment_id, vcn_response.id)
+
+    # Loop subnet response to get individual subnets.
+    subnet_entity_payload = []
+    subnet_response.each do |subnet|
+      next if subnet.nil?
+
+      subnet_entity_payload.push(Dto::Infra::SubnetEntityPayload.new(subnet.display_name, subnet.id))
+    end
+
+    # Use resource search query to fetch load balancers associated with cluster OCID
+    logger.debug('Fetching load balancers associated with cluster.')
+    free_text_response = Discover::Infrastructure.fetch_free_text_details(auth_object, ce_response.id)
+
+    load_balancer_entity_payload = nil
+    free_text_response.items.each do |item|
+      next if item.nil?
+
+      next unless item.resource_type == 'LoadBalancer'
+
+      logger.info('Fetching load balancers details.')
+      lb_response = Discover::Infrastructure.fetch_load_balancer_details(auth_object, item.identifier)
+      lb_compartment_id = lb_response.defined_tags['Oracle-Tags']['CreatedBy']
+      if lb_compartment_id == ce_response.id
+        load_balancer_entity_payload = Dto::Infra::LoadBalancerEntityPayload.new(lb_response.display_name, lb_compartment_id)
+      end
+    end
+
+    # Iterate through all the compartments (starting from root tenancy level) and fetch node pools
+    # associated with the cluster OCID
+    node_pool_entity_payload = []
+    compartment_list_response = Discover::Infrastructure.fetch_compartment_list(auth_object, auth_object[:oci_config].tenancy)
+    compartment_list_response.each do |compartment|
+      next if compartment.nil?
+
+      node_pool_response = Discover::Infrastructure.fetch_node_pool_lists(auth_object, compartment.id, ce_response.id)
+
+      node_pool_response.each do |node_pool|
+        next if node_pool.nil?
+
+        node_pool_entity_payload.push(Dto::Infra::NodePoolEntityPayload.new(node_pool.node_image_name, node_pool.node_image_id, node_pool.compartment_id))
+      end
+    end
+
+    Dto::InfraObjectsPayLoad.new(
+      cluster_entity_payload, vcn_entity_payload, subnet_entity_payload,
+      load_balancer_entity_payload, node_pool_entity_payload
+    )
+  rescue StandardError => e
+    logger.error("Error in getting infrastructure resources: #{e}")
+    raise
+  end
+end

data/lib/objects_resources.rb

@@ -0,0 +1,174 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+# Objects
+require_relative './discover/object'
+require_relative './dto/kubernetes_objects_payload'
+
+# Enum
+require_relative './enum/kubernetes_objects_enum'
+require_relative './enum/object_client_mapping_enum'
+
+# Util
+require_relative './util/logging'
+require_relative './util/kube_client'
+
+# Threading
+require 'concurrent'
+
+module ObjectsResources
+  extend Util::Logging
+
+  module_function
+
+  attr_accessor :nodes, :pods, :services, :endpoint_slices, :deployments, :replica_sets,
+                :daemon_sets, :cron_jobs, :jobs, :stateful_sets, :events, :namespaces,
+                :persistent_volumes, :persistent_volume_claims
+
+  Cluster = Struct.new(:id, :name)
+
+  def get_objects_resources_helper(cluster_id, cluster_name, app_config)
+    cluster = Cluster.new(cluster_id, cluster_name)
+
+    enable_threading = app_config[:enable_threading]
+    if enable_threading
+      thread_count = app_config[:thread_count]
+      work_queue_size = app_config[:work_queue_size]
+
+      begin
+        thread_pool = Concurrent::FixedThreadPool.new(
+          Concurrent.processor_count,
+          min_threads: thread_count,
+          max_threads: thread_count,
+          max_queue: work_queue_size,
+          fallback_policy: :abort
+        )
+
+        logger.debug("Thread count set to '#{thread_count}' for fixed thread pool.")
+        logger.debug("Work queue size set to '#{work_queue_size}'.")
+      rescue StandardError => e
+        logger.warn("Error occurred while creating thread pool. Falling back to sequential execution. Error: #{error}")
+        enable_threading = false
+      end
+    end
+
+    start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    begin
+      payload_executors = Enum::KubernetesObjectsEnum.constants(false).map do |object_type|
+        if Enum::ObjectClientMapingEnum.const_defined? object_type
+          if enable_threading == true
+            Concurrent::Future.execute({ executor: thread_pool }) do
+              get_objects_resources(object_type)
+            end
+          else
+            get_objects_resources(object_type)
+          end
+        end
+      end
+
+      if enable_threading
+        payload_executors.map do |executor|
+          executor.value! unless executor.nil?
+        end
+      end
+    rescue StandardError => e
+      logger.error("Error while fetching object details - #{e}")
+      raise
+    end
+
+    end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    logger.debug("Time elapsed in collecting objects: #{(end_time - start_time).round(2)} seconds.")
+
+    kubernetes_objects_processed = Dto::KubernetesObjectsPayLoad.new(
+      cluster, @nodes[:processed], @pods[:processed], @services[:processed], @endpoint_slices[:processed],
+      @deployments[:processed], @replica_sets[:processed], @daemon_sets[:processed], @cron_jobs[:processed],
+      @jobs[:processed], @stateful_sets[:processed], @events[:processed], @namespaces[:processed]
+    )
+
+    kubernetes_objects_raw_responses = {
+      nodes: @nodes[:raw],
+      pods: @pods[:raw],
+      services: @services[:raw],
+      endpoint_slices: @endpoint_slices[:raw],
+      deployments: @deployments[:raw],
+      replica_sets: @replica_sets[:raw],
+      daemon_sets: @daemon_sets[:raw],
+      cron_jobs: @cron_jobs[:raw],
+      jobs: @jobs[:raw],
+      stateful_sets: @stateful_sets[:raw],
+      events: @events[:raw],
+      namespaces: @namespaces[:raw],
+      persistent_volumes: @persistent_volumes[:raw],
+      persistent_volume_claims: @persistent_volume_claims[:raw]
+    }
+
+    { processed_objects: kubernetes_objects_processed, raw_response: kubernetes_objects_raw_responses }
+  end
+
+  def get_objects_resources(object_type)
+    object_payload = object_type.downcase.to_s
+
+    case object_payload
+    when Enum::KubernetesObjectsEnum::CRON_JOBS.to_s
+      @cron_jobs = Discover::Object.get_cron_jobs
+      log_individual_payload_items(object_type.to_s, @cron_jobs[:processed])
+    when Enum::KubernetesObjectsEnum::DAEMON_SETS.to_s
+      @daemon_sets = Discover::Object.get_daemon_sets
+      log_individual_payload_items(object_type.to_s, @daemon_sets[:processed])
+    when Enum::KubernetesObjectsEnum::DEPLOYMENTS.to_s
+      @deployments = Discover::Object.get_deployments
+      log_individual_payload_items(object_type.to_s, @deployments[:processed])
+    when Enum::KubernetesObjectsEnum::ENDPOINT_SLICES.to_s
+      @endpoint_slices = Discover::Object.get_endpoint_slices
+      log_individual_payload_items(object_type.to_s, @endpoint_slices[:processed])
+    when Enum::KubernetesObjectsEnum::JOBS.to_s
+      @jobs = Discover::Object.get_jobs
+      log_individual_payload_items(object_type.to_s, @jobs[:processed])
+    when Enum::KubernetesObjectsEnum::NODES.to_s
+      @nodes = Discover::Object.get_nodes
+      log_individual_payload_items(object_type.to_s, @nodes[:processed])
+    when Enum::KubernetesObjectsEnum::PODS.to_s
+      @pods = Discover::Object.get_pods
+      log_individual_payload_items(object_type.to_s, @pods[:processed])
+    when Enum::KubernetesObjectsEnum::REPLICA_SETS.to_s
+      @replica_sets = Discover::Object.get_replica_sets
+      log_individual_payload_items(object_type.to_s, @replica_sets[:processed])
+    when Enum::KubernetesObjectsEnum::SERVICES.to_s
+      @services = Discover::Object.get_services
+      log_individual_payload_items(object_type.to_s, @services[:processed])
+    when Enum::KubernetesObjectsEnum::STATEFUL_SETS.to_s
+      @stateful_sets = Discover::Object.get_stateful_sets
+      log_individual_payload_items(object_type.to_s, @stateful_sets[:processed])
+    when Enum::KubernetesObjectsEnum::EVENTS.to_s
+      @events = Discover::Object.get_events
+      log_individual_payload_items(object_type.to_s, @events[:processed])
+    when Enum::KubernetesObjectsEnum::NAMESPACES.to_s
+      @namespaces = Discover::Object.get_namespaces
+      log_individual_payload_items(object_type.to_s, @namespaces[:processed])
+    when Enum::KubernetesObjectsEnum::PERSISTENT_VOLUMES.to_s
+      @persistent_volumes = Discover::Object.get_persistent_volumes
+      unless @persistent_volumes[:processed].nil?
+        log_individual_payload_items(object_type.to_s, @persistent_volumes[:processed])
+      end
+    when Enum::KubernetesObjectsEnum::PERSISTENT_VOLUME_CLAIMS.to_s
+      @persistent_volume_claims = Discover::Object.get_persistent_volume_claims
+      unless @persistent_volume_claims[:processed].nil?
+        log_individual_payload_items(object_type.to_s, @persistent_volume_claims[:processed])
+      end
+    else
+      logger.error("Object type '#{object_payload}' not supported")
+    end
+  end
+
+  def log_individual_payload_items(object_name, arr)
+    logger.info("Total count of '#{object_name}': #{arr.length}")
+
+    return if arr.empty?
+
+    logger.trace("Iterating over each '#{object_name}'.")
+    arr.each do |item|
+      logger.trace("#{object_name}: #{item.to_h.to_json}")
+    end
+  end
+end

data/lib/oci_loganalytics_resources_discovery.rb

@@ -0,0 +1,293 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+# Libraries
+require 'kubeclient'
+require 'zip'
+require 'yajl'
+
+# Resources
+require_relative './infra_resources'
+require_relative './objects_resources'
+
+# Util
+require_relative './util/logging'
+require_relative './util/kube_client'
+require_relative './util/oci_clients'
+require_relative './util/log_analytics'
+require_relative './util/state_manager'
+
+# DTO
+require_relative './dto/payload/log_events'
+require_relative './dto/payload/log_events_json'
+
+module OCI
+  class << self
+    attr_accessor :sdk_name
+
+    # Defines the logger used for debugging for the OCI module.
+    # For example, log to STDOUT by setting this to Logger.new(STDOUT).
+    #
+    # @return [Logger]
+    attr_accessor :logger
+  end
+end
+
+module OciLogAnalyticsResourcesDiscovery
+  extend Util::Logging
+
+  module_function
+
+  attr_accessor :cluster_config_hash, :auth_config_hash, :kube_config_hash, :app_config_hash,
+                :auth_object, :kube_clients, :oci_clients, :snapshot_id
+
+  @@oci_la_log_source_name = 'Kubernetes Objects Discovery Payload Logs'
+  @@oci_la_log_path = 'UNDEFINED'
+
+  def initialize(cluster_config_hash:, auth_config_hash:, kube_config_hash:, app_config_hash:)
+    @cluster_config_hash = cluster_config_hash
+    @auth_config_hash = auth_config_hash
+    @kube_config_hash = kube_config_hash
+    @app_config_hash = app_config_hash
+
+    # OCI.logger = Util::Logging.logger
+
+    get_auth_object
+
+    options = { mode: app_config_hash[:mode] }
+    get_oci_clients(options)
+
+    if @app_config_hash[:mode] == 'object'
+      @snapshot_id = Time.now.to_i
+      get_kube_clients
+      Util::StateManager.init(@cluster_config_hash[:kubernetes_resourcename_prefix], @cluster_config_hash[:kubernetes_cluster_namespace])
+      Util::KubectlOps.set_chunk_limit @app_config_hash[:chunk_limit]
+      logger.debug("Chunk limit set to - #{@app_config_hash[:chunk_limit]}")
+    end
+  end
+
+  def initiate_infra_discovery
+    logger.info('Initiating Kubernetes infrastructure discovery.')
+    begin
+      infra_objects_payload = get_infra_resources_payload
+    rescue StandardError => e
+      logger.error("Error occurred while fetching infrastructure resources. Error: #{e}")
+      raise StandardError, 'Error occurred while fetching infrastructure resources.'
+    end
+
+    logger.info('Infrastructure discovery payload created.')
+    logger.trace("Infrastructure discovery payload: \n#{Yajl.dump(infra_objects_payload.to_hash)}")
+
+    if @app_config_hash[:skip_upload]
+      logger.warn('--skip_upload Flag is set: Skipping payload upload to OCI Logging Analytics.')
+    else
+      if !infra_objects_payload.nil?
+        # TODO: Upload implemetation pending.
+      else
+        logger.warn('No infrastructure objects discovered.')
+      end
+    end
+  end
+
+  def initiate_object_discovery
+    kubernetes_objects_payload = get_objects_resources_payload
+    if @app_config_hash[:collect_warning_events_only]
+      logger.info('Only Warning event logs will be collected as the following flag is set: --collect_warning_events_only')
+    end
+    kubernetes_objects_payload = filter_event_logs(kubernetes_objects_payload,
+                                                   @app_config_hash[:collect_warning_events_only])
+
+    logger.info('Kubernetes objects discovery payload created.')
+    logger.trace("Object Discovery Payload: \n#{Yajl.dump(kubernetes_objects_payload[:processed_objects].to_hash)}")
+    logger.trace("Object Log Collection Payload: \n#{Yajl.dump(kubernetes_objects_payload[:raw_response].to_hash)}")
+
+    if @app_config_hash[:skip_upload]
+      logger.warn('--skip_upload Flag is set: Skipping payload upload to OCI logging analytics.')
+    else
+      upload_failure = false
+
+      begin
+        # Upload object logs with relevant header parameters
+        logger.info('Uploading Object Logs to OCI')
+        object_logs_opts = {
+          payload_type: 'ZIP',
+          discovery_data_type: 'K8S_OBJECTS',
+          opc_meta_properties: "cluster-entity-id:#{@cluster_config_hash[:oci_la_cluster_entity_id]};sub-type:object_logs;snapshot-id:#{@snapshot_id};log-group-id:#{@cluster_config_hash[:oci_la_log_group_id]};cluster-id:#{@cluster_config_hash[:kubernetes_cluster_id]};cluster-name:#{@cluster_config_hash[:kubernetes_cluster_name]}"
+        }
+        zip_object_log_payload = get_zippedstream(kubernetes_objects_payload)
+        upload_data_via_discovery_api(zip_object_log_payload, object_logs_opts) unless zip_object_log_payload.nil?
+        logger.info('Successfully uploaded object logs to OCI')
+
+        # Update the Events tracker config map only after successful objects logs upload
+        Util::StateManager.update_state_configmap
+      rescue StandardError
+        logger.error('Error occurred while uploading object logs to OCI')
+        upload_failure = true
+      end
+
+      unless @cluster_config_hash[:oci_la_cluster_entity_id].nil?
+        begin
+          # Upload object discovery payload with relevant header parameters
+          logger.info('Uploading Object Discovery Payload to OCI')
+          object_discovery_opts = {
+            payload_type: 'JSON',
+            discovery_data_type: 'K8S_OBJECTS',
+            opc_meta_properties: "cluster-entity-id:#{@cluster_config_hash[:oci_la_cluster_entity_id]};sub-type:discovery_payload;cluster-id:#{@cluster_config_hash[:kubernetes_cluster_id]};cluster-name:#{@cluster_config_hash[:kubernetes_cluster_name]}"
+          }
+          json_object_discovery_payload = Yajl.dump(kubernetes_objects_payload[:processed_objects].to_hash)
+          upload_data_via_discovery_api(json_object_discovery_payload, object_discovery_opts)
+          logger.info('Successfully uploaded object discovery payload to OCI')
+        rescue StandardError
+          logger.error('Error occurred while uploading object discovery payload to OCI')
+          upload_failure = true
+        end
+      end
+
+      raise StandardError, 'Error occurred while uploading payloads to OCI' if upload_failure
+    end
+  end
+
+  def get_auth_object
+    begin
+      Util::OCIClients.initialize_auth_config(@auth_config_hash)
+    rescue StandardError => e
+      logger.error("Error occurred in creating authentication object - #{e}")
+      raise e
+    end
+    @auth_object = Util::OCIClients.get_auth_config_object
+    logger.debug('Successfully loaded the OCI auth config.')
+  end
+
+  def get_kube_clients
+    begin
+      Util::KubeClient.create_clients(@kube_config_hash)
+    rescue StandardError => e
+      logger.error("Error occurred in creating kubeclients - #{e}")
+      raise e
+    end
+    @kube_clients = Util::KubeClient.get_clients
+    logger.debug('Kubeclients created successfully.')
+  end
+
+  def get_oci_clients(options)
+    begin
+      Util::OCIClients.create_clients(@auth_object, options)
+    rescue StandardError => e
+      logger.error("Error occurred in creating OCI clients - #{e}")
+      raise e
+    end
+    @oci_clients = Util::OCIClients.get_clients
+    logger.debug('OCI clients created successfully.')
+  end
+
+  def get_infra_resources_payload
+    logger.debug('Discovering Infrastructure Resources')
+    infra_resources_payload = nil
+
+    unless @auth_object.nil?
+      begin
+        infra_resources_payload = InfraResources.get_infra_resources(@auth_object, @cluster_config_hash[:oci_la_cluster_entity_id])
+      rescue StandardError => e
+        logger.error("Error in getting infrastructure resources: #{e}")
+      end
+    end
+    infra_resources_payload
+  end
+
+  def get_objects_resources_payload
+    logger.info('Discovering Kubernetes Objects.')
+    raise StandardError, 'Kubeclients not initialized.' if @kube_clients.nil?
+
+    ObjectsResources.get_objects_resources_helper(
+      @cluster_config_hash[:kubernetes_cluster_id],
+      @cluster_config_hash[:kubernetes_cluster_name],
+      @app_config_hash
+    )
+  end
+
+  def filter_event_logs(kubernetes_objects_payload, collect_warning_events_only)
+    cluster_events_processed_time = Util::StateManager.state.last_timestamp
+    new_processed_time = cluster_events_processed_time
+    filtered_events = []
+    logger.info("Filtering events with EPOCH cut-off : #{cluster_events_processed_time} - (#{Time.at(cluster_events_processed_time).getutc})")
+    begin
+      eventlist_response = JSON.parse(kubernetes_objects_payload[:raw_response][:events], symbolize_names: true)
+      eventslist = eventlist_response[:items]
+      eventslist.each do |event|
+        # Skip all non warning events if collect_warning_events_only == true
+        next if collect_warning_events_only && event[:type] != 'Warning'
+
+        # if lastTimeStamp is present, use it, otherwise fallback to creationTimestamp
+        event_timestamp = if !event[:lastTimestamp].nil?
+                            event[:lastTimestamp]
+                          elsif !event[:metadata][:creationTimestamp].nil?
+                            event[:metadata][:creationTimestamp]
+                          else
+                            logger.warn("Not able to find event timestamp in following event (uid) - #{event[:uid]}")
+                            next
+                          end
+
+        event_timestamp = Time.parse(event_timestamp).to_i
+
+        if event_timestamp > cluster_events_processed_time
+          filtered_events.push(event)
+          new_processed_time = event_timestamp if event_timestamp > new_processed_time
+        end
+      end
+      logger.info("Event Filter Stats: #{filtered_events.length} out of #{eventslist.length} events included.")
+      Util::StateManager.state.last_timestamp = new_processed_time
+    rescue StandardError => e
+      logger.error("Failed to filter event logs - #{e}")
+      raise e
+    end
+    eventlist_response[:items] = filtered_events
+    kubernetes_objects_payload[:raw_response][:events] = JSON.dump(eventlist_response)
+    kubernetes_objects_payload
+  end
+
+  def get_zippedstream(kubernetes_object_payload)
+    logger.debug('Creating Zipped stream for object log collection payload')
+    begin
+      zippedstream = Zip::OutputStream.write_buffer do |zos|
+        kubernetes_object_payload[:raw_response].each do |key, value|
+          next if value.nil? || value.empty?
+
+          next_entry = "k8s.#{key}.json"
+          zos.put_next_entry(next_entry)
+
+          zos.write(value)
+        end
+      end
+      zippedstream.rewind
+    rescue StandardError => e
+      logger.error("Failed to generate zipped stream of discovery payload - #{e}")
+      raise e
+    end
+    zippedstream
+  end
+
+  def upload_data_via_discovery_api(payload, opts)
+    response = if @app_config_hash[:enable_raw_request]
+                 Util::LogAnalytics.upload_discovery_data_raw_request(@cluster_config_hash[:oci_la_namespace],
+                                                               payload,
+                                                               opts,
+                                                               @auth_object)
+               else
+                 Util::LogAnalytics.upload_discovery_data(@cluster_config_hash[:oci_la_namespace],
+                                                   payload,
+                                                   opts)
+               end
+    unless response.nil?
+      resp_headers = response.headers
+      logger.info("Response OPC object ID: #{resp_headers['opc-object-id']}")
+      logger.info("Response OPC request ID: #{resp_headers['opc-request-id']}")
+    end
+  rescue OCI::Errors::ServiceError => e
+    error_code = e.status_code
+    logger.error("Error code '#{error_code}' encountered while uploading payload, error: #{e}")
+    raise e
+  rescue StandardError => e
+    logger.error("Error in uploading data - #{e}")
+    raise e
+  end
+end