oci-logging-analytics-kubernetes-discovery 1.0.2 → 1.1.0

data/lib/oci_loganalytics_resources_discovery.rb

@@ -16,6 +16,7 @@ require_relative './util/kube_client'
 require_relative './util/oci_clients'
 require_relative './util/log_analytics'
 require_relative './util/state_manager'
+require_relative './util/service_logs'
 
 # DTO
 require_relative './dto/payload/log_events'
@@ -39,7 +40,7 @@ module OciLogAnalyticsResourcesDiscovery
   module_function
 
   attr_accessor :cluster_config_hash, :auth_config_hash, :kube_config_hash, :app_config_hash,
-                :auth_object, :kube_clients, :oci_clients, :snapshot_id
+                :auth_object, :kube_clients, :oci_clients, :oci_region, :oci_domain, :snapshot_id
 
   @@oci_la_log_source_name = 'Kubernetes Objects Discovery Payload Logs'
   @@oci_la_log_path = 'UNDEFINED'
@@ -49,58 +50,78 @@ module OciLogAnalyticsResourcesDiscovery
     @auth_config_hash = auth_config_hash
     @kube_config_hash = kube_config_hash
     @app_config_hash = app_config_hash
+  end
 
-    # OCI.logger = Util::Logging.logger
+  def initiate_discovery
+    Util::OCIClients.initialize(@auth_config_hash)
 
-    get_auth_object
+    @auth_object = Util::OCIClients.get_auth_config_object
+    @oci_clients = Util::OCIClients.get_clients
+    @oci_region = Util::OCIClients.get_region
+    @oci_domain = Util::OCIClients.get_domain
 
-    options = { mode: app_config_hash[:mode] }
-    get_oci_clients(options)
+    @kube_clients = Util::KubeClient.create_clients(@kube_config_hash)
 
-    if @app_config_hash[:mode] == 'object'
-      @snapshot_id = Time.now.to_i
-      get_kube_clients
-      Util::StateManager.init(@cluster_config_hash[:kubernetes_resourcename_prefix], @cluster_config_hash[:kubernetes_cluster_namespace])
-      Util::KubectlOps.set_chunk_limit @app_config_hash[:chunk_limit]
-      logger.debug("Chunk limit set to - #{@app_config_hash[:chunk_limit]}")
-    end
-  end
+    Util::StateManager.init(@cluster_config_hash[:kubernetes_resourcename_prefix], @cluster_config_hash[:kubernetes_cluster_namespace])
 
-  def initiate_infra_discovery
-    logger.info('Initiating Kubernetes infrastructure discovery.')
-    begin
-      infra_objects_payload = get_infra_resources_payload
-    rescue StandardError => e
-      logger.error("Error occurred while fetching infrastructure resources. Error: #{e}")
-      raise StandardError, 'Error occurred while fetching infrastructure resources.'
-    end
+    # Collect infrastructure discovery data
+    infra_objects_payload = nil
+    infra_discovery_failed = false
 
-    logger.info('Infrastructure discovery payload created.')
-    logger.trace("Infrastructure discovery payload: \n#{Yajl.dump(infra_objects_payload.to_hash)}")
+    logger.info('Initiating Kubernetes Objects discovery.')
 
-    if @app_config_hash[:skip_upload]
-      logger.warn('--skip_upload Flag is set: Skipping payload upload to OCI Logging Analytics.')
-    else
-      if !infra_objects_payload.nil?
-        # TODO: Upload implemetation pending.
-      else
-        logger.warn('No infrastructure objects discovered.')
-      end
-    end
-  end
+    # Collect platform discovery data
+    @snapshot_id = Time.now.to_i
+    Util::KubectlOps.set_chunk_limit @app_config_hash[:chunk_limit]
+    logger.debug("Chunk limit set to - #{@app_config_hash[:chunk_limit]}")
 
-  def initiate_object_discovery
     kubernetes_objects_payload = get_objects_resources_payload
+
     if @app_config_hash[:collect_warning_events_only]
       logger.info('Only Warning event logs will be collected as the following flag is set: --collect_warning_events_only')
     end
+
     kubernetes_objects_payload = filter_event_logs(kubernetes_objects_payload,
                                                    @app_config_hash[:collect_warning_events_only])
 
     logger.info('Kubernetes objects discovery payload created.')
-    logger.trace("Object Discovery Payload: \n#{Yajl.dump(kubernetes_objects_payload[:processed_objects].to_hash)}")
     logger.trace("Object Log Collection Payload: \n#{Yajl.dump(kubernetes_objects_payload[:raw_response].to_hash)}")
 
+    if @cluster_config_hash[:enable_infra_discovery]
+      logger.info('Initiating kubernetes infrastructure discovery.')
+      begin
+        lb_ip_array = parse_lb_ips(kubernetes_objects_payload)
+        logger.info("Discovered Load Balancer IPs: #{lb_ip_array}")
+        infra_objects_payload = InfraResources.get_infra_resources(
+          @app_config_hash,
+          @cluster_config_hash,
+          @cluster_config_hash[:kubernetes_cluster_id],
+          lb_ip_array
+        )
+        logger.trace("Infrastructure discovery payload: \n#{Yajl.dump(infra_objects_payload.to_hash)}")
+        logger.info('Infrastructure discovery payload created.')
+      rescue StandardError => e
+        # Not raising an error as platform discovery still needs to continue.
+        infra_discovery_failed = true
+        logger.error("Error occurred while fetching infrastructure resources. Error: #{e}")
+        logger.info('Proceeding with object discovery.')
+      end
+    else
+      logger.info('--kubernetes_cluster_id not provided. Infrastructure discovery skipped.') # TODO: should be Warn ?
+    end
+
+    # Combine discovery and infrastructure data
+    processed_objects = kubernetes_objects_payload[:processed_objects]
+    combined_data = Dto::KubernetesObjectsPayLoad.new(
+      processed_objects[:cluster], processed_objects[:nodes], processed_objects[:pods], processed_objects[:services], processed_objects[:endpoint_slices],
+      processed_objects[:deployments], processed_objects[:replica_sets], processed_objects[:daemon_sets], processed_objects[:cron_jobs],
+      processed_objects[:jobs], processed_objects[:stateful_sets], processed_objects[:events], processed_objects[:namespaces], infra_objects_payload
+    )
+
+    logger.info('Combined discovery payload created.')
+    logger.trace("Combined discovery payload: \n#{Yajl.dump(combined_data.to_hash)}")
+
+    # Send collected data
     if @app_config_hash[:skip_upload]
       logger.warn('--skip_upload Flag is set: Skipping payload upload to OCI logging analytics.')
     else
@@ -118,7 +139,8 @@ module OciLogAnalyticsResourcesDiscovery
         upload_data_via_discovery_api(zip_object_log_payload, object_logs_opts) unless zip_object_log_payload.nil?
         logger.info('Successfully uploaded object logs to OCI')
 
-        # Update the Events tracker config map only after successful objects logs upload
+        # DEV NOTE: Update the Events tracker config map only after successful objects logs upload
+        # Carefully check how last_timestamp update is managed if you wish to move the below code
         Util::StateManager.update_state_configmap
       rescue StandardError
         logger.error('Error occurred while uploading object logs to OCI')
@@ -128,34 +150,54 @@ module OciLogAnalyticsResourcesDiscovery
       unless @cluster_config_hash[:oci_la_cluster_entity_id].nil?
         begin
           # Upload object discovery payload with relevant header parameters
-          logger.info('Uploading Object Discovery Payload to OCI')
+          logger.info('Uploading Discovery Payload to OCI')
           object_discovery_opts = {
             payload_type: 'JSON',
             discovery_data_type: 'K8S_OBJECTS',
             opc_meta_properties: "cluster-entity-id:#{@cluster_config_hash[:oci_la_cluster_entity_id]};sub-type:discovery_payload;cluster-id:#{@cluster_config_hash[:kubernetes_cluster_id]};cluster-name:#{@cluster_config_hash[:kubernetes_cluster_name]}"
           }
-          json_object_discovery_payload = Yajl.dump(kubernetes_objects_payload[:processed_objects].to_hash)
+          json_object_discovery_payload = Yajl.dump(combined_data.to_hash)
           upload_data_via_discovery_api(json_object_discovery_payload, object_discovery_opts)
-          logger.info('Successfully uploaded object discovery payload to OCI')
+          logger.info('Successfully uploaded discovery payload to OCI')
         rescue StandardError
-          logger.error('Error occurred while uploading object discovery payload to OCI')
+          logger.error('Error occurred while uploading discovery payload to OCI')
           upload_failure = true
         end
       end
 
-      raise StandardError, 'Error occurred while uploading payloads to OCI' if upload_failure
+      raise StandardError, 'Error uploading payloads to OCI' if upload_failure
     end
-  end
 
-  def get_auth_object
-    begin
-      Util::OCIClients.initialize_auth_config(@auth_config_hash)
-    rescue StandardError => e
-      logger.error("Error occurred in creating authentication object - #{e}")
-      raise e
+    # Execute Resource Manager Stack
+    if !infra_discovery_failed && @app_config_hash[:enable_service_log]
+      logger.info('Initiating service logs collection.')
+      log_analytics_entity_details = nil
+      begin
+        log_analytics_entity_details = Util::ServiceLogs.fetch_log_analytics_entity_details(
+          @cluster_config_hash[:oci_la_namespace],
+          @cluster_config_hash[:oci_la_cluster_entity_id]
+        )
+      rescue StandardError => e
+        logger.error('Error fetching log analytics entity details.')
+      end
+      unless log_analytics_entity_details.nil?
+        begin
+          logger.info('Invoking resource manager stack for service log creation')
+          Util::ServiceLogs.invoke_resource_manager_stack(infra_objects_payload,
+                                                          @app_config_hash,
+                                                          @cluster_config_hash[:oci_la_log_group_id],
+                                                          log_analytics_entity_details.compartment_id,
+                                                          @oci_region,
+                                                          @oci_domain)
+        rescue StandardError => e
+          logger.error('Error invoking/updating service logs collection')
+          raise e
+        end
+      end
+    else
+      log_msg = infra_discovery_failed ? 'No infrastructure objects discovered' : 'Flag --enable_service_log not provided'
+      logger.info("#{log_msg}. Stack operations skipped")
     end
-    @auth_object = Util::OCIClients.get_auth_config_object
-    logger.debug('Successfully loaded the OCI auth config.')
   end
 
   def get_kube_clients
@@ -165,33 +207,9 @@ module OciLogAnalyticsResourcesDiscovery
       logger.error("Error occurred in creating kubeclients - #{e}")
       raise e
     end
-    @kube_clients = Util::KubeClient.get_clients
-    logger.debug('Kubeclients created successfully.')
-  end
-
-  def get_oci_clients(options)
-    begin
-      Util::OCIClients.create_clients(@auth_object, options)
-    rescue StandardError => e
-      logger.error("Error occurred in creating OCI clients - #{e}")
-      raise e
-    end
-    @oci_clients = Util::OCIClients.get_clients
-    logger.debug('OCI clients created successfully.')
-  end
-
-  def get_infra_resources_payload
-    logger.debug('Discovering Infrastructure Resources')
-    infra_resources_payload = nil
-
-    unless @auth_object.nil?
-      begin
-        infra_resources_payload = InfraResources.get_infra_resources(@auth_object, @cluster_config_hash[:oci_la_cluster_entity_id])
-      rescue StandardError => e
-        logger.error("Error in getting infrastructure resources: #{e}")
-      end
-    end
-    infra_resources_payload
+    @kube_clients =
+    logger.debug('Kube client created successfully.')
+    Util::KubeClient.get_clients
   end
 
   def get_objects_resources_payload
@@ -205,6 +223,15 @@ module OciLogAnalyticsResourcesDiscovery
     )
   end
 
+  def parse_lb_ips(kubernetes_objects_payload)
+    ip_array = []
+    lbs = kubernetes_objects_payload[:processed_objects][:services].select { |service| service[:serviceType] == "LoadBalancer" }
+    lbs.each do |lb|
+      ip_array.append lb.loadBalancerIP unless lb.loadBalancerIP.nil?
+    end
+    ip_array
+  end
+
   def filter_event_logs(kubernetes_objects_payload, collect_warning_events_only)
     cluster_events_processed_time = Util::StateManager.state.last_timestamp
     new_processed_time = cluster_events_processed_time

data/lib/util/helper.rb

@@ -0,0 +1,15 @@
+module Util
+  module Helper
+    module_function
+    # Function to check if a value is defined in the enum
+    def enum_value_defined?(enum_module, value)
+      enum_module.constants(false).any? { |const| enum_module.const_get(const) == value }
+    end
+
+    # Function to get enum values as list
+    def enum_values(enum_module)
+      enum_module.constants(false).map { |const| enum_module.const_get(const) }
+    end
+
+  end
+end

data/lib/util/kube_client.rb

@@ -27,6 +27,7 @@ module Util
         raise e
       end
 
+      logger.debug('Kube clients created successfully.')
       set_clients(@core_client, @apps_client, @batch_client, @discover_v1_client)
     end
 

data/lib/util/kubectl_ops.rb

@@ -48,7 +48,7 @@ module Util
         case method
         when :get_objects
           if client.nil?
-            client = Util::KubeClient.get_clients[Enum::ObjectClientMapingEnum.const_get(object_type.upcase.to_s)]
+            client = Util::KubeClient.get_clients[Enum::ObjectClientMappingEnum.const_get(object_type.upcase.to_s)]
           end
           logger.debug("Fetching '#{object_type}' details from all namespaces")
           method_verb = 'GET'

data/lib/util/log_analytics.rb

@@ -33,7 +33,7 @@ module Util
                                                 upload_discovery_data_details = payload,
                                                 opts)
       rescue StandardError => e
-        logger.error("Error while uploading payload to 'Log Analytics': #{e}")
+        logger.error("Error while uploading payload to 'Log Analytics'. Error: #{e}")
         raise StandardError, "Unable to upload data to 'Log Analytics' client."
       end
       response
@@ -56,7 +56,7 @@ module Util
         region = signer.region
       else
         logger.warn("Unknown auth_type provided for Discovery API (raw request): #{auth_object[:auth_type]}")
-        raise StandardError 'Unknown auth_type for Discovery API (raw request)'
+        raise StandardError, 'Unknown auth_type for Discovery API (raw request)'
       end
 
       # Construct URL protocol, subdomain, domain and port