oci-logging-analytics-kubernetes-discovery 1.0.2 → 1.1.0

data/lib/dto/infra_objects_payload.rb

@@ -31,9 +31,9 @@ module Dto
       {
         cluster: @cluster.to_hash,
         vcn: @vcn.to_hash,
-        subnet: get_hash(@subnet, SUBNET.to_s),
-        load_balancer: @load_balancer.to_hash,
-        node_pools: get_hash(@node_pools, NODE_POOLS.to_s)
+        subnets: get_hash(@subnet, SUBNET.to_s),
+        loadbalancers: get_hash(@load_balancer, LOAD_BALANCER.to_s),
+        nodepools: get_hash(@node_pools, NODE_POOLS.to_s)
       }.compact
     end
   end

data/lib/dto/kubernetes_objects_payload.rb

@@ -8,24 +8,27 @@ module Dto
     include Enum::KubernetesObjectsEnum
     extend Util::Logging
 
-    attr_accessor :cluster, :nodes, :pods, :services, :endPointSlices, :deployments,
-                  :replicaSets, :daemonSets, :cronJobs, :jobs, :statefulSets, :events, :namespaces
+    attr_accessor :cluster, :nodes, :pods, :services, :endpointslices, :deployments,
+                  :replicasets, :daemonsets, :cronjobs, :jobs, :statefulsets, :events, :namespaces,
+                  :infra
 
-    def initialize(cluster, nodes, pods, services, endPointSlices, deployments,
-                   replicaSets, daemonSets, cronJobs, jobs, statefulSets, events, namespaces)
+    def initialize(cluster, nodes, pods, services, endpointslices, deployments,
+                   replicasets, daemonsets, cronjobs, jobs, statefulsets, events, namespaces,
+                   infra)
       @cluster = cluster
       @nodes = nodes
       @pods = pods
       @services = services
-      @endPointSlices = endPointSlices
+      @endpointslices = endpointslices
       @deployments = deployments
-      @replicaSets = replicaSets
-      @daemonSets = daemonSets
-      @cronJobs = cronJobs
+      @replicasets = replicasets
+      @daemonsets = daemonsets
+      @cronjobs = cronjobs
       @jobs = jobs
-      @statefulSets = statefulSets
+      @statefulsets = statefulsets
       @events = events
       @namespaces = namespaces
+      @infra = infra
     end
 
     def get_hash(obj, obj_name)
@@ -43,15 +46,16 @@ module Dto
         nodes: get_hash(@nodes, NODES.to_s),
         pods: get_hash(@pods, PODS.to_s),
         services: get_hash(@services, SERVICES.to_s),
-        endpointslices: get_hash(@endPointSlices, ENDPOINT_SLICES.to_s),
+        endpointslices: get_hash(@endpointslices, ENDPOINT_SLICES.to_s),
         deployments: get_hash(@deployments, DEPLOYMENTS.to_s),
-        replicasets: get_hash(@replicaSets, REPLICA_SETS.to_s),
-        daemonsets: get_hash(@daemonSets, DAEMON_SETS.to_s),
-        cronjobs: get_hash(@cronJobs, CRON_JOBS.to_s),
+        replicasets: get_hash(@replicasets, REPLICA_SETS.to_s),
+        daemonsets: get_hash(@daemonsets, DAEMON_SETS.to_s),
+        cronjobs: get_hash(@cronjobs, CRON_JOBS.to_s),
         jobs: get_hash(@jobs, JOBS.to_s),
-        statefulsets: get_hash(@statefulSets, STATEFUL_SETS.to_s),
+        statefulsets: get_hash(@statefulsets, STATEFUL_SETS.to_s),
         events: get_hash(@events, EVENTS.to_s),
-        namespaces: get_hash(@namespaces, NAMESPACES.to_s)
+        namespaces: get_hash(@namespaces, NAMESPACES.to_s),
+        infra: @infra.nil? ? nil : @infra.to_hash
       }.compact
     end
   end

data/lib/dto/state.rb

@@ -3,16 +3,20 @@
 
 module Dto
   class State
-    attr_accessor :last_timestamp
+    attr_accessor :last_timestamp, :retry_eligible_resources, :svc_log_stack_id
 
-    def initialize(last_timestamp = nil)
+    def initialize(last_timestamp = nil, retry_eligible_resources = nil, svc_log_stack_id = nil)
       @last_timestamp = last_timestamp unless last_timestamp.nil?
+      @retry_eligible_resources = retry_eligible_resources unless retry_eligible_resources.nil?
+      @svc_log_stack_id = svc_log_stack_id unless svc_log_stack_id.nil?
     end
 
     # make sure all values are strings
     def to_hash
       {
-        last_timestamp: @last_timestamp.to_s
+        last_timestamp: @last_timestamp.to_s,
+        retry_eligible_resources: @retry_eligible_resources.to_s,
+        svc_log_stack_id: @svc_log_stack_id.to_s
       }.compact
     end
   end

data/lib/enum/auth_type_enum.rb

@@ -5,5 +5,6 @@ module Enum
   module AuthTypeEnum
     CONFIG = 'config'.freeze
     INSTANCE_PRINCIPAL = 'instance_principal'.freeze
+    # OKE_WORKLOAD_IDENTITY = 'oke_workload_identity'.freeze
   end
 end

data/lib/enum/infrastructure_resource_discovery.rb

@@ -5,5 +5,6 @@ module Enum
   module InfrastructureResourceDiscovery
     SUBNET = 'subnet'.freeze
     NODE_POOLS = 'node_pools'.freeze
+    LOAD_BALANCER = 'load_balancers'.freeze
   end
 end

data/lib/enum/object_client_mapping_enum.rb

@@ -2,7 +2,7 @@
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 module Enum
-  module ObjectClientMapingEnum
+  module ObjectClientMappingEnum
     NODES = :core_client
     PODS = :core_client
     SERVICES = :core_client

data/lib/enum/stack_job_lifecycle_state_enum.rb

@@ -0,0 +1,14 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Enum
+  module StackJobLifecycleStateEnum
+    # Documentation: https://docs.oracle.com/en-us/iaas/Content/ResourceManager/Tasks/jobs.htm
+    ACCEPTED = 'ACCEPTED'.freeze
+    IN_PROGRESS = 'IN_PROGRESS'.freeze
+    FAILED = 'FAILED'.freeze
+    SUCCEEDED = 'SUCCEEDED'.freeze
+    CANCELING = 'CANCELING'.freeze
+    CANCELED = 'CANCELED'.freeze
+  end
+end

data/lib/enum/stack_job_operation_enum.rb

@@ -0,0 +1,10 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Enum
+  module StackJobOperationEnum
+    APPLY = 'APPLY'.freeze
+    PLAN = 'PLAN'.freeze
+    DESTROY = 'DESTROY'.freeze
+  end
+end

data/lib/infra_resources.rb

@@ -1,6 +1,8 @@
 ## Copyright (c) 2024 Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+require 'set'
+
 # Infra
 require_relative './discover/infrastructure'
 
@@ -8,11 +10,10 @@ require_relative './discover/infrastructure'
 require_relative './dto/infra_objects_payload'
 
 # DTOs
-require_relative './dto/infra/cluster_entity_payload'
-require_relative './dto/infra/subnet_entity_payload'
-require_relative './dto/infra/vcn_entity_payload'
-require_relative './dto/infra/load_balancers_entity_payload'
-require_relative './dto/infra/node_pool_entity_payload'
+require_relative './dto/infra/resource_payload'
+require_relative './dto/infra/subnet_payload'
+require_relative './dto/infra/node_pool_payload'
+require_relative './dto/infra/load_balancer_payload'
 
 # Util
 require_relative './util/logging'
@@ -23,66 +24,166 @@ module InfraResources
 
   module_function
 
-  def get_infra_resources(auth_object, cluster_id)
+  IP_Address = Struct.new(:ip, :isPublic)
+  Nodes = Struct.new(:id, :name, :subnetId, :privateIp, :publicIp, :lifecycleState)
+
+  def get_infra_resources(app_config, cluster_config, cluster_id, lb_ip_array)
+
     # Fetch cluster details using cluster OCID provided as argument during program run
     logger.debug('Fetching cluster details.')
-    ce_response = Discover::Infrastructure.fetch_cluster_details(auth_object, cluster_id)
-    cluster_entity_payload = Dto::Infra::ClusterEntityPayload.new(ce_response.name, ce_response.id)
+    ce_response = Discover::Infrastructure.fetch_cluster_details(cluster_id)
+    cluster_details = Dto::Infra::ResourcePayload.new(ce_response.name, ce_response.id, ce_response.compartment_id)
 
     # Fetch VCN associated with cluster
     logger.debug('Fetching virtual cloud network details.')
-    vcn_response = Discover::Infrastructure.fetch_vcn_response(auth_object, ce_response.vcn_id)
-    vcn_entity_payload = Dto::Infra::VcnEntityPayload.new(vcn_response.display_name, vcn_response.id)
+    vcn_response = Discover::Infrastructure.fetch_vcn_response(ce_response.vcn_id)
+    vcn_details = Dto::Infra::ResourcePayload.new(vcn_response.display_name, vcn_response.id, vcn_response.compartment_id)
 
-    # Fetch subnets falling under same compartment as that of cluster using VCN OCID
-    logger.debug('Fetching subnet details.')
-    subnet_response = Discover::Infrastructure.fetch_subnet_list_response(auth_object, ce_response.compartment_id, vcn_response.id)
+    # Use resource search query to fetch load balancers associated with cluster OCID
+    logger.debug('Fetching load balancers associated with cluster.')
 
-    # Loop subnet response to get individual subnets.
-    subnet_entity_payload = []
-    subnet_response.each do |subnet|
-      next if subnet.nil?
+    load_balancer_details = []
 
-      subnet_entity_payload.push(Dto::Infra::SubnetEntityPayload.new(subnet.display_name, subnet.id))
-    end
+    lb_ip_array.each do |ip|
+      structured_query = "query loadbalancer resources matching '#{ip}'"
 
-    # Use resource search query to fetch load balancers associated with cluster OCID
-    logger.debug('Fetching load balancers associated with cluster.')
-    free_text_response = Discover::Infrastructure.fetch_free_text_details(auth_object, ce_response.id)
+      resource_query_search_response = Discover::Infrastructure.search_with_structured_query(structured_query)
+
+      # TODO: We only expect one LB per IP address, remove looping over result ?
 
-    load_balancer_entity_payload = nil
-    free_text_response.items.each do |item|
-      next if item.nil?
+      # Load Balancer Lifecycle State: CREATING, ACTIVE, DELETING
+      resource_query_search_response.items.each do |item|
+        lb_response = Discover::Infrastructure.fetch_load_balancer_details(item.identifier)
+        next if lb_response[:data].nil?
+        lb_ip_addresses = []
 
-      next unless item.resource_type == 'LoadBalancer'
+        lb_response[:data].ip_addresses.each do |item|
+          lb_ip_addresses.push(IP_Address.new(item.ip_address, item.is_public))
+        end
 
-      logger.info('Fetching load balancers details.')
-      lb_response = Discover::Infrastructure.fetch_load_balancer_details(auth_object, item.identifier)
-      lb_compartment_id = lb_response.defined_tags['Oracle-Tags']['CreatedBy']
-      if lb_compartment_id == ce_response.id
-        load_balancer_entity_payload = Dto::Infra::LoadBalancerEntityPayload.new(lb_response.display_name, lb_compartment_id)
+        subnet_ids = lb_response[:data].subnet_ids.nil? ? [] : lb_response[:data].subnet_ids
+
+        load_balancer_details.push(Dto::Infra::LoadBalancerPayload.new(lb_response[:data].display_name,
+                                                                       lb_response[:data].id,
+                                                                       lb_response[:data].compartment_id,
+                                                                       lb_response[:data].is_private,
+                                                                       lb_ip_addresses,
+                                                                       lb_response[:data].lifecycle_state,
+                                                                       subnet_ids))
+
+        logger.debug("LB detected : #{lb_response}")
       end
     end
 
+    logger.info("Discovered load balancer count: #{load_balancer_details.length}")
+
     # Iterate through all the compartments (starting from root tenancy level) and fetch node pools
     # associated with the cluster OCID
-    node_pool_entity_payload = []
-    compartment_list_response = Discover::Infrastructure.fetch_compartment_list(auth_object, auth_object[:oci_config].tenancy)
-    compartment_list_response.each do |compartment|
-      next if compartment.nil?
+    node_pool_details = []
+    node_pool_subnet_ids = []
+    worker_node_subnet_ids = []
+
+    logger.debug('Fetching node pools associated with cluster.')
+
+    node_pool_compartment_id = nil
+    scoped_at_tenancy = false
+    if app_config[:probe_all_compartments]
+      # Tenancy OCID is compulsory input if probe_all_compartments flag is set
+      logger.info('Node pool discovery scope set at tenancy level.')
+
+      node_pool_compartment_id = app_config[:tenancy_ocid]
+      scoped_at_tenancy = true
+    else
+      logger.info('Node pool discovery scoped to cluster compartment.')
+      node_pool_compartment_id = ce_response.compartment_id
+    end
+
+    begin
+      compartment_list_response = Discover::Infrastructure.fetch_compartment_list(node_pool_compartment_id, scoped_at_tenancy)
+    rescue StandardError => e
+      logger.error("Error while fetching compartment list. Error: [#{e}]")
+      raise StandardError, 'Error while fetching compartment list.'
+    end
+
+    compartment_list = []
+    compartment_list_response[:data]&.each do |compartment|
+      compartment_list.push(compartment.id)
+    end
+
+    if scoped_at_tenancy
+      compartment_list.push(app_config[:tenancy_ocid])
+    else
+      compartment_list.push(ce_response.compartment_id)
+    end
+
+    compartment_list.each do |compartment|
+
+      node_pool_list_response = Discover::Infrastructure.fetch_node_pool_lists(compartment, ce_response.id)
+
+      node_pool_list_response.each do |node_pool|
+
+        subnet_ids = node_pool.subnet_ids.nil? ? [] : node_pool.subnet_ids
+
+        subnet_ids.each do |node_pool_subnet|
+          node_pool_subnet_ids.push(node_pool_subnet)
+        end
+
+        node_pool_response = Discover::Infrastructure.fetch_node_pool(node_pool.id)
+
+        nodes = []
+        node_pool_response&.nodes&.each do |node_instance|
+          nodes.push(Nodes.new(node_instance.id, node_instance.name, node_instance&.subnet_id,
+                               node_instance&.private_ip, node_instance&.public_ip, node_instance.lifecycle_state))
+        end
+
+        node_pool_details.push(Dto::Infra::NodePoolPayload.new(node_pool.name, node_pool.id,
+                                                               node_pool.compartment_id, subnet_ids, nodes))
+
+        # Fetch worker node subnet details from placement configurations
+        placement_configs_details = node_pool.node_config_details.placement_configs
+        placement_configs_details.each do |placement_config|
+          next if placement_config.nil?
+
+          worker_node_subnet_ids.push(placement_config.subnet_id)
+        end
+      end
+    end
+
+    # Fetch details of subnets associated with cluster
+    # Subnet Lifecycle State:  AVAILABLE, PROVISIONING, TERMINATED, TERMINATING, UPDATING
+    logger.debug('Fetching subnet details associated with cluster.')
+
+    subnet_resource_details = []
+    cluster_subnet_ids = []
+
+    kubernetes_api_endpoint_subnet_id = ce_response.endpoint_config.subnet_id
+    service_loadbalancer_subnet_ids = ce_response.options.service_lb_subnet_ids
+
+    cluster_subnet_ids.push(kubernetes_api_endpoint_subnet_id)
+    cluster_subnet_ids += service_loadbalancer_subnet_ids
+    cluster_subnet_ids += node_pool_subnet_ids
+    cluster_subnet_ids += worker_node_subnet_ids
+
+    # Remove duplicates. The | operator is used to compute the union of two arrays
+    # and ensures there are no duplicates. Adding the union of an array with [] does
+    # not change the original array but removes the duplicates
+    cluster_subnet_ids = cluster_subnet_ids | []
 
-      node_pool_response = Discover::Infrastructure.fetch_node_pool_lists(auth_object, compartment.id, ce_response.id)
+    cluster_subnet_ids.each do |subnet_id|
 
-      node_pool_response.each do |node_pool|
-        next if node_pool.nil?
+      subnet_details = Discover::Infrastructure.fetch_subnet_details(subnet_id)
 
-        node_pool_entity_payload.push(Dto::Infra::NodePoolEntityPayload.new(node_pool.node_image_name, node_pool.node_image_id, node_pool.compartment_id))
+      if !subnet_details[:data].nil? && subnet_details[:status] == 200
+        subnet = subnet_details[:data]
+        subnet_resource_details.push(Dto::Infra::SubnetPayload.new(subnet.display_name, subnet.id, subnet.compartment_id,
+                                                                   subnet.prohibit_internet_ingress, subnet.prohibit_public_ip_on_vnic,
+                                                                   subnet.cidr_block, subnet.lifecycle_state))
       end
     end
 
     Dto::InfraObjectsPayLoad.new(
-      cluster_entity_payload, vcn_entity_payload, subnet_entity_payload,
-      load_balancer_entity_payload, node_pool_entity_payload
+      cluster_details, vcn_details, subnet_resource_details,
+      load_balancer_details, node_pool_details
     )
   rescue StandardError => e
     logger.error("Error in getting infrastructure resources: #{e}")

data/lib/objects_resources.rb

@@ -56,7 +56,7 @@ module ObjectsResources
 
     begin
       payload_executors = Enum::KubernetesObjectsEnum.constants(false).map do |object_type|
-        if Enum::ObjectClientMapingEnum.const_defined? object_type
+        if Enum::ObjectClientMappingEnum.const_defined? object_type
           if enable_threading == true
             Concurrent::Future.execute({ executor: thread_pool }) do
               get_objects_resources(object_type)
@@ -80,11 +80,21 @@ module ObjectsResources
     end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     logger.debug("Time elapsed in collecting objects: #{(end_time - start_time).round(2)} seconds.")
 
-    kubernetes_objects_processed = Dto::KubernetesObjectsPayLoad.new(
-      cluster, @nodes[:processed], @pods[:processed], @services[:processed], @endpoint_slices[:processed],
-      @deployments[:processed], @replica_sets[:processed], @daemon_sets[:processed], @cron_jobs[:processed],
-      @jobs[:processed], @stateful_sets[:processed], @events[:processed], @namespaces[:processed]
-    )
+    kubernetes_objects_processed = {
+      cluster: cluster,
+      nodes: @nodes[:processed],
+      pods: @pods[:processed],
+      services: @services[:processed],
+      endpoint_slices: @endpoint_slices[:processed],
+      deployments: @deployments[:processed],
+      replica_sets: @replica_sets[:processed],
+      daemon_sets: @daemon_sets[:processed],
+      cron_jobs: @cron_jobs[:processed],
+      jobs: @jobs[:processed],
+      stateful_sets: @stateful_sets[:processed],
+      events: @events[:processed],
+      namespaces: @namespaces[:processed]
+    }
 
     kubernetes_objects_raw_responses = {
       nodes: @nodes[:raw],