oci-logging-analytics-kubernetes-discovery 1.0.0

data/lib/discover/object.rb

@@ -0,0 +1,347 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require_relative '../util/logging'
+require_relative '../util/kubectl_ops'
+
+# Util
+require_relative '../util/kube_client'
+
+# Enum
+require_relative '../enum/kubernetes_objects_enum'
+
+module Discover
+  module Object
+    extend Util::Logging
+
+    module_function
+
+    Controller = Struct.new(:kind, :name, :uid)
+    Generic_object = Struct.new(:name, :namespace, :uid)
+    End_point_slice = Struct.new(:name, :namespace, :uid, :endpoints, :ports)
+    Job = Struct.new(:name, :namespace, :uid, :controller)
+    Node = Struct.new(:name, :displayName, :uid, :internalIP, :externalIP)
+    Pod = Struct.new(:name, :namespace, :uid, :podIP, :hostIP, :nodeIP, :controller)
+    Replica_set = Struct.new(:name, :namespace, :uid, :controller)
+    Service = Struct.new(:name, :namespace, :uid, :clusterIP, :externalIP, :loadBalancerIP, :ports)
+    Namespace = Struct.new(:name, :uid)
+
+    def get_generic_object_details(item, object_type)
+      if item.nil? || item[:metadata].nil?
+        logger.debug("Null or empty object detected, skipping the processing for object '#{object_type}'.")
+        return nil
+      end
+      name = item[:metadata][:name]
+      namespace = item[:metadata][:namespace]
+      uid = item[:metadata][:uid]
+      Generic_object.new(name, namespace, uid)
+    end
+
+    def get_cron_jobs
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::CRON_JOBS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          cron_job = get_generic_object_details(item, resource_name)
+          processed_data << cron_job unless cron_job.nil?
+        end
+      rescue StandardError => e
+        logger.error("Unable to parse '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_daemon_sets
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::DAEMON_SETS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          daemon_set = get_generic_object_details(item, resource_name)
+          processed_data << daemon_set unless daemon_set.nil?
+        end
+      rescue StandardError => e
+        logger.error("Unable to parse '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_deployments
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::DEPLOYMENTS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          deployment = get_generic_object_details(item, resource_name)
+          processed_data << deployment unless deployment.nil?
+        end
+      rescue StandardError => e
+        logger.error("Unable to parse '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_endpoint_slices
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::ENDPOINT_SLICES.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          generic_object = get_generic_object_details(item, resource_name)
+          next if generic_object.nil?
+
+          endpoints = item[:endpoints]
+          endpoint_arr = []
+          if !endpoints.nil? && !endpoints.empty?
+            endpoints.each do |endpoint_item|
+              next if endpoint_item.nil?
+
+              endpoint_arr.push(endpoint_item.to_h)
+            end
+          end
+
+          ports = item[:ports]
+          port_arr = []
+          if !ports.nil? && !ports.empty?
+            ports.each do |port_item|
+              next if port_item.nil?
+
+              port_arr.push(port_item.to_h)
+            end
+          end
+          processed_data.push(End_point_slice.new(generic_object[:name], generic_object[:namespace], generic_object[:uid], endpoint_arr, port_arr))
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_jobs
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::JOBS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          generic_object = get_generic_object_details(item, resource_name)
+          next if generic_object.nil?
+
+          controller = nil
+          if !item[:metadata][:ownerReferences].nil? && !item[:metadata][:ownerReferences].empty?
+            item[:metadata][:ownerReferences].each do |owner_reference|
+              next if owner_reference.nil?
+
+              if owner_reference[:controller]
+                controller = Controller.new(owner_reference[:kind], owner_reference[:name], owner_reference[:uid])
+              end
+            end
+          end
+          processed_data.push(Job.new(generic_object[:name], generic_object[:namespace], generic_object[:uid], controller.to_h))
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_nodes
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::NODES.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          next if item.nil? || item.empty?
+
+          name = item[:metadata][:name]
+
+          displayName = nil
+          if !item[:metadata][:labels].nil? || !item[:metadata][:labels].empty?
+            displayName = item[:metadata][:labels][:displayName].nil? ? nil : item[:metadata][:labels][:displayName]
+          end
+
+          uid = item[:metadata][:uid]
+          addresses = item[:status][:addresses]
+          internalIP = nil
+          externalIP = nil
+          addresses.each do |address|
+            next if address.nil?
+
+            if address.to_h[:type].eql?('InternalIP')
+              internalIP = address.to_hash[:address]
+            elsif address.to_h[:type].eql?('ExternalIP')
+              externalIP = address.to_hash[:address]
+            end
+          end
+          processed_data.push(Node.new(name, displayName, uid, internalIP, externalIP))
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_pods
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::PODS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          generic_object = get_generic_object_details(item, resource_name)
+          next if generic_object.nil?
+
+          podIP = item[:status][:podIP]
+          hostIP = item[:status][:hostIP]
+          nodeIP = item[:status][:nodeIP]
+          controller = nil
+          if !item[:metadata][:ownerReferences].nil? && !item[:metadata][:ownerReferences].empty?
+            item[:metadata][:ownerReferences].each do |owner_reference|
+              next if owner_reference.nil?
+
+              if owner_reference[:controller]
+                controller = Controller.new(owner_reference[:kind], owner_reference[:name], owner_reference[:uid])
+              end
+            end
+          end
+          processed_data.push(Pod.new(generic_object[:name], generic_object[:namespace], generic_object[:uid], podIP, hostIP, nodeIP, controller.to_h))
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_replica_sets
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::REPLICA_SETS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          generic_object = get_generic_object_details(item, resource_name)
+          next if generic_object.nil?
+
+          controller = nil
+          if !item[:metadata][:ownerReferences].nil? && !item[:metadata][:ownerReferences].empty?
+            item[:metadata][:ownerReferences].each do |owner_reference|
+              next if owner_reference.nil?
+
+              if owner_reference[:controller]
+                controller = Controller.new(owner_reference[:kind], owner_reference[:name], owner_reference[:uid])
+              end
+            end
+          end
+          processed_data.push(Replica_set.new(generic_object[:name], generic_object[:namespace], generic_object[:uid], controller.to_h))
+        end
+      rescue StandardError => e
+        logger.error "Unable to fetch '#{resource_name}'"
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_services
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::SERVICES.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          generic_object = get_generic_object_details(item, resource_name)
+          next if generic_object.nil?
+
+          clusterIP = item[:spec][:clusterIP]
+          externalIP = item[:spec][:externalIP]
+          loadBalancerIP = item[:status][:loadBalancer][:ingress].nil? ? nil : item[:status][:loadBalancer][:ingress][0][:ip]
+          ports = item[:spec][:ports]
+          ports_arr = []
+          if !ports.nil? && !ports.empty?
+            ports.each do |port_item|
+              next if port_item.nil? || port_item.empty?
+
+              ports_arr.push(port_item.to_h)
+            end
+          end
+          processed_data.push(Service.new(generic_object[:name], generic_object[:namespace], generic_object[:uid], clusterIP, externalIP, loadBalancerIP, ports_arr))
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_stateful_sets
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::STATEFUL_SETS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          stateful_set = get_generic_object_details(item, resource_name)
+          processed_data << stateful_set unless stateful_set.nil?
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_events
+      # Note - only collects warning logs for processed/discovery data
+      #      - collects all events (incl. warning) in raw paylaod
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::EVENTS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |event|
+          # filter out empty and warning events
+          next if event.nil? || event.empty? || event[:type] != 'Warning'
+
+          processed_data.push(event)
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_namespaces
+      processed_data = []
+      resource_name = Enum::KubernetesObjectsEnum::NAMESPACES.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      begin
+        response[:parsed_symbolized][:items].each do |item|
+          next if item.nil? || item.empty?
+
+          processed_data.push(Namespace.new(item[:metadata][:name], item[:metadata][:uid]))
+        end
+      rescue StandardError => e
+        logger.error("Unable to fetch '#{resource_name}'")
+        raise e
+      end
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_persistent_volumes
+      processed_data = nil
+      resource_name = Enum::KubernetesObjectsEnum::PERSISTENT_VOLUMES.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      { processed: processed_data, raw: response[:raw] }
+    end
+
+    def get_persistent_volume_claims
+      processed_data = nil
+      resource_name = Enum::KubernetesObjectsEnum::PERSISTENT_VOLUME_CLAIMS.to_s
+      response = Util::KubectlOps.get_objects(resource_name)
+      { processed: processed_data, raw: response[:raw] }
+    end
+  end
+end

data/lib/dto/infra/cluster_entity_payload.rb

@@ -0,0 +1,22 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Dto
+  module Infra
+    class ClusterEntityPayload
+      attr_accessor :name, :id
+
+      def initialize(name, id)
+        @name = name
+        @id = id
+      end
+
+      def to_hash
+        {
+          name: @name,
+          id: @id
+        }.compact
+      end
+    end
+  end
+end

data/lib/dto/infra/load_balancers_entity_payload.rb

@@ -0,0 +1,22 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Dto
+  module Infra
+    class LoadBalancerEntityPayload
+      attr_accessor :name, :id
+
+      def initialize(name, id)
+        @name = name
+        @id = id
+      end
+
+      def to_hash
+        {
+          name: @name,
+          id: @id
+        }.compact
+      end
+    end
+  end
+end

data/lib/dto/infra/node_pool_entity_payload.rb

@@ -0,0 +1,24 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Dto
+  module Infra
+    class NodePoolEntityPayload
+      attr_accessor :name, :id, :compartment_id
+
+      def initialize(name, id, compartment_id)
+        @name = name
+        @id = id
+        @compartment_id = compartment_id
+      end
+
+      def to_hash
+        {
+          name: @name,
+          id: @id,
+          compartment_id: @compartment_id
+        }.compact
+      end
+    end
+  end
+end

data/lib/dto/infra/subnet_entity_payload.rb

@@ -0,0 +1,22 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Dto
+  module Infra
+    class SubnetEntityPayload
+      attr_accessor :name, :id
+
+      def initialize(name, id)
+        @name = name
+        @id = id
+      end
+
+      def to_hash
+        {
+          name: @name,
+          id: @id
+        }.compact
+      end
+    end
+  end
+end

data/lib/dto/infra/vcn_entity_payload.rb

@@ -0,0 +1,22 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Dto
+  module Infra
+    class VcnEntityPayload
+      attr_accessor :name, :id
+
+      def initialize(name, id)
+        @name = name
+        @id = id
+      end
+
+      def to_hash
+        {
+          name: @name,
+          id: @id
+        }.compact
+      end
+    end
+  end
+end

data/lib/dto/infra_objects_payload.rb

@@ -0,0 +1,40 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require_relative '../enum/infrastructure_resource_discovery'
+
+module Dto
+  class InfraObjectsPayLoad
+    include Enum::InfrastructureResourceDiscovery
+    extend Util::Logging
+
+    attr_accessor :cluster, :vcn, :subnet, :load_balancer, :node_pools
+
+    def initialize(cluster, vcn, subnet, load_balancer, node_pools)
+      @cluster = cluster
+      @vcn = vcn
+      @subnet = subnet
+      @load_balancer = load_balancer
+      @node_pools = node_pools
+    end
+
+    def get_hash(obj, obj_name)
+      if !obj.nil?
+        obj.map(&:to_hash)
+      else
+        Util::Logging.logger.error("Cannot generate hash for #{obj_name}: as obj is nil")
+        nil
+      end
+    end
+
+    def to_hash
+      {
+        cluster: @cluster.to_hash,
+        vcn: @vcn.to_hash,
+        subnet: get_hash(@subnet, SUBNET.to_s),
+        load_balancer: @load_balancer.to_hash,
+        node_pools: get_hash(@node_pools, NODE_POOLS.to_s)
+      }.compact
+    end
+  end
+end

data/lib/dto/kubernetes_objects_payload.rb

@@ -0,0 +1,58 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require_relative '../enum/kubernetes_objects_enum'
+
+module Dto
+  class KubernetesObjectsPayLoad
+    include Enum::KubernetesObjectsEnum
+    extend Util::Logging
+
+    attr_accessor :cluster, :nodes, :pods, :services, :endPointSlices, :deployments,
+                  :replicaSets, :daemonSets, :cronJobs, :jobs, :statefulSets, :events, :namespaces
+
+    def initialize(cluster, nodes, pods, services, endPointSlices, deployments,
+                   replicaSets, daemonSets, cronJobs, jobs, statefulSets, events, namespaces)
+      @cluster = cluster
+      @nodes = nodes
+      @pods = pods
+      @services = services
+      @endPointSlices = endPointSlices
+      @deployments = deployments
+      @replicaSets = replicaSets
+      @daemonSets = daemonSets
+      @cronJobs = cronJobs
+      @jobs = jobs
+      @statefulSets = statefulSets
+      @events = events
+      @namespaces = namespaces
+    end
+
+    def get_hash(obj, obj_name)
+      if !obj.nil?
+        obj.map(&:to_h)
+      else
+        Util::Logging.logger.error("Could not generate hash for #{obj_name}: as obj is nil")
+        nil
+      end
+    end
+
+    def to_hash
+      {
+        cluster: @cluster.nil? ? nil : @cluster.to_h,
+        nodes: get_hash(@nodes, NODES.to_s),
+        pods: get_hash(@pods, PODS.to_s),
+        services: get_hash(@services, SERVICES.to_s),
+        endpointslices: get_hash(@endPointSlices, ENDPOINT_SLICES.to_s),
+        deployments: get_hash(@deployments, DEPLOYMENTS.to_s),
+        replicasets: get_hash(@replicaSets, REPLICA_SETS.to_s),
+        daemonsets: get_hash(@daemonSets, DAEMON_SETS.to_s),
+        cronjobs: get_hash(@cronJobs, CRON_JOBS.to_s),
+        jobs: get_hash(@jobs, JOBS.to_s),
+        statefulsets: get_hash(@statefulSets, STATEFUL_SETS.to_s),
+        events: get_hash(@events, EVENTS.to_s),
+        namespaces: get_hash(@namespaces, NAMESPACES.to_s)
+      }.compact
+    end
+  end
+end

data/lib/dto/payload/log_events.rb

@@ -0,0 +1,26 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Dto
+  module Payload
+    class LogEvents
+      attr_accessor :metadata, :logSourceName, :logPath, :logRecords
+
+      def initialize(metadata, log_source_name, log_path, records)
+        @metadata = metadata
+        @logSourceName = log_source_name
+        @logPath = log_path
+        @logRecords = Array.new(1, records['message'])
+      end
+
+      def to_hash
+        {
+          metadata: @metadata,
+          logSourceName: @logSourceName,
+          logPath: @logPath,
+          logRecords: @logRecords
+        }.compact
+      end
+    end
+  end
+end

data/lib/dto/payload/log_events_json.rb

@@ -0,0 +1,22 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+require_relative './log_events'
+
+module Dto
+  module Payload
+    class LogEventsJson
+      attr_accessor :LogEvents
+
+      def initialize(logEvents)
+        @LogEvents = Array.new(1, logEvents)
+      end
+
+      def to_hash
+        {
+          logEvents: @LogEvents.map(&:to_hash)
+        }.compact
+      end
+    end
+  end
+end

data/lib/dto/state.rb

@@ -0,0 +1,19 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Dto
+  class State
+    attr_accessor :last_timestamp
+
+    def initialize(last_timestamp = nil)
+      @last_timestamp = last_timestamp unless last_timestamp.nil?
+    end
+
+    # make sure all values are strings
+    def to_hash
+      {
+        last_timestamp: @last_timestamp.to_s
+      }.compact
+    end
+  end
+end

data/lib/enum/auth_type_enum.rb

@@ -0,0 +1,9 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Enum
+  module AuthTypeEnum
+    CONFIG = 'config'.freeze
+    INSTANCE_PRINCIPAL = 'instance_principal'.freeze
+  end
+end

data/lib/enum/infrastructure_resource_discovery.rb

@@ -0,0 +1,9 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Enum
+  module InfrastructureResourceDiscovery
+    SUBNET = 'subnet'.freeze
+    NODE_POOLS = 'node_pools'.freeze
+  end
+end

data/lib/enum/kubernetes_objects_enum.rb

@@ -0,0 +1,22 @@
+## Copyright (c) 2024 Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Enum
+  module KubernetesObjectsEnum
+    CLUSTER = 'cluster'.freeze
+    NODES = 'nodes'.freeze
+    PODS = 'pods'.freeze
+    SERVICES = 'services'.freeze
+    ENDPOINT_SLICES = 'endpoint_slices'.freeze
+    DEPLOYMENTS = 'deployments'.freeze
+    REPLICA_SETS = 'replica_sets'.freeze
+    DAEMON_SETS = 'daemon_sets'.freeze
+    CRON_JOBS = 'cron_jobs'.freeze
+    JOBS = 'jobs'.freeze
+    STATEFUL_SETS = 'stateful_sets'.freeze
+    EVENTS = 'events'.freeze
+    NAMESPACES = 'namespaces'.freeze
+    PERSISTENT_VOLUMES = 'persistent_volumes'.freeze
+    PERSISTENT_VOLUME_CLAIMS = 'persistent_volume_claims'.freeze
+  end
+end