fluent-plugin-oci-logging-analytics 2.0.6 → 2.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da1ab666ee3e8c8dd828f2c929290f652a9c6d8a2bda423970eb809c700639da
-  data.tar.gz: 14b8caa8d04efb8f3b38043b061dc82c28a4ace4f91661417d9c16a8ced51b37
+  metadata.gz: e5a472e64d8b068a1d4ae4d9c8007c0cfcfae37290ec188ff6a1e7a97f9b3005
+  data.tar.gz: be3762be1e1e6794672bc9ff21871a96d6a5a82053743c6432bab4ca209e0d22
 SHA512:
-  metadata.gz: d07374a5e2556eaba00f47dd68b967a06b8c1449eb38496dc76e1fc87bf63e400078494f5559b4e52d628344bf730659c3cbdd621d58c88fa74c92a742753992
-  data.tar.gz: 47ba0f5b4d056c20c4ddc0c8938e3cbc718e65e108235ee41ec6d8d31b24f9d48db6da059840f18ecb01f309c62c2b4929c48002d70e6cd04cafb83d8107cfac
+  metadata.gz: ed1e0a0ee7201c4ad6a2b0cb632fddf155778569ff47762d12407f1bf76c431d8cffb32bdb391f84b484eaf4698d1b457c2ddfcfa4e46b61a02ff550d4c3a650
+  data.tar.gz: 685a2f9fe44d8fbe8b49c64880a2f2bafe065242e1a2700b8ac3e40cb14b66ff0aa3b3a3975a9da7d62bbec203f3fb59700f950a33fbee78ab1e1e45dc1738f6

data/lib/fluent/dto/logEvents.rb

@@ -1,10 +1,10 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 class LogEvents
-  attr_accessor :entityId, :entityType, :logSourceName, :logPath, :logRecords , :metadata
+  attr_accessor :entityId, :entityType, :logSourceName, :logPath, :logRecords , :metadata, :timezone
   def initialize(lrpe_key, fluentd_records)
-    @metadata, @entityId, @entityType, @logSourceName, @logPath = lrpe_key
+    @metadata, @entityId, @entityType, @logSourceName, @logPath, @timezone = lrpe_key
     @logRecords = fluentd_records.map{ |record|
         record['message']
     }
@@ -17,8 +17,8 @@ class LogEvents
       entityType: @entityType,
       logSourceName: @logSourceName,
       logPath: @logPath,
-      logRecords: @logRecords
+      logRecords: @logRecords,
+      timezone:@timezone
     }.compact
   end
-  
 end

data/lib/fluent/dto/logEventsJson.rb

@@ -1,4 +1,4 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 require_relative './logEvents'
@@ -20,5 +20,4 @@ class LogEventsJson
       end
     }.compact
   end
-  
 end

data/lib/fluent/enums/source.rb

@@ -0,0 +1,7 @@
+## Copyright (c) 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Source
+  FLUENTD = :fluentd
+  KUBERNETES_SOLUTION = :kubernetes_solution
+end

data/lib/fluent/metrics/metricsLabels.rb

@@ -1,5 +1,8 @@
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 class MetricsLabels
-  attr_accessor :worker_id, :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency
+  attr_accessor :worker_id, :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency,:timezone
   def initialize
       @worker_id = nil
       @tag = nil
@@ -10,5 +13,6 @@ class MetricsLabels
       @records_valid = 0
       @records_per_tag = 0
       @latency = 0
+      @timezone = nil
     end
-end
+end

data/lib/fluent/metrics/prometheusMetrics.rb

@@ -1,3 +1,6 @@
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 require 'prometheus/client'
 require 'prometheus/client/registry'
 require 'prometheus/client/gauge'
@@ -35,4 +38,4 @@ class PrometheusMetrics
     registry.register(@chunk_time_to_receive) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_receive')
     registry.register(@chunk_time_to_upload) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_post')
   end
-end
+end

data/lib/fluent/plugin/out_oci-logging-analytics.rb

@@ -1,4 +1,4 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 require 'fluent/plugin/output'
@@ -7,11 +7,13 @@ require 'zip'
 require 'yajl'
 require 'yajl/json_gem'
 
+# require 'tzinfo'
 require 'logger'
 require_relative '../dto/logEventsJson'
 require_relative '../dto/logEvents'
 require_relative '../metrics/prometheusMetrics'
 require_relative '../metrics/metricsLabels'
+require_relative '../enums/source'
 
 # Import only specific OCI modules to improve load times and reduce the memory requirements.
 require 'oci/auth/auth'
@@ -36,7 +38,6 @@ require 'oci/version'
 require 'oci/waiter'
 require 'oci/retry/retry'
 require 'oci/object_storage/object_storage'
-
 module OCI
    class << self
      attr_accessor :sdk_name
@@ -91,13 +92,16 @@ module Fluent::Plugin
     config_param :endpoint,                    :string, :default => nil
     desc 'AuthType to be used.'
     config_param :auth_type,                    :string, :default => 'InstancePrincipal'
+    desc 'OCI Domain'
+    config_param :oci_domain,                    :string, :default => nil
     desc 'Enable local payload dump.'
     config_param :dump_zip_file,                    :bool, :default => false
     desc 'Payload zip File Location.'
     config_param :zip_file_location,                    :string, :default => nil
     desc 'The kubernetes_metadata_keys_mapping.'
     config_param :kubernetes_metadata_keys_mapping,                    :hash, :default => {"container_name":"Container","namespace_name":"Namespace","pod_name":"Pod","container_image":"Container Image Name","host":"Node"}
-
+    desc 'opc-meta-properties'
+    config_param :collection_source, :string, :default => Source::FLUENTD
 
     #****************************************************************
     desc 'The http proxy to be used.'
@@ -247,26 +251,62 @@ module Fluent::Plugin
         if is_valid(@config_file_location)
             @auth_type = "ConfigFile"
         end
+
         case @auth_type
-          when "InstancePrincipal"
-            instance_principals_signer = OCI::Auth::Signers::InstancePrincipalsSecurityTokenSigner.new
-            if is_valid(@endpoint)
-              @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, endpoint: @endpoint, signer: instance_principals_signer)
-              @@logger.info {"loganalytics_client initialised with endpoint: #{@endpoint}"}
-            else
-              @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, signer: instance_principals_signer)
-            end
-          when "ConfigFile"
-            my_config = OCI::ConfigFileLoader.load_config(config_file_location: @config_file_location, profile_name: @profile_name)
-            if is_valid(@endpoint)
-              @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: my_config, endpoint: @endpoint)
-              @@logger.info {"loganalytics_client initialised with endpoint: #{@endpoint}"}
-            else
-              @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config:my_config)
-            end
+        when "InstancePrincipal"
+          instance_principals_signer = nil
+          la_endpoint = nil
+          if is_valid(@oci_domain)
+            fedration_endpoint = "https://auth.#{@oci_domain}/v1/x509"
+            instance_principals_signer = OCI::Auth::Signers::InstancePrincipalsSecurityTokenSigner.new(
+              federation_endpoint: fedration_endpoint)
+            @@logger.info "Custom Federation Endpoint: #{fedration_endpoint}"
           else
-            raise Fluent::ConfigError, "Invalid authType @auth_type, authType must be either InstancePrincipal or ConfigFile."
-            abort
+            instance_principals_signer = OCI::Auth::Signers::InstancePrincipalsSecurityTokenSigner.new
+          end
+          if is_valid(@endpoint)
+            la_endpoint = @endpoint
+            @@logger.info "Initializing loganalytics_client with endpoint: #{la_endpoint}"
+          elsif is_valid(@oci_domain)
+            la_endpoint = "https://loganalytics.#{@oci_domain}"
+            @@logger.info "Initializing loganalytics_client with custom domain endpoint: #{la_endpoint}"
+          end
+          @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(
+            config: OCI::Config.new,
+            endpoint: la_endpoint,
+            signer: instance_principals_signer)
+          @@logger.info 'loganalytics_client initialized.'
+        when "WorkloadIdentity"
+          la_endpoint = nil
+          workload_identity_signer = OCI::Auth::Signers::oke_workload_resource_principal_signer
+          if is_valid(@endpoint)
+            la_endpoint = @endpoint
+            @@logger.info "Initializing loganalytics_client with endpoint: #{@endpoint}"
+          elsif is_valid(@oci_domain)
+            la_endpoint = "https://loganalytics.#{@oci_domain}"
+            @@logger.info "Initializing loganalytics_client with custom domain endpoint: #{la_endpoint}"
+          end
+          @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(
+            config: OCI::Config.new,
+            endpoint: la_endpoint,
+            signer: workload_identity_signer)
+          @@logger.info 'loganalytics_client initialized.'
+        when "ConfigFile"
+          my_config = OCI::ConfigFileLoader.load_config(
+            config_file_location: @config_file_location,
+            profile_name: @profile_name)
+          la_endpoint = nil
+          if is_valid(@endpoint)
+            la_endpoint = @endpoint
+            @@logger.info "Initializing loganalytics_client with endpoint: #{la_endpoint}"
+          elsif is_valid(@oci_domain)
+            la_endpoint = "https://loganalytics.#{@oci_domain}"
+            @@logger.info "Initializing loganalytics_client with custom domain endpoint: #{la_endpoint}"
+          end
+          @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: my_config, endpoint: la_endpoint)
+          @@logger.info 'loganalytics_client initialised'
+        else
+          raise Fluent::ConfigError, "Invalid authType: #{@auth_type}, valid inputs are -  InstancePrincipal, ConfigFile, WorkloadIdentity"
         end
 
         if is_valid(@proxy_ip) && is_number(@proxy_port)
@@ -278,13 +318,18 @@ module Fluent::Plugin
         end
 
         rescue => ex
-                @@logger.error {"Error occurred while initializing LogAnalytics Client:
-                                  authType: #{@auth_type},
-                                  errorMessage: #{ex}"}
+          @@logger.error {"Error occurred while initializing LogAnalytics Client:
+                              authType: #{@auth_type},
+                              errorMessage: #{ex}"}
     end
 
     def configure(conf)
       super
+
+      if is_valid(@oci_domain) && !@oci_domain.match(/\S.oci.\S/)
+        raise Fluent::ConfigError,  "Invalid oci_domain: #{@oci_domain}, valid fmt: <oci-region>.oci.<oci-domain> | ex: us-ashburn-1.oci.oraclecloud.com"
+      end
+
       @@prometheusMetrics = PrometheusMetrics.instance
       initialize_logger
 
@@ -628,6 +673,8 @@ module Fluent::Plugin
          latency = 0
          records_per_tag = 0
 
+
+
          tag_metrics_set = Hash.new
          logGroup_labels_set = Hash.new
 
@@ -637,8 +684,8 @@ module Fluent::Plugin
          tags_per_logGroupId = Hash.new
          tag_logSet_map = Hash.new
          tag_metadata_map = Hash.new
+         timezoneValuesByTag = Hash.new
          incoming_records = 0
-
          chunk.each do |time, record|
            incoming_records += 1
            metricsLabels = MetricsLabels.new
@@ -722,6 +769,8 @@ module Fluent::Plugin
                     end
                     next
                   end
+
+                  # metricsLabels.timezone = record["oci_la_timezone"]
                   metricsLabels.logGroupId = record["oci_la_log_group_id"]
                   metricsLabels.logSourceName = record["oci_la_log_source_name"]
                   if record["oci_la_log_set"] != nil
@@ -770,6 +819,25 @@ module Fluent::Plugin
                       tags_per_logGroupId[record["oci_la_log_group_id"]] = record["tag"]
                     end
                   end
+                  # validating the timezone field
+                   if !timezoneValuesByTag.has_key?(record["tag"])
+                     begin
+                       timezoneIdentifier = record["oci_la_timezone"]
+                       unless is_valid(timezoneIdentifier)
+                         record["oci_la_timezone"] = nil
+                       else
+                         isTimezoneExist = timezone_exist? timezoneIdentifier
+                         unless isTimezoneExist
+                           @@logger.warn { "Invalid timezone '#{timezoneIdentifier}', using default UTC." }
+                           record["oci_la_timezone"] = "UTC"
+                         end
+
+                       end
+                       timezoneValuesByTag[record["tag"]] = record["oci_la_timezone"]
+                     end
+                   else
+                     record["oci_la_timezone"] = timezoneValuesByTag[record["tag"]]
+                   end
 
                   records << record
               ensure
@@ -916,6 +984,14 @@ module Fluent::Plugin
         end
       end
     end
+    def timezone_exist?(tz)
+      begin
+        TZInfo::Timezone.get(tz)
+        return true
+      rescue TZInfo::InvalidTimezoneIdentifier
+        return false
+      end
+    end
 
     # Each oci_la_log_set will correspond to a separate file in the zip
     # Only MAX_FILES_PER_ZIP files are allowed per zip.
@@ -958,6 +1034,21 @@ module Fluent::Plugin
 
     # takes a fluentD chunk and converts it to an in-memory zipfile, populating metrics hash provided
     # Any exception raised is passed into the metrics hash, to be re-thrown from write()
+    def getCollectionSource(input)
+      collections_src = []
+      if !is_valid input
+        collections_src.unshift("source:#{Source::FLUENTD}")
+      else
+        if input == Source::FLUENTD.to_s or input == Source::KUBERNETES_SOLUTION.to_s
+          collections_src.unshift("source:#{input}")
+        else
+          # source not define ! using default source 'fluentd'
+          collections_src.unshift("source:#{Source::FLUENTD}")
+        end
+      end
+      collections_src
+    end
+
     def get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_logSet_map)
        begin
         current,  = Time.now
@@ -970,8 +1061,9 @@ module Fluent::Plugin
                   record['oci_la_metadata'],
                   record['oci_la_entity_id'],
                   record['oci_la_entity_type'],
-                  record['oci_la_log_source_name'] ,
-                  record['oci_la_log_path']
+                  record['oci_la_log_source_name'],
+                  record['oci_la_log_path'],
+                  record['oci_la_timezone']
                 ]}.map { |lrpe_key, records_per_lrpe|
                   number_of_records += records_per_lrpe.length
                   LogEvents.new(lrpe_key, records_per_lrpe)
@@ -1021,9 +1113,10 @@ module Fluent::Plugin
     # upload zipped stream to oci
     def upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream, metricsLabels_array)
       begin
+        collection_src_prop = getCollectionSource @collection_source
         error_reason = nil
         error_code = nil
-        opts = {payload_type: "ZIP"}
+        opts = { payload_type: "ZIP", opc_meta_properties:collection_src_prop}
 
             response = @@loganalytics_client.upload_log_events_file(namespace_name=@namespace,
                                             logGroupId=oci_la_log_group_id ,

data/lib/fluent/version/version.rb

@@ -0,0 +1,8 @@
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+# frozen_string_literal: true
+
+module Version
+  VERSION = "2.0.8".freeze
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-oci-logging-analytics
 version: !ruby/object:Gem::Version
-  version: 2.0.6
+  version: 2.0.8
 platform: ruby
 authors:
 - Oracle
 - 'OCI Observability: Logging Analytics'
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-05 00:00:00.000000000 Z
+date: 2024-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -131,17 +131,19 @@ extra_rdoc_files: []
 files:
 - lib/fluent/dto/logEvents.rb
 - lib/fluent/dto/logEventsJson.rb
+- lib/fluent/enums/source.rb
 - lib/fluent/metrics/metricsLabels.rb
 - lib/fluent/metrics/prometheusMetrics.rb
 - lib/fluent/plugin/out_oci-logging-analytics.rb
-homepage: 
+- lib/fluent/version/version.rb
+homepage:
 licenses:
 - UPL-1.0
 metadata:
   documentation_uri: https://docs.oracle.com/en/learn/oci_logging_analytics_fluentd/
   source_code_uri: https://github.com/oracle-quickstart/oci-logan-fluentd-output-plugin
   changelog_uri: https://github.com/oracle-quickstart/oci-logan-fluentd-output-plugin/blob/main/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -156,8 +158,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.3.27
+signing_key:
 specification_version: 4
 summary: Fluentd output plugin for OCI Logging Analytics.
 test_files: []