fluent-plugin-lm-logs-gcp 1.0.4 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fa80af2c5d4c94e6997fd770c42c45160b7ea2348ef39d30d7cbff31d774eb4
-  data.tar.gz: bdecae119e5d62ec47aa95486757e55bab5a5fac4f2d11abff003488c717d8cb
+  metadata.gz: 213a63d23d18929286a8201af426fe7e2b1ff68ac8a30cc9d50dc53cff532786
+  data.tar.gz: 9e161f20a42725baf4b32ea931cfe225516802aa0952a221a8d86ce1dbec457e
 SHA512:
-  metadata.gz: a87ab7ea7267c820609f9710b10ea1b0f812cb7e7c39aa4a75ee4e4b8b0a75bf36a58745d6f9be02f699f96956328bae19fa9e70e873a5ffea28d44a4ea3cb36
-  data.tar.gz: ba57fd1f6ea7f7b64d0525a39f826c5d1f8a5ca8fb44289dede47fc292906b3fb7a5912ff51792edc3647c0245f3ab4c87849741b162eb8355fd109e7e063aee
+  metadata.gz: 4a1e38a85f385fc408f3b74b40503ff255b2c3a3c29afa37854e4562b55ee21cc1d4c348f3ffda1f2378d22195ce995e842e15cf887d32eba73891027362fbe1
+  data.tar.gz: fd3f360ef17b0a568f30dadf56b6df6417fe725a566ece3c3e5c4af7478b2c5132fe94d0866d1a66604bd383d0ccbd66f9df903aa5df0c7d7b4b381c70fc6e78

data/README.md

@@ -7,3 +7,9 @@ This filter plugin is provided for mapping the logs coming from GCP pubsub to '_
 * Google Cloud Composer
 * Google Cloud Run
 
+## LogicMonitor properties
+
+| Property | Description |
+| --- | --- |
+| `metadata_keys` | Array of keys to be added as metadata. Filter will look for these keys in fluentd record and extract those if exist. in case of nested json, whole sub json would be added. default ` ["severity", "logName", "labels", "resource.type", "resource.labels", "httpRequest"] ` . few keys will be renamed as part of metadata standardization in Logicmonitor : ` {"trace" => "trace_id", "spanId" => "span_id","resource.type" => "_type"}`|
+| `use_default_severity` | When `true`, and log record does not have severity, severity=`DEFAULT` would be added to log. default `false`.  |

data/fluent-plugin-lm-logs-gcp.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs-gcp"
-  spec.version                        = '1.0.4'
+  spec.version                        = '1.0.6'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor with GCP logs fluentd filter plugin"
@@ -21,4 +21,5 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.0.0'
 
   spec.add_runtime_dependency "fluentd", [">= 1", "< 2"]
+  spec.add_development_dependency 'hashdiff', '>= 1.0.0'
 end

data/lib/fluent/plugin/filter_gcplm.rb

@@ -4,6 +4,12 @@ module Fluent::Plugin
   class GCPLMFilter < Filter
     Fluent::Plugin.register_filter('gcplm', self)
 
+    METADATA_KEYS_TO_RENAME = {"trace" => "trace_id", "spanId" => "span_id","resource.type" => "_type"}.freeze
+    STATIC_METADATA = {"_integration" => "gcp"}
+
+    config_param :metadata_keys, :array, default: ["severity", "logName", "labels", "resource.type", "resource.labels", "httpRequest"], value_type: :string
+    config_param :use_default_severity, :bool, default: false 
+
     def configure(conf)
       super
       # Do the usual configuration here
@@ -12,6 +18,12 @@ module Fluent::Plugin
 # This method is called when starting.
     # Open sockets or files here.
     def start
+      @final_metadata_keys = Hash.new
+      if @metadata_keys
+        @metadata_keys.each do | nested_key |
+          @final_metadata_keys[nested_key] = nested_key.to_s.split('.') 
+        end  
+      end  
       super
     end
 
@@ -64,6 +76,8 @@ module Fluent::Plugin
         resourceMap = {"system.gcp.resourcename" => record.dig("resource", "labels", "service_name"), "system.cloud.category" => 'GCP/CloudRun'}
       when 'cloud_composer_environment'
         resourceMap = {"system.gcp.resourcename" => "projects/" + project_id + "/locations/" + record.dig("resource", "labels", "location") + "/environments/" + record.dig("resource", "labels", "environment_name"), "system.cloud.category" => 'GCP/CloudComposer'}
+      when 'k8s_container'
+        resourceMap = {"system.gcp.resourcename" => record.dig("labels", "compute.googleapis.com/resource_name"), "system.cloud.category" => 'GCP/ComputeEngine'}
       end
 
       if(record.key?("protoPayload") && record.dig('protoPayload', '@type') == "type.googleapis.com/google.cloud.audit.AuditLog")
@@ -76,6 +90,27 @@ module Fluent::Plugin
       filteredRecord['message'] = message
       filteredRecord['_lm.resourceId'] = resourceMap
       filteredRecord['timestamp'] = record['timestamp']
+
+      # Add metadata
+      if @final_metadata_keys 
+        @final_metadata_keys.each do | key, value |
+          final_key = METADATA_KEYS_TO_RENAME[key] ? METADATA_KEYS_TO_RENAME[key] : key
+          nestedVal = record
+          value.each { |x| nestedVal = nestedVal[x] }
+          if nestedVal != nil
+            filteredRecord[final_key] = nestedVal
+          end  
+        end  
+      end  
+
+      # Add static metadata
+      STATIC_METADATA.each { | key, value| filteredRecord[key] = value }
+
+      # Add default severity if does not exist 
+      if !filteredRecord["severity"] and @use_default_severity
+        filteredRecord["severity"] = "DEFAULT"
+      end   
+
       filteredRecord
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs-gcp
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.6
 platform: ruby
 authors:
 - LogicMonitor
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-29 00:00:00.000000000 Z
+date: 2023-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,6 +30,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: hashdiff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: This filter plugin filters fluentd records in gcp to the configured LogicMonitor
   account.
 email: rubygems@logicmonitor.com
@@ -50,7 +64,7 @@ licenses:
 metadata:
   source_code_uri: https://github.com/logicmonitor/lm-logs-fluentd-gcp-filter
   documentation_uri: https://www.rubydoc.info/gems/lm-logs-fluentd-gcp-filter
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -65,8 +79,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: LogicMonitor with GCP logs fluentd filter plugin
 test_files: []