fluent-plugin-lm-logs-gcp 1.0.3 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05a8a2120e8eee056999b25aab7d56ed8dc7c14abc60c6f66a1d8dd92975dfcf
-  data.tar.gz: c37ede1a847c00c1640ab41d9077cdb1ab6c2ad56d55abd33090ed5c3618282f
+  metadata.gz: 10b17ee7b034c6e827b5213c154e07ac6098f5976ef6614c6b1b6922a156e996
+  data.tar.gz: 77a5ff349177a50024a8223ce723603580eaedefed302e30674f94b924def144
 SHA512:
-  metadata.gz: 543af75d4d79de61ba51866d8df21ae6ab25d9aebced7032c8f3df00e2d1125012ccd4585829da8f189a1139a9c0743a1eafaa8b469b78d5bcabe8d69722b046
-  data.tar.gz: df917d5b650d76f24b940dea9ec9308502b8d1ffa1a72b00145c350910b52c3d586641e729bd60cf7762be3e09a36fd723afd0f0922c7d37d550617a5af86379
+  metadata.gz: b17a8137c85fab6ce5e7df780fe601428f74697c94578155be2048f93a1fde9c932515635baf6ee0bc26b286c0bfb7c8eeb3668b8b82443e086998d29dce487e
+  data.tar.gz: 694caa8636745a60a1630e2d40e5f3f24da5bb7dfae6a9a3c48def7f93c24c7904e0b101e786c8cd25ebd297e4ccd799b46bd22ca37b15c4b362da4d32799e61

data/README.md

@@ -4,4 +4,12 @@ This filter plugin is provided for mapping the logs coming from GCP pubsub to '_
 * Google Cloud Functions
 * Google Cloud SQL
 * Google Audit Logs
+* Google Cloud Composer
+* Google Cloud Run
 
+## LogicMonitor properties
+
+| Property | Description |
+| --- | --- |
+| `metadata_keys` | Array of keys to be added as metadata. Filter will look for these keys in fluentd record and extract those if exist. in case of nested json, whole sub json would be added. default ` ["severity", "logName", "labels", "resource.type", "resource.labels", "httpRequest"] ` . few keys will be renamed as part of metadata standardization in Logicmonitor : ` {"trace" => "trace_id", "spanId" => "span_id","resource.type" => "_type"}`|
+| `use_default_severity` | When `true`, and log record does not have severity, severity=`DEFAULT` would be added to log. default `false`.  |

data/fluent-plugin-lm-logs-gcp.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs-gcp"
-  spec.version                        = '1.0.3'
+  spec.version                        = '1.0.5'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor with GCP logs fluentd filter plugin"
@@ -21,4 +21,5 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.0.0'
 
   spec.add_runtime_dependency "fluentd", [">= 1", "< 2"]
+  spec.add_development_dependency 'hashdiff', '>= 1.0.0'
 end

data/lib/fluent/plugin/filter_gcplm.rb

@@ -4,6 +4,12 @@ module Fluent::Plugin
   class GCPLMFilter < Filter
     Fluent::Plugin.register_filter('gcplm', self)
 
+    METADATA_KEYS_TO_RENAME = {"trace" => "trace_id", "spanId" => "span_id","resource.type" => "_type"}.freeze
+    STATIC_METADATA = {"_integration" => "gcp"}
+
+    config_param :metadata_keys, :array, default: ["severity", "logName", "labels", "resource.type", "resource.labels", "httpRequest"], value_type: :string
+    config_param :use_default_severity, :bool, default: false 
+
     def configure(conf)
       super
       # Do the usual configuration here
@@ -12,6 +18,12 @@ module Fluent::Plugin
 # This method is called when starting.
     # Open sockets or files here.
     def start
+      @final_metadata_keys = Hash.new
+      if @metadata_keys
+        @metadata_keys.each do | nested_key |
+          @final_metadata_keys[nested_key] = nested_key.to_s.split('.') 
+        end  
+      end  
       super
     end
 
@@ -67,13 +79,36 @@ module Fluent::Plugin
       end
 
       if(record.key?("protoPayload") && record.dig('protoPayload', '@type') == "type.googleapis.com/google.cloud.audit.AuditLog")
-        resourceMap = {"system.gcp.projectId" => project_id, "system.cloud.category" => 'GCP/LMAccount'}
+        resourceMap = {"system.cloud.category" => 'GCP/LMAccount'}
       end
 
+
+      resourceMap['system.gcp.projectId'] = project_id if !resourceMap.empty?
       # Creating a new record which is further sent to LM
       filteredRecord['message'] = message
       filteredRecord['_lm.resourceId'] = resourceMap
       filteredRecord['timestamp'] = record['timestamp']
+
+      # Add metadata
+      if @final_metadata_keys 
+        @final_metadata_keys.each do | key, value |
+          final_key = METADATA_KEYS_TO_RENAME[key] ? METADATA_KEYS_TO_RENAME[key] : key
+          nestedVal = record
+          value.each { |x| nestedVal = nestedVal[x] }
+          if nestedVal != nil
+            filteredRecord[final_key] = nestedVal
+          end  
+        end  
+      end  
+
+      # Add static metadata
+      STATIC_METADATA.each { | key, value| filteredRecord[key] = value }
+
+      # Add default severity if does not exist 
+      if !filteredRecord["severity"] and @use_default_severity
+        filteredRecord["severity"] = "DEFAULT"
+      end   
+
       filteredRecord
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs-gcp
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.5
 platform: ruby
 authors:
 - LogicMonitor
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-20 00:00:00.000000000 Z
+date: 2023-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,6 +30,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: hashdiff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: This filter plugin filters fluentd records in gcp to the configured LogicMonitor
   account.
 email: rubygems@logicmonitor.com
@@ -65,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.4.10
 signing_key: 
 specification_version: 4
 summary: LogicMonitor with GCP logs fluentd filter plugin