fluent-plugin-elastic-log 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f99628e3afa35a188a2f1b94cf56b44a1cbac89f541852cd1791d4a635b6e658
-  data.tar.gz: ae773fc1151757b4b1ebdc5ef463c05c722ab1d50e89d7e5b01f8c5187505252
+  metadata.gz: '089f4560a3510e19726b2236b1085bfd9d2bcc751ef9fbdb3200385348eb2505'
+  data.tar.gz: 0b09fcff0fd33999c05a55191a766161ebf09464fc130a4ab80271352ec896e0
 SHA512:
-  metadata.gz: e9b473e3db2182d81103f0b5a39c8c7a240c2bcc00dd2e01f852baa55aca426c3edc466595c6feb878ead856cb3f787c9f0cd7f2d182bfb0dcc38d9d5b0498f7
-  data.tar.gz: 544ddb9b34eed81fe9e758b65736ab4335b975ee575afddf36083ec91a026f506b1edea1d6f6a68fb914a88b12e234b375e58dcf84db1f89cd5615088a115ae4
+  metadata.gz: 93fbd475a195e8cd124160cbbf3ffb76f77023e4926bae097e533398950b27a68d99e9d13515b872963b6541f753941041b2b9a09a5c195cbbffc7d2cad1f095
+  data.tar.gz: 608e7042c5ec2facaf5a5ae58fc9cd718ff2aa010dcfee4a12e785e22ce685100571389e97107f16f1cf144d7e6d2b01d93390b36ad58243963e69a07ec17cc5

data/.rubocop.yml

@@ -17,6 +17,10 @@ Metrics/ClassLength:
 Metrics/MethodLength:
   Max: 20
 
+Metrics/ParameterLists:
+  Exclude:
+    - test/helper.rb
+
 Naming/MethodParameterName:
   Exclude:
     - lib/fluent/plugin/out_elastic_audit_log_metric.rb

data/fluent-plugin-elastic-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-elastic-log'
-  spec.version = '0.3.0'
+  spec.version = '0.4.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative 'granted_privileges_metric'
+
+module Fluent
+  module Plugin
+    module ElasticLog
+      # convert audit log event stream to metric event stream
+      class AuditLogToMetricProcessor
+        attr_reader :conf
+
+        def initialize(conf:)
+          @conf = conf
+        end
+
+        def process(_tag, log_es)
+          metric_es = MultiEventStream.new
+
+          log_es.each do |time, record|
+            next unless record
+            next unless (category = record[conf.category_key])
+            next unless conf.categories.include? category
+
+            new_records = send("generate_#{category.downcase}_metrics_for", record)
+            new_records.each { |new_record| metric_es.add(time, new_record) }
+          end
+          metric_es
+        end
+
+        private
+
+        # rubocop:disable Metrics/AbcSize
+        def generate_granted_privileges_metrics_for(record)
+          return unless record[conf.privilege_key]
+
+          GrantedPrivilegesMetric.new(
+            record: {
+              timestamp: record[conf.timestamp_key],
+              privilege: record[conf.privilege_key],
+              user: record[conf.user_key],
+              cluster: record[conf.cluster_key],
+              indices: record[conf.indices_key],
+              r_indices: record[conf.r_indices_key],
+              layer: record[conf.layer_key],
+              request_type: record[conf.request_type_key]
+            },
+            conf: conf
+          ).generate_metrics
+        end
+        # rubocop:enable Metrics/AbcSize
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elastic_log/granted_privileges_metric.rb

@@ -3,8 +3,6 @@
 require 'set'
 require 'time'
 
-require 'fluent/event'
-
 module Fluent
   module Plugin
     module ElasticLog
@@ -23,43 +21,39 @@ module Fluent
         PRIVILEGE_MAP = {
           'cluster:admin/' => 'admin',
           'cluster:monitor/' => 'monitor',
+          'indices:admin/delete' => 'destroy',
           'indices:admin/' => 'admin',
           'indices:data/read/' => 'read',
+          'indices:data/write/delete' => 'delete',
           'indices:data/write/' => 'write',
           'indices:monitor/' => 'monitor'
         }.freeze
 
         ILM_PATTERN = /^(.*)-\d{6}$/.freeze
 
-        attr_reader :time, :record, :conf, :prefix
+        attr_reader :record, :conf
 
-        def initialize(time:, record:, conf:, prefix: '')
-          @time = time
+        def initialize(record:, conf:)
           @record = record
           @conf = conf
-          @prefix = prefix
         end
 
-        # rubocop:disable Metrics/AbcSize
         def timestamp
-          begin
-            timestamp = Time.parse(record[:timestamp])
-          rescue StandardError
-            timestamp = time.to_time
-          end
+          timestamp = Time.parse(record[:timestamp])
 
           return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
           return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
 
           timestamp.utc.iso8601(3)
+        rescue StandardError
+          nil
         end
-        # rubocop:enable Metrics/AbcSize
 
         def query_type
           PRIVILEGE_MAP.each do |pattern, name|
             return name if record[:privilege].to_s.start_with?(pattern)
           end
-          'unknown_count'
+          'unknown'
         end
 
         def base
@@ -67,9 +61,9 @@ module Fluent
             'timestamp' => timestamp,
             'metric_name' => 'query_count',
             'metric_value' => 1,
-            "#{prefix}user" => record[:user],
-            "#{prefix}cluster" => record[:cluster],
-            "#{prefix}query_type" => query_type
+            "#{conf.prefix}user" => record[:user],
+            "#{conf.prefix}cluster" => record[:cluster],
+            "#{conf.prefix}query_type" => query_type
           }
         end
 
@@ -77,19 +71,19 @@ module Fluent
           indices = record[:r_indices] || record[:indices] || [nil]
           if conf.aggregate_ilm
             indices = indices.inject(Set.new) do |acc, index|
-              aggregated_format = index[ILM_PATTERN, 1]
+              aggregated_format = index && index[ILM_PATTERN, 1]
               acc << (aggregated_format || index)
             end.to_a
           end
           indices
         end
 
-        def generate_event_stream
-          metric_es = MultiEventStream.new
+        def generate_metrics
+          metrics = []
           indices.each do |indice|
-            metric_es.add(time, base.merge("#{prefix}technical_name" => indice))
+            metrics << base.merge("#{conf.prefix}technical_name" => indice)
           end
-          metric_es
+          metrics
         end
       end
     end

data/lib/fluent/plugin/out_elastic_audit_log_metric.rb

@@ -16,7 +16,9 @@
 # limitations under the License.
 
 require 'fluent/plugin/output'
-require 'fluent/plugin/elastic_log/granted_privileges_metric'
+require 'fluent/event'
+
+require_relative 'elastic_log/audit_log_to_metric_processor'
 
 module Fluent
   module Plugin
@@ -44,16 +46,6 @@ module Fluent
       DEFAULT_PRIVILEGE_KEY = 'audit_request_privilege'
       DEFAULT_PREFIX = ''
 
-      # REQUEST PRIVILEGE:
-      # cluster:
-      #   admin/*       => admin
-      #   monitor/*     => monitor
-      # indices:
-      #   admin/*       => admin
-      #   data/read/*   => read
-      #   data/write/*  => write
-      #   monitor/*     => monitor
-
       desc 'Tag to emit metric events on'
       config_param :tag, :string, default: nil
       desc 'Categories selected to be converted to metrics'
@@ -85,6 +77,8 @@ module Fluent
       desc 'Aggregate ILM'
       config_param :aggregate_ilm, :bool, default: true
 
+      attr_reader :metric_processor
+
       def configure(conf)
         super
         raise Fluent::ConfigError, "#{NAME}: tag is mandatory" if !tag || tag.to_s.empty?
@@ -97,6 +91,8 @@ module Fluent
           @categories = categories - unsupported_categories
         end
 
+        @metric_processor = ElasticLog::AuditLogToMetricProcessor.new(conf: self)
+
         true
       end
 
@@ -112,43 +108,9 @@ module Fluent
       end
 
       def process(_tag, es)
-        es.each do |time, record|
-          next unless record
-          next unless (category = record[category_key])
-          next unless ALLOWED_CATEGORIES.include? category
-
-          event_time = Fluent::EventTime.from_time(time)
-          metric_es = send("generate_#{category.downcase}_metrics_for", event_time, record)
-          router.emit_stream(tag, metric_es) if metric_es
-        end
-      end
-
-      # es = Fluent::MultiEventStream.new
-      # router.emit_stream(tag, es)
-
-      private
-
-      # rubocop:disable Metrics/AbcSize
-      def generate_granted_privileges_metrics_for(time, record)
-        return unless record[privilege_key]
-
-        Fluent::Plugin::ElasticLog::GrantedPrivilegesMetric.new(
-          time: time,
-          record: {
-            timestamp: record[timestamp_key],
-            privilege: record[privilege_key],
-            user: record[user_key],
-            cluster: record[cluster_key],
-            indices: record[indices_key],
-            r_indices: record[r_indices_key],
-            layer: record[layer_key],
-            request_type: record[request_type_key]
-          },
-          conf: self,
-          prefix: prefix
-        ).generate_event_stream
+        metrics = metric_processor.process(tag, es) || []
+        router.emit_stream(tag, metrics) if metrics
       end
-      # rubocop:enable Metrics/AbcSize
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elastic-log
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-13 00:00:00.000000000 Z
+date: 2023-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -162,6 +162,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elastic-log.gemspec
+- lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb
 - lib/fluent/plugin/elastic_log/granted_privileges_metric.rb
 - lib/fluent/plugin/out_elastic_audit_log_metric.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-elastic-log