fluent-plugin-elastic-log 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ef4e2a586f566d7b309ef59d3c754bbccaffe6ba545eb31a4715683403f67af
-  data.tar.gz: fd5f957963bb03ca6bc4c2f456c8cf3c9053f35a53cadb2594280bc5899858c1
+  metadata.gz: '089f4560a3510e19726b2236b1085bfd9d2bcc751ef9fbdb3200385348eb2505'
+  data.tar.gz: 0b09fcff0fd33999c05a55191a766161ebf09464fc130a4ab80271352ec896e0
 SHA512:
-  metadata.gz: 482176de12204c607485f80d2234106e20c6d618f8bd6c04155105ff335a20e8d04c71778e9e3bc1382e3efbacda46b2aa1c95fbc2521cebe061503970cdecab
-  data.tar.gz: f1a408614e789248cc1486ca9af9a11c3c8a66502c2bfbd3fe655e9dcd60f66bfbf3853082f99f78b368b39dfe15d67ae223b3a4574b1fdd8018e4651e1e3944
+  metadata.gz: 93fbd475a195e8cd124160cbbf3ffb76f77023e4926bae097e533398950b27a68d99e9d13515b872963b6541f753941041b2b9a09a5c195cbbffc7d2cad1f095
+  data.tar.gz: 608e7042c5ec2facaf5a5ae58fc9cd718ff2aa010dcfee4a12e785e22ce685100571389e97107f16f1cf144d7e6d2b01d93390b36ad58243963e69a07ec17cc5

data/.rubocop.yml

@@ -10,9 +10,17 @@ Metrics/BlockLength:
     - fluent-plugin-elastic-log.gemspec
     - test/**/*.rb
 
+Metrics/ClassLength:
+  Exclude:
+    - test/**/*.rb
+
 Metrics/MethodLength:
   Max: 20
 
+Metrics/ParameterLists:
+  Exclude:
+    - test/helper.rb
+
 Naming/MethodParameterName:
   Exclude:
     - lib/fluent/plugin/out_elastic_audit_log_metric.rb

data/fluent-plugin-elastic-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-elastic-log'
-  spec.version = '0.2.0'
+  spec.version = '0.4.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative 'granted_privileges_metric'
+
+module Fluent
+  module Plugin
+    module ElasticLog
+      # convert audit log event stream to metric event stream
+      class AuditLogToMetricProcessor
+        attr_reader :conf
+
+        def initialize(conf:)
+          @conf = conf
+        end
+
+        def process(_tag, log_es)
+          metric_es = MultiEventStream.new
+
+          log_es.each do |time, record|
+            next unless record
+            next unless (category = record[conf.category_key])
+            next unless conf.categories.include? category
+
+            new_records = send("generate_#{category.downcase}_metrics_for", record)
+            new_records.each { |new_record| metric_es.add(time, new_record) }
+          end
+          metric_es
+        end
+
+        private
+
+        # rubocop:disable Metrics/AbcSize
+        def generate_granted_privileges_metrics_for(record)
+          return unless record[conf.privilege_key]
+
+          GrantedPrivilegesMetric.new(
+            record: {
+              timestamp: record[conf.timestamp_key],
+              privilege: record[conf.privilege_key],
+              user: record[conf.user_key],
+              cluster: record[conf.cluster_key],
+              indices: record[conf.indices_key],
+              r_indices: record[conf.r_indices_key],
+              layer: record[conf.layer_key],
+              request_type: record[conf.request_type_key]
+            },
+            conf: conf
+          ).generate_metrics
+        end
+        # rubocop:enable Metrics/AbcSize
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elastic_log/granted_privileges_metric.rb

@@ -3,8 +3,6 @@
 require 'set'
 require 'time'
 
-require 'fluent/event'
-
 module Fluent
   module Plugin
     module ElasticLog
@@ -21,50 +19,51 @@ module Fluent
         #   data/write/*  => write
         #   monitor/*     => monitor
         PRIVILEGE_MAP = {
-          "cluster:admin/" => 'admin_query',
-          "cluster:monitor/" => 'monitor_query',
-          "indices:admin/" => 'admin_query',
-          "indices:data/read/" => 'read_query',
-          "indices:data/write/" => 'write_query',
-          "indices:monitor/" => 'monitor_query'
+          'cluster:admin/' => 'admin',
+          'cluster:monitor/' => 'monitor',
+          'indices:admin/delete' => 'destroy',
+          'indices:admin/' => 'admin',
+          'indices:data/read/' => 'read',
+          'indices:data/write/delete' => 'delete',
+          'indices:data/write/' => 'write',
+          'indices:monitor/' => 'monitor'
         }.freeze
 
         ILM_PATTERN = /^(.*)-\d{6}$/.freeze
 
-        attr_reader :time, :record, :conf
+        attr_reader :record, :conf
 
-        def initialize(time:, record:, conf:)
-          @time = time
+        def initialize(record:, conf:)
           @record = record
           @conf = conf
         end
 
         def timestamp
-          begin
-            timestamp = Time.parse(record[:timestamp])
-          rescue StandardError
-            timestamp = time.to_time
-          end
+          timestamp = Time.parse(record[:timestamp])
 
-          return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis
+          return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
+          return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
 
           timestamp.utc.iso8601(3)
+        rescue StandardError
+          nil
         end
 
-        def metric_name
+        def query_type
           PRIVILEGE_MAP.each do |pattern, name|
-            return "#{name}_count" if record[:privilege].to_s.start_with?(pattern)
+            return name if record[:privilege].to_s.start_with?(pattern)
           end
-          "unknown_count"
+          'unknown'
         end
 
         def base
           {
             'timestamp' => timestamp,
-            'metric_name' => metric_name,
+            'metric_name' => 'query_count',
             'metric_value' => 1,
-            'tags_user' => record[:user],
-            'tags_cluster' => record[:cluster]
+            "#{conf.prefix}user" => record[:user],
+            "#{conf.prefix}cluster" => record[:cluster],
+            "#{conf.prefix}query_type" => query_type
           }
         end
 
@@ -72,19 +71,19 @@ module Fluent
           indices = record[:r_indices] || record[:indices] || [nil]
           if conf.aggregate_ilm
             indices = indices.inject(Set.new) do |acc, index|
-              aggregated_format = index[ILM_PATTERN, 1]
+              aggregated_format = index && index[ILM_PATTERN, 1]
               acc << (aggregated_format || index)
             end.to_a
           end
           indices
         end
 
-        def generate_event_stream
-          metric_es = MultiEventStream.new
+        def generate_metrics
+          metrics = []
           indices.each do |indice|
-            metric_es.add(time, base.merge(tags_technical_name: indice))
+            metrics << base.merge("#{conf.prefix}technical_name" => indice)
           end
-          metric_es
+          metrics
         end
       end
     end

data/lib/fluent/plugin/out_elastic_audit_log_metric.rb

@@ -16,7 +16,9 @@
 # limitations under the License.
 
 require 'fluent/plugin/output'
-require 'fluent/plugin/elastic_log/granted_privileges_metric'
+require 'fluent/event'
+
+require_relative 'elastic_log/audit_log_to_metric_processor'
 
 module Fluent
   module Plugin
@@ -42,16 +44,7 @@ module Fluent
       DEFAULT_R_INDICES_KEY = 'audit_trace_resolved_indices'
       DEFAULT_TIMESTAMP_KEY = '@timestamp'
       DEFAULT_PRIVILEGE_KEY = 'audit_request_privilege'
-
-      # REQUEST PRIVILEGE:
-      # cluster:
-      #   admin/*       => admin
-      #   monitor/*     => monitor
-      # indices:
-      #   admin/*       => admin
-      #   data/read/*   => read
-      #   data/write/*  => write
-      #   monitor/*     => monitor
+      DEFAULT_PREFIX = ''
 
       desc 'Tag to emit metric events on'
       config_param :tag, :string, default: nil
@@ -78,11 +71,14 @@ module Fluent
       config_param :privilege_key, :string, default: DEFAULT_PRIVILEGE_KEY
 
       desc 'Timestamp format'
-      config_param :timestamp_format, :enum, list: %i[iso epochmillis], default: :iso
-
+      config_param :timestamp_format, :enum, list: %i[iso epochmillis epochmillis_str], default: :iso
+      desc 'Attribute prefix'
+      config_param :prefix, :string, default: DEFAULT_PREFIX
       desc 'Aggregate ILM'
       config_param :aggregate_ilm, :bool, default: true
 
+      attr_reader :metric_processor
+
       def configure(conf)
         super
         raise Fluent::ConfigError, "#{NAME}: tag is mandatory" if !tag || tag.to_s.empty?
@@ -95,6 +91,8 @@ module Fluent
           @categories = categories - unsupported_categories
         end
 
+        @metric_processor = ElasticLog::AuditLogToMetricProcessor.new(conf: self)
+
         true
       end
 
@@ -110,42 +108,9 @@ module Fluent
       end
 
       def process(_tag, es)
-        es.each do |time, record|
-          next unless record
-          next unless (category = record[category_key])
-          next unless ALLOWED_CATEGORIES.include? category
-
-          event_time = Fluent::EventTime.from_time(time)
-          metric_es = send("generate_#{category.downcase}_metrics_for", event_time, record)
-          router.emit_stream(tag, metric_es) if metric_es
-        end
-      end
-
-      # es = Fluent::MultiEventStream.new
-      # router.emit_stream(tag, es)
-
-      private
-
-      # rubocop:disable Metrics/AbcSize
-      def generate_granted_privileges_metrics_for(time, record)
-        return unless record[privilege_key]
-
-        Fluent::Plugin::ElasticLog::GrantedPrivilegesMetric.new(
-          time: time,
-          record: {
-            timestamp: record[timestamp_key],
-            privilege: record[privilege_key],
-            user: record[user_key],
-            cluster: record[cluster_key],
-            indices: record[indices_key],
-            r_indices: record[r_indices_key],
-            layer: record[layer_key],
-            request_type: record[request_type_key]
-          },
-          conf: self
-        ).generate_event_stream
+        metrics = metric_processor.process(tag, es) || []
+        router.emit_stream(tag, metrics) if metrics
       end
-      # rubocop:enable Metrics/AbcSize
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elastic-log
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-08 00:00:00.000000000 Z
+date: 2023-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -162,6 +162,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elastic-log.gemspec
+- lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb
 - lib/fluent/plugin/elastic_log/granted_privileges_metric.rb
 - lib/fluent/plugin/out_elastic_audit_log_metric.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-elastic-log