fluent-plugin-elastic-log 0.4.2 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89de05403ee1b497031fd3d6a833dadf22c47d5d78fdcabed897be853cebd8e5
-  data.tar.gz: f71df586cf16bc07db17f7fc0ed4b24ee08458ad3b02d9e62152e94cebe5f647
+  metadata.gz: a09b3853194e612ab30572268a5fd930995f02fbc7a7f1430e3809222df3f093
+  data.tar.gz: be9066ab3630764364e51772a5ea8a71dee68d5e0cd6da4d332198f00ec804bc
 SHA512:
-  metadata.gz: 4b749a87135558490c9fa2fd8475e03a24870c602f31b049548ea87907dfd092a37c6024506c4240055173993793a9368d3e256bc69dca97587168f758a4402a
-  data.tar.gz: c9b796602e030148fb46c50c981b09d333c32897025da0afd5bd977a21080a030d8e0de177924f455a952d1ae4aac4b02c9304ff46a48968b57ea76bb97f82d4
+  metadata.gz: 720cf4ed71f63792a55dec5e0fccdb601a14ee8119eb4c3264f8827c9625b3f9bb7142eaffc6db8b5d8c0214596ca2ceec47014f7e491ae882781ec9d2e98487
+  data.tar.gz: c5ca8575b0050568388685bf74222080799952394f884f0e568a996c05a5c190db697ad73529e8909dae81aabd8c11b519bef144fe4f5e41403bfa64393562a8

data/README.md

@@ -35,11 +35,13 @@ parameters for input record:
 * r_indices_key: Resolved indices key in input record
 * timestamp_key: Timestamp key in input record
 * privilege_key: Request privilege key in input record
+* rest_request_path_key: Rest request path key in input record
+* request_body_key: Request body key in input record
 
 parameters for output metric:
 * timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
 * prefix: Attribute prefix for output metric
-* aggregate_ilm: Aggregate ILM on resolved indices
+* aggregate_index: Aggregate index (remove ilm suffix, wildcard suffix)
 
 More details from the
 [elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)

data/fluent-plugin-elastic-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-elastic-log'
-  spec.version = '0.4.2'
+  spec.version = '0.5.1'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'granted_privileges_metric'
+require_relative 'failed_login_metric'
 
 module Fluent
   module Plugin
@@ -48,6 +49,22 @@ module Fluent
           ).generate_metrics
         end
         # rubocop:enable Metrics/AbcSize
+
+        # rubocop:disable Metrics/AbcSize
+        def generate_failed_login_metrics_for(record)
+          FailedLoginMetric.new(
+            record: {
+              timestamp: record[conf.timestamp_key],
+              user: record[conf.user_key],
+              cluster: record[conf.cluster_key],
+              layer: record[conf.layer_key],
+              request_path: record[conf.rest_request_path_key],
+              request_body: record[conf.request_body_key]
+            },
+            conf: conf
+          ).generate_metrics
+        end
+        # rubocop:enable Metrics/AbcSize
       end
     end
   end

data/lib/fluent/plugin/elastic_log/failed_login_metric.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'time'
+require 'json'
+
+module Fluent
+  module Plugin
+    module ElasticLog
+      # record to metric converter
+      #   for FAILED_LOGIN
+      class FailedLoginMetric
+        attr_reader :record, :conf
+
+        ELASTIC_URL_PATTERN = %r{(?:/(?<target>[^/]*))?/(?<action>_\w+)}.freeze
+        QUERY_TYPE_MAP = {
+          '_msearch' => 'msearch',
+          '_bulk' => 'bulk',
+          '_doc' => 'write',
+          '_create' => 'write',
+          '_search' => 'search'
+        }.freeze
+
+        INDEX_PATTERN = /-?\*$/.freeze
+        ILM_PATTERN = /-\d{6}$/.freeze
+
+        def initialize(record:, conf:)
+          @record = record
+          @conf = conf
+        end
+
+        def timestamp
+          timestamp = Time.parse(record[:timestamp])
+
+          return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
+          return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
+
+          timestamp.utc.iso8601(3)
+        rescue StandardError
+          nil
+        end
+
+        def query_details
+          if (match = ELASTIC_URL_PATTERN.match(record[:request_path]))
+            return [QUERY_TYPE_MAP.fetch(match[:action], 'other'),
+                    match[:target]]
+          end
+          ['other', nil]
+        end
+
+        def base
+          {
+            'timestamp' => timestamp,
+            'metric_name' => 'failed_login_count',
+            'metric_value' => 1,
+            "#{conf.prefix}user" => record[:user],
+            "#{conf.prefix}cluster" => record[:cluster]
+          }
+        end
+
+        def bulk_indices
+          req_body = record.fetch(:request_body, {})
+          return [] if req_body.empty?
+
+          req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
+            cmd = JSON.parse(cmd_line)
+            acc << aggregate_index(cmd[cmd.keys.first]['_index'])
+          end
+        end
+
+        def msearch_indices
+          req_body = record.fetch(:request_body, {})
+          return [] if req_body.empty?
+
+          req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
+            cmd = JSON.parse(cmd_line)
+            acc << aggregate_index(cmd['index'])
+          end
+        end
+
+        def aggregate_index(index)
+          return unless index
+          return index unless conf.aggregate_index
+
+          index.sub(INDEX_PATTERN, '').sub(ILM_PATTERN, '')
+        end
+
+        def generate_metrics
+          query_action, query_index = query_details
+          indices = case query_action
+                    when 'bulk' then bulk_indices
+                    when 'msearch' then msearch_indices
+                    else []
+                    end
+          indices << aggregate_index(query_index) if query_index || indices.empty?
+
+          indices.inject([]) do |metrics, index|
+            metrics << base.merge("#{conf.prefix}index" => index,
+                                  "#{conf.prefix}query_type" => query_action)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elastic_log/granted_privileges_metric.rb

@@ -29,7 +29,8 @@ module Fluent
           'indices:monitor/' => 'monitor'
         }.freeze
 
-        ILM_PATTERN = /^(.*)-\d{6}$/.freeze
+        INDEX_PATTERN = /-?\*$/.freeze
+        ILM_PATTERN = /-\d{6}$/.freeze
 
         attr_reader :record, :conf
 
@@ -69,19 +70,25 @@ module Fluent
 
         def indices
           indices = record[:r_indices] || record[:indices] || [nil]
-          if conf.aggregate_ilm
+          if conf.aggregate_index
             indices = indices.inject(Set.new) do |acc, index|
-              aggregated_format = index && index[ILM_PATTERN, 1]
-              acc << (aggregated_format || index)
-            end.to_a
+              acc << aggregate_index(index)
+            end
           end
           indices
         end
 
+        def aggregate_index(index)
+          return unless index
+          return index unless conf.aggregate_index
+
+          index.sub(INDEX_PATTERN, '').sub(ILM_PATTERN, '')
+        end
+
         def generate_metrics
           metrics = []
-          indices.each do |indice|
-            metrics << base.merge("#{conf.prefix}technical_name" => indice)
+          indices.each do |index|
+            metrics << base.merge("#{conf.prefix}index" => index)
           end
           metrics
         end

data/lib/fluent/plugin/out_elastic_audit_log_metric.rb

@@ -31,7 +31,7 @@ module Fluent
 
       helpers :event_emitter
 
-      ALLOWED_CATEGORIES = %w[GRANTED_PRIVILEGES].freeze
+      ALLOWED_CATEGORIES = %w[GRANTED_PRIVILEGES FAILED_LOGIN].freeze
       # FAILED_LOGIN AUTHENTICATED MISSING_PRIVILEGES SSL_EXCEPTION
       # OPENDISTRO_SECURITY_INDEX_ATTEMPT BAD_HEADERS
 
@@ -42,6 +42,8 @@ module Fluent
       DEFAULT_USER_KEY = 'audit_request_effective_user'
       DEFAULT_INDICES_KEY = 'audit_trace_indices'
       DEFAULT_R_INDICES_KEY = 'audit_trace_resolved_indices'
+      DEFAULT_REST_REQUEST_PATH = 'audit_rest_request_path'
+      DEFAULT_REQUEST_BODY = 'audit_request_body'
       DEFAULT_TIMESTAMP_KEY = '@timestamp'
       DEFAULT_PRIVILEGE_KEY = 'audit_request_privilege'
       DEFAULT_PREFIX = ''
@@ -69,13 +71,17 @@ module Fluent
       config_param :timestamp_key, :string, default: DEFAULT_TIMESTAMP_KEY
       desc 'Request privilege key'
       config_param :privilege_key, :string, default: DEFAULT_PRIVILEGE_KEY
+      desc 'Rest request path key'
+      config_param :rest_request_path_key, :string, default: DEFAULT_REST_REQUEST_PATH
+      desc 'Request body key'
+      config_param :request_body_key, :string, default: DEFAULT_REQUEST_BODY
 
       desc 'Timestamp format'
       config_param :timestamp_format, :enum, list: %i[iso epochmillis epochmillis_str], default: :iso
       desc 'Attribute prefix'
       config_param :prefix, :string, default: DEFAULT_PREFIX
-      desc 'Aggregate ILM'
-      config_param :aggregate_ilm, :bool, default: true
+      desc 'Aggregate index'
+      config_param :aggregate_index, :bool, default: true
       desc 'Events block size'
       config_param :event_stream_size, :integer, default: 1000
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elastic-log
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.1
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-19 00:00:00.000000000 Z
+date: 2023-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -163,6 +163,7 @@ files:
 - Rakefile
 - fluent-plugin-elastic-log.gemspec
 - lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb
+- lib/fluent/plugin/elastic_log/failed_login_metric.rb
 - lib/fluent/plugin/elastic_log/granted_privileges_metric.rb
 - lib/fluent/plugin/out_elastic_audit_log_metric.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-elastic-log