fluent-plugin-elastic-log 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08f18d938d7e838acc560c6857affc1821290a66bb4c9a203e039ca75de0b1db'
-  data.tar.gz: 3be561a99555350c031f26d0b1d97de9162a8deee8d32eb2dfb3c41e4008957a
+  metadata.gz: 8fc9200f1f1d69c9899d9d0a3daf550ea117c40794bfb0c4aea36bf98514df60
+  data.tar.gz: 5d6d1da9522cbd4e4888f791323f1402107714d522bd30a44eb1d04f58bbacfb
 SHA512:
-  metadata.gz: 41dc18b2e54f6d852caca69fffacce7ffed2cdf8a46dcbde37f4b9b0120940fd680a535178fbef331b669b9c60ba0154b87596bba741b35fa6d9abce22a361fa
-  data.tar.gz: b344dc9bf21982ff52a81b0ad0e395969f5e1d9e52efee63e75042d9a0b9a7c29313410a91c19f62d00fe15e5921ee9f1e0a47668017aa4f034be50393a9ad16
+  metadata.gz: a98aad4e1f27636f65a677680146fbc0e286f3875f175ec172b2e33f286e60219d24d8945bae7022498c8d92f07a2df2e2e2e35e94d5101eef4c49a1e0f19d1f
+  data.tar.gz: 51a82ded9b94c7841b9da3d67611c928b0ec315e276cdd314c09ca15fe4d5aef3e0073d8a2614e21871d31019148fc04bd45227b15f5d9f2717d5a98645ab206

data/README.md

@@ -1,40 +1,67 @@
 # fluent-plugin-elastic-log
 
-[Fluentd](https://fluentd.org/) filter plugin to do something.
+[Fluentd](https://fluentd.org/) filter plugin to process elastic logs.
 
-TODO: write description for you plugin.
+## plugins
 
-## Installation
+### out - elastic_audit_log_metric
 
-### RubyGems
+process audit logs and transform to metrics.
 
-```
-$ gem install fluent-plugin-elastic-log
+Example:
+
+``` conf
+<match my_tag_pattern>
+  @type elastic_audit_log_metric
+
+  tag elastic_audit_log_metric
+  timestamp_key timestamp
+  timestamp_format epochmillis
+  prefix tags_
+</match>
 ```
 
-### Bundler
+parameters are:
+* tag : Tag to emit metric events
+
+parameters for input record:
+* categories: Categories selected to be converted to metrics
+* category_key: Category key in input record
+* layer_key: Layer key in input record
+* request_type_key: Request type key in input record
+* cluster_key: Cluster key in input record
+* user_key: Request user key in input record
+* indices_key: Indices key in input record
+* r_indices_key: Resolved indices key in input record
+* timestamp_key: Timestamp key in input record
+* privilege_key: Request privilege key in input record
+* rest_request_path_key: Rest request path key in input record
+* request_body_key: Request body key in input record
+
+parameters for output metric:
+* timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
+* prefix: Attribute prefix for output metric
+* aggregate_index: Aggregate index (remove ilm suffix, wildcard suffix)
+
+More details from the
+[elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
 
-Add following line to your Gemfile:
+## Installation
 
-```ruby
-gem "fluent-plugin-elastic-log"
-```
 
-And then execute:
+Manual install, by executing:
 
-```
-$ bundle
-```
+    $ gem install fluent-plugin-elastic-log
 
-## Configuration
+Add to Gemfile with:
 
-You can generate configuration template:
+    $ bundle add fluent-plugin-elastic-log
 
-```
-$ fluent-plugin-config-format filter elastic-log
-```
+## Compatibility
 
-You can copy and paste generated documents here.
+plugin in 1.x.x will work with:
+- ruby >= 2.4.10
+- td-agent >= 3.8.1-0
 
 ## Copyright
 

data/fluent-plugin-elastic-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-elastic-log'
-  spec.version = '0.4.1'
+  spec.version = '0.5.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'granted_privileges_metric'
+require_relative 'failed_login_metric'
 
 module Fluent
   module Plugin
@@ -14,7 +15,7 @@ module Fluent
         end
 
         def process(_tag, log_es)
-          metric_es = MultiEventStream.new
+          metric_es = []
 
           log_es.each do |time, record|
             next unless record
@@ -22,7 +23,7 @@ module Fluent
             next unless conf.categories.include? category
 
             new_records = send("generate_#{category.downcase}_metrics_for", record)
-            new_records&.each { |new_record| metric_es.add(time, new_record) }
+            new_records&.each { |new_record| metric_es << [time, new_record] }
           end
           metric_es
         end
@@ -48,6 +49,22 @@ module Fluent
           ).generate_metrics
         end
         # rubocop:enable Metrics/AbcSize
+
+        # rubocop:disable Metrics/AbcSize
+        def generate_failed_login_metrics_for(record)
+          FailedLoginMetric.new(
+            record: {
+              timestamp: record[conf.timestamp_key],
+              user: record[conf.user_key],
+              cluster: record[conf.cluster_key],
+              layer: record[conf.layer_key],
+              request_path: record[conf.rest_request_path_key],
+              request_body: record[conf.request_body_key]
+            },
+            conf: conf
+          ).generate_metrics
+        end
+        # rubocop:enable Metrics/AbcSize
       end
     end
   end

data/lib/fluent/plugin/elastic_log/failed_login_metric.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'time'
+require 'json'
+
+module Fluent
+  module Plugin
+    module ElasticLog
+      # record to metric converter
+      #   for FAILED_LOGIN
+      class FailedLoginMetric
+        attr_reader :record, :conf
+
+        ELASTIC_URL_PATTERN = %r{(?:/(?<target>[^/]*))?/(?<action>_\w+)}.freeze
+        QUERY_TYPE_MAP = {
+          '_msearch' => 'msearch',
+          '_bulk' => 'bulk',
+          '_doc' => 'write',
+          '_create' => 'write',
+          '_search' => 'search'
+        }.freeze
+
+        INDEX_PATTERN = /-?\*$/.freeze
+        ILM_PATTERN = /-\d{6}$/.freeze
+
+        def initialize(record:, conf:)
+          @record = record
+          @conf = conf
+        end
+
+        def timestamp
+          timestamp = Time.parse(record[:timestamp])
+
+          return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
+          return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
+
+          timestamp.utc.iso8601(3)
+        rescue StandardError
+          nil
+        end
+
+        def query_details
+          if (match = ELASTIC_URL_PATTERN.match(record[:request_path]))
+            return [QUERY_TYPE_MAP.fetch(match[:action], 'other'),
+                    match[:target]]
+          end
+          ['other', nil]
+        end
+
+        def base
+          {
+            'timestamp' => timestamp,
+            'metric_name' => 'failed_login_count',
+            'metric_value' => 1,
+            "#{conf.prefix}user" => record[:user],
+            "#{conf.prefix}cluster" => record[:cluster]
+          }
+        end
+
+        def bulk_indices
+          req_body = record.fetch(:request_body, {})
+          return [] if req_body.empty?
+
+          req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
+            cmd = JSON.parse(cmd_line)
+            acc << aggregate_index(cmd[cmd.keys.first]['_index'])
+          end
+        end
+
+        def msearch_indices
+          req_body = record.fetch(:request_body, {})
+          return [] if req_body.empty?
+
+          req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
+            cmd = JSON.parse(cmd_line)
+            acc << aggregate_index(cmd['index'])
+          end
+        end
+
+        def aggregate_index(index)
+          return unless index
+          return index unless conf.aggregate_index
+
+          index.sub(INDEX_PATTERN, '').sub(ILM_PATTERN, '')
+        end
+
+        def generate_metrics
+          query_action, query_index = query_details
+          indices = case query_action
+                    when 'bulk' then bulk_indices
+                    when 'msearch' then msearch_indices
+                    else []
+                    end
+          indices << query_index if query_index || indices.empty?
+
+          indices.inject([]) do |metrics, index|
+            metrics << base.merge("#{conf.prefix}index" => index,
+                                  "#{conf.prefix}query_type" => query_action)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/elastic_log/granted_privileges_metric.rb

@@ -29,7 +29,8 @@ module Fluent
           'indices:monitor/' => 'monitor'
         }.freeze
 
-        ILM_PATTERN = /^(.*)-\d{6}$/.freeze
+        INDEX_PATTERN = /-?\*$/.freeze
+        ILM_PATTERN = /-\d{6}$/.freeze
 
         attr_reader :record, :conf
 
@@ -69,19 +70,25 @@ module Fluent
 
         def indices
           indices = record[:r_indices] || record[:indices] || [nil]
-          if conf.aggregate_ilm
+          if conf.aggregate_index
             indices = indices.inject(Set.new) do |acc, index|
-              aggregated_format = index && index[ILM_PATTERN, 1]
-              acc << (aggregated_format || index)
-            end.to_a
+              acc << aggregate_index(index)
+            end
           end
           indices
         end
 
+        def aggregate_index(index)
+          return unless index
+          return index unless conf.aggregate_index
+
+          index.sub(INDEX_PATTERN, '').sub(ILM_PATTERN, '')
+        end
+
         def generate_metrics
           metrics = []
-          indices.each do |indice|
-            metrics << base.merge("#{conf.prefix}technical_name" => indice)
+          indices.each do |index|
+            metrics << base.merge("#{conf.prefix}index" => index)
           end
           metrics
         end

data/lib/fluent/plugin/out_elastic_audit_log_metric.rb

@@ -31,7 +31,7 @@ module Fluent
 
       helpers :event_emitter
 
-      ALLOWED_CATEGORIES = %w[GRANTED_PRIVILEGES].freeze
+      ALLOWED_CATEGORIES = %w[GRANTED_PRIVILEGES FAILED_LOGIN].freeze
       # FAILED_LOGIN AUTHENTICATED MISSING_PRIVILEGES SSL_EXCEPTION
       # OPENDISTRO_SECURITY_INDEX_ATTEMPT BAD_HEADERS
 
@@ -42,6 +42,8 @@ module Fluent
       DEFAULT_USER_KEY = 'audit_request_effective_user'
       DEFAULT_INDICES_KEY = 'audit_trace_indices'
       DEFAULT_R_INDICES_KEY = 'audit_trace_resolved_indices'
+      DEFAULT_REST_REQUEST_PATH = 'audit_rest_request_path'
+      DEFAULT_REQUEST_BODY = 'audit_request_body'
       DEFAULT_TIMESTAMP_KEY = '@timestamp'
       DEFAULT_PRIVILEGE_KEY = 'audit_request_privilege'
       DEFAULT_PREFIX = ''
@@ -69,13 +71,19 @@ module Fluent
       config_param :timestamp_key, :string, default: DEFAULT_TIMESTAMP_KEY
       desc 'Request privilege key'
       config_param :privilege_key, :string, default: DEFAULT_PRIVILEGE_KEY
+      desc 'Rest request path key'
+      config_param :rest_request_path_key, :string, default: DEFAULT_REST_REQUEST_PATH
+      desc 'Request body key'
+      config_param :request_body_key, :string, default: DEFAULT_REQUEST_BODY
 
       desc 'Timestamp format'
       config_param :timestamp_format, :enum, list: %i[iso epochmillis epochmillis_str], default: :iso
       desc 'Attribute prefix'
       config_param :prefix, :string, default: DEFAULT_PREFIX
-      desc 'Aggregate ILM'
-      config_param :aggregate_ilm, :bool, default: true
+      desc 'Aggregate index'
+      config_param :aggregate_index, :bool, default: true
+      desc 'Events block size'
+      config_param :event_stream_size, :integer, default: 1000
 
       attr_reader :metric_processor
 
@@ -109,7 +117,11 @@ module Fluent
 
       def process(_tag, es)
         metrics = metric_processor.process(tag, es) || []
-        router.emit_stream(tag, metrics) if metrics
+        metrics.each_slice(event_stream_size) do |metrics_slice|
+          metrics_es = MultiEventStream.new
+          metrics_slice.each { |time, record| metrics_es.add(time, record) }
+          router.emit_stream(tag, metrics_es)
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elastic-log
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-15 00:00:00.000000000 Z
+date: 2023-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -163,6 +163,7 @@ files:
 - Rakefile
 - fluent-plugin-elastic-log.gemspec
 - lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb
+- lib/fluent/plugin/elastic_log/failed_login_metric.rb
 - lib/fluent/plugin/elastic_log/granted_privileges_metric.rb
 - lib/fluent/plugin/out_elastic_audit_log_metric.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-elastic-log