fluent-plugin-elastic-log 0.4.2 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 89de05403ee1b497031fd3d6a833dadf22c47d5d78fdcabed897be853cebd8e5
4
- data.tar.gz: f71df586cf16bc07db17f7fc0ed4b24ee08458ad3b02d9e62152e94cebe5f647
3
+ metadata.gz: 8fc9200f1f1d69c9899d9d0a3daf550ea117c40794bfb0c4aea36bf98514df60
4
+ data.tar.gz: 5d6d1da9522cbd4e4888f791323f1402107714d522bd30a44eb1d04f58bbacfb
5
5
  SHA512:
6
- metadata.gz: 4b749a87135558490c9fa2fd8475e03a24870c602f31b049548ea87907dfd092a37c6024506c4240055173993793a9368d3e256bc69dca97587168f758a4402a
7
- data.tar.gz: c9b796602e030148fb46c50c981b09d333c32897025da0afd5bd977a21080a030d8e0de177924f455a952d1ae4aac4b02c9304ff46a48968b57ea76bb97f82d4
6
+ metadata.gz: a98aad4e1f27636f65a677680146fbc0e286f3875f175ec172b2e33f286e60219d24d8945bae7022498c8d92f07a2df2e2e2e35e94d5101eef4c49a1e0f19d1f
7
+ data.tar.gz: 51a82ded9b94c7841b9da3d67611c928b0ec315e276cdd314c09ca15fe4d5aef3e0073d8a2614e21871d31019148fc04bd45227b15f5d9f2717d5a98645ab206
data/README.md CHANGED
@@ -35,11 +35,13 @@ parameters for input record:
35
35
  * r_indices_key: Resolved indices key in input record
36
36
  * timestamp_key: Timestamp key in input record
37
37
  * privilege_key: Request privilege key in input record
38
+ * rest_request_path_key: Rest request path key in input record
39
+ * request_body_key: Request body key in input record
38
40
 
39
41
  parameters for output metric:
40
42
  * timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
41
43
  * prefix: Attribute prefix for output metric
42
- * aggregate_ilm: Aggregate ILM on resolved indices
44
+ * aggregate_index: Aggregate index (remove ilm suffix, wildcard suffix)
43
45
 
44
46
  More details from the
45
47
  [elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'fluent-plugin-elastic-log'
8
- spec.version = '0.4.2'
8
+ spec.version = '0.5.0'
9
9
  spec.authors = ['Thomas Tych']
10
10
  spec.email = ['thomas.tych@gmail.com']
11
11
 
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'granted_privileges_metric'
4
+ require_relative 'failed_login_metric'
4
5
 
5
6
  module Fluent
6
7
  module Plugin
@@ -48,6 +49,22 @@ module Fluent
48
49
  ).generate_metrics
49
50
  end
50
51
  # rubocop:enable Metrics/AbcSize
52
+
53
+ # rubocop:disable Metrics/AbcSize
54
+ def generate_failed_login_metrics_for(record)
55
+ FailedLoginMetric.new(
56
+ record: {
57
+ timestamp: record[conf.timestamp_key],
58
+ user: record[conf.user_key],
59
+ cluster: record[conf.cluster_key],
60
+ layer: record[conf.layer_key],
61
+ request_path: record[conf.rest_request_path_key],
62
+ request_body: record[conf.request_body_key]
63
+ },
64
+ conf: conf
65
+ ).generate_metrics
66
+ end
67
+ # rubocop:enable Metrics/AbcSize
51
68
  end
52
69
  end
53
70
  end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+ require 'time'
5
+ require 'json'
6
+
7
+ module Fluent
8
+ module Plugin
9
+ module ElasticLog
10
+ # record to metric converter
11
+ # for FAILED_LOGIN
12
+ class FailedLoginMetric
13
+ attr_reader :record, :conf
14
+
15
+ ELASTIC_URL_PATTERN = %r{(?:/(?<target>[^/]*))?/(?<action>_\w+)}.freeze
16
+ QUERY_TYPE_MAP = {
17
+ '_msearch' => 'msearch',
18
+ '_bulk' => 'bulk',
19
+ '_doc' => 'write',
20
+ '_create' => 'write',
21
+ '_search' => 'search'
22
+ }.freeze
23
+
24
+ INDEX_PATTERN = /-?\*$/.freeze
25
+ ILM_PATTERN = /-\d{6}$/.freeze
26
+
27
+ def initialize(record:, conf:)
28
+ @record = record
29
+ @conf = conf
30
+ end
31
+
32
+ def timestamp
33
+ timestamp = Time.parse(record[:timestamp])
34
+
35
+ return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
36
+ return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
37
+
38
+ timestamp.utc.iso8601(3)
39
+ rescue StandardError
40
+ nil
41
+ end
42
+
43
+ def query_details
44
+ if (match = ELASTIC_URL_PATTERN.match(record[:request_path]))
45
+ return [QUERY_TYPE_MAP.fetch(match[:action], 'other'),
46
+ match[:target]]
47
+ end
48
+ ['other', nil]
49
+ end
50
+
51
+ def base
52
+ {
53
+ 'timestamp' => timestamp,
54
+ 'metric_name' => 'failed_login_count',
55
+ 'metric_value' => 1,
56
+ "#{conf.prefix}user" => record[:user],
57
+ "#{conf.prefix}cluster" => record[:cluster]
58
+ }
59
+ end
60
+
61
+ def bulk_indices
62
+ req_body = record.fetch(:request_body, {})
63
+ return [] if req_body.empty?
64
+
65
+ req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
66
+ cmd = JSON.parse(cmd_line)
67
+ acc << aggregate_index(cmd[cmd.keys.first]['_index'])
68
+ end
69
+ end
70
+
71
+ def msearch_indices
72
+ req_body = record.fetch(:request_body, {})
73
+ return [] if req_body.empty?
74
+
75
+ req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
76
+ cmd = JSON.parse(cmd_line)
77
+ acc << aggregate_index(cmd['index'])
78
+ end
79
+ end
80
+
81
+ def aggregate_index(index)
82
+ return unless index
83
+ return index unless conf.aggregate_index
84
+
85
+ index.sub(INDEX_PATTERN, '').sub(ILM_PATTERN, '')
86
+ end
87
+
88
+ def generate_metrics
89
+ query_action, query_index = query_details
90
+ indices = case query_action
91
+ when 'bulk' then bulk_indices
92
+ when 'msearch' then msearch_indices
93
+ else []
94
+ end
95
+ indices << query_index if query_index || indices.empty?
96
+
97
+ indices.inject([]) do |metrics, index|
98
+ metrics << base.merge("#{conf.prefix}index" => index,
99
+ "#{conf.prefix}query_type" => query_action)
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -29,7 +29,8 @@ module Fluent
29
29
  'indices:monitor/' => 'monitor'
30
30
  }.freeze
31
31
 
32
- ILM_PATTERN = /^(.*)-\d{6}$/.freeze
32
+ INDEX_PATTERN = /-?\*$/.freeze
33
+ ILM_PATTERN = /-\d{6}$/.freeze
33
34
 
34
35
  attr_reader :record, :conf
35
36
 
@@ -69,19 +70,25 @@ module Fluent
69
70
 
70
71
  def indices
71
72
  indices = record[:r_indices] || record[:indices] || [nil]
72
- if conf.aggregate_ilm
73
+ if conf.aggregate_index
73
74
  indices = indices.inject(Set.new) do |acc, index|
74
- aggregated_format = index && index[ILM_PATTERN, 1]
75
- acc << (aggregated_format || index)
76
- end.to_a
75
+ acc << aggregate_index(index)
76
+ end
77
77
  end
78
78
  indices
79
79
  end
80
80
 
81
+ def aggregate_index(index)
82
+ return unless index
83
+ return index unless conf.aggregate_index
84
+
85
+ index.sub(INDEX_PATTERN, '').sub(ILM_PATTERN, '')
86
+ end
87
+
81
88
  def generate_metrics
82
89
  metrics = []
83
- indices.each do |indice|
84
- metrics << base.merge("#{conf.prefix}technical_name" => indice)
90
+ indices.each do |index|
91
+ metrics << base.merge("#{conf.prefix}index" => index)
85
92
  end
86
93
  metrics
87
94
  end
@@ -31,7 +31,7 @@ module Fluent
31
31
 
32
32
  helpers :event_emitter
33
33
 
34
- ALLOWED_CATEGORIES = %w[GRANTED_PRIVILEGES].freeze
34
+ ALLOWED_CATEGORIES = %w[GRANTED_PRIVILEGES FAILED_LOGIN].freeze
35
35
  # FAILED_LOGIN AUTHENTICATED MISSING_PRIVILEGES SSL_EXCEPTION
36
36
  # OPENDISTRO_SECURITY_INDEX_ATTEMPT BAD_HEADERS
37
37
 
@@ -42,6 +42,8 @@ module Fluent
42
42
  DEFAULT_USER_KEY = 'audit_request_effective_user'
43
43
  DEFAULT_INDICES_KEY = 'audit_trace_indices'
44
44
  DEFAULT_R_INDICES_KEY = 'audit_trace_resolved_indices'
45
+ DEFAULT_REST_REQUEST_PATH = 'audit_rest_request_path'
46
+ DEFAULT_REQUEST_BODY = 'audit_request_body'
45
47
  DEFAULT_TIMESTAMP_KEY = '@timestamp'
46
48
  DEFAULT_PRIVILEGE_KEY = 'audit_request_privilege'
47
49
  DEFAULT_PREFIX = ''
@@ -69,13 +71,17 @@ module Fluent
69
71
  config_param :timestamp_key, :string, default: DEFAULT_TIMESTAMP_KEY
70
72
  desc 'Request privilege key'
71
73
  config_param :privilege_key, :string, default: DEFAULT_PRIVILEGE_KEY
74
+ desc 'Rest request path key'
75
+ config_param :rest_request_path_key, :string, default: DEFAULT_REST_REQUEST_PATH
76
+ desc 'Request body key'
77
+ config_param :request_body_key, :string, default: DEFAULT_REQUEST_BODY
72
78
 
73
79
  desc 'Timestamp format'
74
80
  config_param :timestamp_format, :enum, list: %i[iso epochmillis epochmillis_str], default: :iso
75
81
  desc 'Attribute prefix'
76
82
  config_param :prefix, :string, default: DEFAULT_PREFIX
77
- desc 'Aggregate ILM'
78
- config_param :aggregate_ilm, :bool, default: true
83
+ desc 'Aggregate index'
84
+ config_param :aggregate_index, :bool, default: true
79
85
  desc 'Events block size'
80
86
  config_param :event_stream_size, :integer, default: 1000
81
87
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elastic-log
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.2
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Tych
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-06-19 00:00:00.000000000 Z
11
+ date: 2023-06-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bump
@@ -163,6 +163,7 @@ files:
163
163
  - Rakefile
164
164
  - fluent-plugin-elastic-log.gemspec
165
165
  - lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb
166
+ - lib/fluent/plugin/elastic_log/failed_login_metric.rb
166
167
  - lib/fluent/plugin/elastic_log/granted_privileges_metric.rb
167
168
  - lib/fluent/plugin/out_elastic_audit_log_metric.rb
168
169
  homepage: https://gitlab.com/ttych/fluent-plugin-elastic-log