fluent-plugin-elastic-log 0.5.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +13 -2
- data/README.md +36 -6
- data/Rakefile +12 -5
- data/fluent-plugin-elastic-log.gemspec +16 -17
- data/lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb +13 -30
- data/lib/fluent/plugin/elastic_log/failed_login_metric.rb +56 -25
- data/lib/fluent/plugin/elastic_log/granted_privileges_metric.rb +63 -22
- data/lib/fluent/plugin/elastic_log/metric_accumulator.rb +37 -0
- data/lib/fluent/plugin/out_elastic_audit_log_metric.rb +24 -10
- metadata +103 -23
- data/.ruby-version +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2bf01058201e056f1f320acd85d647feb4c2b133d507889ab8bd88a5da8c487e
|
4
|
+
data.tar.gz: de25f2017972a6dc867034bd536dde0303afd09c5bee70cd816097f2834660e1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8698830c34fc97b50210ed55b7f06d15b229184dbadd493f30d20fa495766a8442a6e25176b76456d9602e1df8af7e7893310c1c5a864011b293beb91b7b4b35
|
7
|
+
data.tar.gz: ddd74c90f196af2857420ae45ee4dcbc3645c3f54581998a1921d07b922dca75bcc034a3064a55efbfa9b15a0fc3ecae0c437d188ff37c4e437bafe63b569324
|
data/.rubocop.yml
CHANGED
@@ -1,10 +1,18 @@
|
|
1
|
-
|
1
|
+
---
|
2
|
+
|
3
|
+
plugins:
|
2
4
|
- rubocop-rake
|
3
5
|
|
4
6
|
AllCops:
|
5
|
-
TargetRubyVersion: 2.
|
7
|
+
TargetRubyVersion: 2.7
|
6
8
|
NewCops: enable
|
7
9
|
|
10
|
+
Gemspec/DevelopmentDependencies:
|
11
|
+
EnforcedStyle: gemspec
|
12
|
+
|
13
|
+
Metrics/AbcSize:
|
14
|
+
Max: 20
|
15
|
+
|
8
16
|
Metrics/BlockLength:
|
9
17
|
Exclude:
|
10
18
|
- fluent-plugin-elastic-log.gemspec
|
@@ -24,3 +32,6 @@ Metrics/ParameterLists:
|
|
24
32
|
Naming/MethodParameterName:
|
25
33
|
Exclude:
|
26
34
|
- lib/fluent/plugin/out_elastic_audit_log_metric.rb
|
35
|
+
|
36
|
+
Style/Documentation:
|
37
|
+
Enabled: false
|
data/README.md
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
[Fluentd](https://fluentd.org/) filter plugin to process elastic logs.
|
4
4
|
|
5
|
+
|
5
6
|
## plugins
|
6
7
|
|
7
8
|
### out - elastic_audit_log_metric
|
@@ -40,14 +41,39 @@ parameters for input record:
|
|
40
41
|
|
41
42
|
parameters for output metric:
|
42
43
|
* timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
|
43
|
-
*
|
44
|
-
|
44
|
+
* metadata_prefix: Metadata prefix for output metric
|
45
|
+
|
46
|
+
parameters to aggregate metrics:
|
47
|
+
* aggregate_index_clean_suffix: pattern to clean on index, to aggregate events
|
48
|
+
* aggregate_interval: aggregate metrics by time interval, to reduce count of emitted events
|
45
49
|
|
46
50
|
More details from the
|
47
51
|
[elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
|
48
52
|
|
49
|
-
## Installation
|
50
53
|
|
54
|
+
produces metrics:
|
55
|
+
|
56
|
+
| from category | metric_name | purpose |
|
57
|
+
|--------------------|--------------------|-----------------------------------------------------|
|
58
|
+
| GRANTED_PRIVILEGES | user_query_count | Query count made by users |
|
59
|
+
| GRANTED_PRIVILEGES | index_query_count | Query count made by index / index pattern with user |
|
60
|
+
|--------------------|--------------------|-----------------------------------------------------|
|
61
|
+
| FAILED_LOGIN | failed_login_count | Count failed login by users without index |
|
62
|
+
|
63
|
+
|
64
|
+
Categories are :
|
65
|
+
|
66
|
+
| Category | Meaning |
|
67
|
+
|--------------------|---------------------------------------------------------------------------------|
|
68
|
+
| FAILED_LOGIN | Indicates unsuccessful authentication attempts (~ 401) |
|
69
|
+
| MISSING_PRIVILEGES | Occurs when a user attempts an action without the necessary permissions (~ 403) |
|
70
|
+
| BAD_HEADERS | Triggered by requests containing malformed or invalid headers |
|
71
|
+
| SSL_EXCEPTION | Relates to issues with SSL/TLS connections, such as handshake failures. |
|
72
|
+
| AUTHENTICATED | Records successful user authentications |
|
73
|
+
| GRANTED_PRIVILEGES | Logs instances where users are granted specific privileges |
|
74
|
+
|
75
|
+
|
76
|
+
## Installation
|
51
77
|
|
52
78
|
Manual install, by executing:
|
53
79
|
|
@@ -57,14 +83,18 @@ Add to Gemfile with:
|
|
57
83
|
|
58
84
|
$ bundle add fluent-plugin-elastic-log
|
59
85
|
|
86
|
+
|
60
87
|
## Compatibility
|
61
88
|
|
62
89
|
plugin in 1.x.x will work with:
|
63
|
-
- ruby >= 2.
|
64
|
-
-
|
90
|
+
- ruby >= 2.7.0
|
91
|
+
- fluentd >= 1.8.0
|
92
|
+
|
93
|
+
see [gemspec](fluent-plugin-elastic-log.gemspec).
|
94
|
+
|
65
95
|
|
66
96
|
## Copyright
|
67
97
|
|
68
|
-
* Copyright(c) 2023- Thomas Tych
|
98
|
+
* Copyright(c) 2023-2025 Thomas Tych
|
69
99
|
* License
|
70
100
|
* Apache License, Version 2.0
|
data/Rakefile
CHANGED
@@ -4,16 +4,23 @@ require 'bundler'
|
|
4
4
|
Bundler::GemHelper.install_tasks
|
5
5
|
|
6
6
|
require 'rake/testtask'
|
7
|
-
require 'rubocop/rake_task'
|
8
|
-
require 'bump/tasks'
|
9
7
|
|
10
8
|
Rake::TestTask.new(:test) do |t|
|
11
9
|
t.libs.push('lib', 'test')
|
12
|
-
|
13
|
-
t.
|
14
|
-
|
10
|
+
|
11
|
+
t.test_files = if ENV['TEST_FILE']
|
12
|
+
FileList["#{ENV['TEST_FILE']}*", "test/**/#{ENV['TEST_FILE']}*"]
|
13
|
+
else
|
14
|
+
FileList['test/**/test_*.rb', 'test/**/*_test.rb']
|
15
|
+
end
|
16
|
+
t.verbose = ENV.fetch('VERBOSE', false)
|
17
|
+
t.warning = ENV.fetch('WARNING', false)
|
15
18
|
end
|
16
19
|
|
20
|
+
require 'rubocop/rake_task'
|
21
|
+
|
17
22
|
RuboCop::RakeTask.new
|
18
23
|
|
24
|
+
require 'bump/tasks'
|
25
|
+
|
19
26
|
task default: %i[test rubocop]
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = 'fluent-plugin-elastic-log'
|
8
|
-
spec.version = '0.
|
8
|
+
spec.version = '1.0.0'
|
9
9
|
spec.authors = ['Thomas Tych']
|
10
10
|
spec.email = ['thomas.tych@gmail.com']
|
11
11
|
|
@@ -13,7 +13,9 @@ Gem::Specification.new do |spec|
|
|
13
13
|
spec.homepage = 'https://gitlab.com/ttych/fluent-plugin-elastic-log'
|
14
14
|
spec.license = 'Apache-2.0'
|
15
15
|
|
16
|
-
spec.required_ruby_version = '>= 2.
|
16
|
+
spec.required_ruby_version = '>= 2.7.0'
|
17
|
+
|
18
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
17
19
|
|
18
20
|
spec.files = Dir.chdir(__dir__) do
|
19
21
|
`git ls-files -z`.split("\x0").reject do |f|
|
@@ -23,20 +25,17 @@ Gem::Specification.new do |spec|
|
|
23
25
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
24
26
|
spec.require_paths = ['lib']
|
25
27
|
|
26
|
-
|
27
|
-
|
28
|
-
# for old version of td-agent
|
29
|
-
|
30
|
-
spec.add_development_dependency 'bump', '~> 0.10.0'
|
31
|
-
spec.add_development_dependency 'bundler', '~> 2.2'
|
28
|
+
spec.add_development_dependency 'bump', '~> 0.10'
|
29
|
+
spec.add_development_dependency 'bundler', '~> 2.6', '>= 2.6.5'
|
32
30
|
spec.add_development_dependency 'byebug', '~> 11.1', '>= 11.1.3'
|
33
|
-
spec.add_development_dependency '
|
34
|
-
spec.add_development_dependency '
|
35
|
-
spec.add_development_dependency '
|
36
|
-
spec.add_development_dependency 'rubocop
|
37
|
-
spec.add_development_dependency '
|
38
|
-
|
39
|
-
spec.
|
40
|
-
|
41
|
-
|
31
|
+
spec.add_development_dependency 'mocha', '~> 2.7', '>= 2.7.1'
|
32
|
+
spec.add_development_dependency 'rake', '~> 13.2', '>= 13.2.1'
|
33
|
+
spec.add_development_dependency 'reek', '~> 6.4'
|
34
|
+
spec.add_development_dependency 'rubocop', '~> 1.73', '>= 1.73.1'
|
35
|
+
spec.add_development_dependency 'rubocop-rake', '~> 0.7', '>= 0.7.1'
|
36
|
+
spec.add_development_dependency 'simplecov', '~> 0.22'
|
37
|
+
spec.add_development_dependency 'test-unit', '~> 3.6', '>= 3.6.7'
|
38
|
+
spec.add_development_dependency 'timecop', '~> 0.9', '>= 0.9.10'
|
39
|
+
|
40
|
+
spec.add_dependency 'fluentd', ['>= 0.14.10', '< 2']
|
42
41
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative 'metric_accumulator'
|
3
4
|
require_relative 'granted_privileges_metric'
|
4
5
|
require_relative 'failed_login_metric'
|
5
6
|
|
@@ -15,56 +16,38 @@ module Fluent
|
|
15
16
|
end
|
16
17
|
|
17
18
|
def process(_tag, log_es)
|
18
|
-
|
19
|
+
metric_acc = new_metric_accumulator
|
19
20
|
|
20
|
-
log_es.
|
21
|
+
log_es.each_value do |record|
|
21
22
|
next unless record
|
22
23
|
next unless (category = record[conf.category_key])
|
23
24
|
next unless conf.categories.include? category
|
24
25
|
|
25
26
|
new_records = send("generate_#{category.downcase}_metrics_for", record)
|
26
|
-
new_records&.each { |new_record|
|
27
|
+
new_records&.each { |new_record| metric_acc << new_record }
|
27
28
|
end
|
28
|
-
|
29
|
+
metric_acc.to_a
|
29
30
|
end
|
30
31
|
|
31
32
|
private
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
|
34
|
+
def new_metric_accumulator
|
35
|
+
MetricAccumulator.new(value_key: 'metric_value')
|
36
|
+
end
|
36
37
|
|
38
|
+
def generate_granted_privileges_metrics_for(record)
|
37
39
|
GrantedPrivilegesMetric.new(
|
38
|
-
|
39
|
-
|
40
|
-
privilege: record[conf.privilege_key],
|
41
|
-
user: record[conf.user_key],
|
42
|
-
cluster: record[conf.cluster_key],
|
43
|
-
indices: record[conf.indices_key],
|
44
|
-
r_indices: record[conf.r_indices_key],
|
45
|
-
layer: record[conf.layer_key],
|
46
|
-
request_type: record[conf.request_type_key]
|
47
|
-
},
|
48
|
-
conf: conf
|
40
|
+
conf: conf,
|
41
|
+
record: record
|
49
42
|
).generate_metrics
|
50
43
|
end
|
51
|
-
# rubocop:enable Metrics/AbcSize
|
52
44
|
|
53
|
-
# rubocop:disable Metrics/AbcSize
|
54
45
|
def generate_failed_login_metrics_for(record)
|
55
46
|
FailedLoginMetric.new(
|
56
|
-
|
57
|
-
|
58
|
-
user: record[conf.user_key],
|
59
|
-
cluster: record[conf.cluster_key],
|
60
|
-
layer: record[conf.layer_key],
|
61
|
-
request_path: record[conf.rest_request_path_key],
|
62
|
-
request_body: record[conf.request_body_key]
|
63
|
-
},
|
64
|
-
conf: conf
|
47
|
+
conf: conf,
|
48
|
+
record: record
|
65
49
|
).generate_metrics
|
66
50
|
end
|
67
|
-
# rubocop:enable Metrics/AbcSize
|
68
51
|
end
|
69
52
|
end
|
70
53
|
end
|
@@ -10,8 +10,6 @@ module Fluent
|
|
10
10
|
# record to metric converter
|
11
11
|
# for FAILED_LOGIN
|
12
12
|
class FailedLoginMetric
|
13
|
-
attr_reader :record, :conf
|
14
|
-
|
15
13
|
ELASTIC_URL_PATTERN = %r{(?:/(?<target>[^/]*))?/(?<action>_\w+)}.freeze
|
16
14
|
QUERY_TYPE_MAP = {
|
17
15
|
'_msearch' => 'msearch',
|
@@ -21,16 +19,43 @@ module Fluent
|
|
21
19
|
'_search' => 'search'
|
22
20
|
}.freeze
|
23
21
|
|
24
|
-
|
25
|
-
ILM_PATTERN = /-\d{6}$/.freeze
|
22
|
+
attr_reader :record, :conf
|
26
23
|
|
27
24
|
def initialize(record:, conf:)
|
28
25
|
@record = record
|
29
26
|
@conf = conf
|
30
27
|
end
|
31
28
|
|
29
|
+
def record_timestamp
|
30
|
+
record[conf.timestamp_key]
|
31
|
+
end
|
32
|
+
|
33
|
+
def record_user
|
34
|
+
record[conf.user_key]
|
35
|
+
end
|
36
|
+
|
37
|
+
def record_cluster
|
38
|
+
record[conf.cluster_key]
|
39
|
+
end
|
40
|
+
|
41
|
+
def record_layer
|
42
|
+
record[conf.layer_key]
|
43
|
+
end
|
44
|
+
|
45
|
+
def record_request_path
|
46
|
+
record[conf.rest_request_path_key]
|
47
|
+
end
|
48
|
+
|
49
|
+
def record_request_body
|
50
|
+
record[conf.request_body_key]
|
51
|
+
end
|
52
|
+
|
32
53
|
def timestamp
|
33
|
-
timestamp = Time.parse(
|
54
|
+
timestamp = Time.parse(record_timestamp)
|
55
|
+
|
56
|
+
if conf.aggregate_interval.to_i.positive?
|
57
|
+
timestamp = Time.at((timestamp.to_i / conf.aggregate_interval) * conf.aggregate_interval)
|
58
|
+
end
|
34
59
|
|
35
60
|
return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
|
36
61
|
return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
|
@@ -41,7 +66,7 @@ module Fluent
|
|
41
66
|
end
|
42
67
|
|
43
68
|
def query_details
|
44
|
-
if (match = ELASTIC_URL_PATTERN.match(
|
69
|
+
if (match = ELASTIC_URL_PATTERN.match(record_request_path))
|
45
70
|
return [QUERY_TYPE_MAP.fetch(match[:action], 'other'),
|
46
71
|
match[:target]]
|
47
72
|
end
|
@@ -53,13 +78,13 @@ module Fluent
|
|
53
78
|
'timestamp' => timestamp,
|
54
79
|
'metric_name' => 'failed_login_count',
|
55
80
|
'metric_value' => 1,
|
56
|
-
"#{conf.
|
57
|
-
"#{conf.
|
81
|
+
"#{conf.metadata_prefix}user" => record_user,
|
82
|
+
"#{conf.metadata_prefix}cluster" => record_cluster
|
58
83
|
}
|
59
84
|
end
|
60
85
|
|
61
86
|
def bulk_indices
|
62
|
-
req_body =
|
87
|
+
req_body = record_request_body || {}
|
63
88
|
return [] if req_body.empty?
|
64
89
|
|
65
90
|
req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
|
@@ -69,7 +94,7 @@ module Fluent
|
|
69
94
|
end
|
70
95
|
|
71
96
|
def msearch_indices
|
72
|
-
req_body =
|
97
|
+
req_body = record_request_body || {}
|
73
98
|
return [] if req_body.empty?
|
74
99
|
|
75
100
|
req_body.each_line.each_slice(2).with_object(Set.new) do |(cmd_line, _data_line), acc|
|
@@ -80,25 +105,31 @@ module Fluent
|
|
80
105
|
end
|
81
106
|
|
82
107
|
def aggregate_index(index)
|
83
|
-
return unless index
|
84
|
-
return index unless conf.aggregate_index
|
108
|
+
return index unless index && conf.aggregate_index_clean_suffix
|
85
109
|
|
86
|
-
|
110
|
+
conf.aggregate_index_clean_suffix.inject(index) do |index_clean, clean_pattern|
|
111
|
+
index_clean.sub(clean_pattern, '')
|
112
|
+
end
|
87
113
|
end
|
88
114
|
|
89
115
|
def generate_metrics
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
116
|
+
metrics = []
|
117
|
+
|
118
|
+
query_action, = query_details
|
119
|
+
|
120
|
+
# indices = case query_action
|
121
|
+
# when 'bulk' then bulk_indices
|
122
|
+
# when 'msearch' then msearch_indices
|
123
|
+
# else []
|
124
|
+
# end
|
125
|
+
# indices << aggregate_index(query_index) if query_index || indices.empty?
|
126
|
+
|
127
|
+
# indices.inject([]) do |metrics, index|
|
128
|
+
# metrics << base.merge("#{conf.metadata_prefix}index" => index,
|
129
|
+
# "#{conf.metadata_prefix}query_type" => query_action)
|
130
|
+
# end
|
131
|
+
|
132
|
+
metrics << base.merge("#{conf.metadata_prefix}query_type" => query_action)
|
102
133
|
end
|
103
134
|
end
|
104
135
|
end
|
@@ -29,9 +29,6 @@ module Fluent
|
|
29
29
|
'indices:monitor/' => 'monitor'
|
30
30
|
}.freeze
|
31
31
|
|
32
|
-
INDEX_PATTERN = /-?\*$/.freeze
|
33
|
-
ILM_PATTERN = /-\d{6}$/.freeze
|
34
|
-
|
35
32
|
attr_reader :record, :conf
|
36
33
|
|
37
34
|
def initialize(record:, conf:)
|
@@ -39,8 +36,44 @@ module Fluent
|
|
39
36
|
@conf = conf
|
40
37
|
end
|
41
38
|
|
39
|
+
def record_timestamp
|
40
|
+
record[conf.timestamp_key]
|
41
|
+
end
|
42
|
+
|
43
|
+
def record_privilege
|
44
|
+
record[conf.privilege_key]
|
45
|
+
end
|
46
|
+
|
47
|
+
def record_user
|
48
|
+
record[conf.user_key]
|
49
|
+
end
|
50
|
+
|
51
|
+
def record_cluster
|
52
|
+
record[conf.cluster_key]
|
53
|
+
end
|
54
|
+
|
55
|
+
def record_indices
|
56
|
+
record[conf.indices_key]
|
57
|
+
end
|
58
|
+
|
59
|
+
def record_r_indices
|
60
|
+
record[conf.r_indices_key]
|
61
|
+
end
|
62
|
+
|
63
|
+
def record_layer
|
64
|
+
record[conf.layer_key]
|
65
|
+
end
|
66
|
+
|
67
|
+
def record_request_type
|
68
|
+
record[conf.request_type_key]
|
69
|
+
end
|
70
|
+
|
42
71
|
def timestamp
|
43
|
-
timestamp = Time.parse(
|
72
|
+
timestamp = Time.parse(record_timestamp)
|
73
|
+
|
74
|
+
if conf.aggregate_interval.to_i.positive?
|
75
|
+
timestamp = Time.at((timestamp.to_i / conf.aggregate_interval) * conf.aggregate_interval)
|
76
|
+
end
|
44
77
|
|
45
78
|
return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
|
46
79
|
return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
|
@@ -52,7 +85,7 @@ module Fluent
|
|
52
85
|
|
53
86
|
def query_type
|
54
87
|
PRIVILEGE_MAP.each do |pattern, name|
|
55
|
-
return name if
|
88
|
+
return name if record_privilege.to_s.start_with?(pattern)
|
56
89
|
end
|
57
90
|
'unknown'
|
58
91
|
end
|
@@ -60,37 +93,45 @@ module Fluent
|
|
60
93
|
def base
|
61
94
|
{
|
62
95
|
'timestamp' => timestamp,
|
63
|
-
'metric_name' => 'query_count',
|
64
96
|
'metric_value' => 1,
|
65
|
-
"#{conf.
|
66
|
-
"#{conf.
|
67
|
-
"#{conf.
|
97
|
+
"#{conf.metadata_prefix}user" => record_user,
|
98
|
+
"#{conf.metadata_prefix}cluster" => record_cluster,
|
99
|
+
"#{conf.metadata_prefix}query_type" => query_type
|
68
100
|
}
|
69
101
|
end
|
70
102
|
|
71
103
|
def indices
|
72
|
-
indices =
|
73
|
-
|
74
|
-
|
75
|
-
acc << aggregate_index(index)
|
76
|
-
end
|
104
|
+
indices = record_r_indices || record_indices || [nil]
|
105
|
+
indices.inject(Set.new) do |acc, index|
|
106
|
+
acc << aggregate_index(index)
|
77
107
|
end
|
78
|
-
indices
|
79
108
|
end
|
80
109
|
|
81
110
|
def aggregate_index(index)
|
82
|
-
return unless index
|
83
|
-
return index unless conf.aggregate_index
|
111
|
+
return index unless index && conf.aggregate_index_clean_suffix
|
84
112
|
|
85
|
-
|
113
|
+
conf.aggregate_index_clean_suffix.inject(index) do |index_clean, clean_pattern|
|
114
|
+
index_clean.sub(clean_pattern, '')
|
115
|
+
end
|
86
116
|
end
|
87
117
|
|
88
118
|
def generate_metrics
|
89
|
-
|
90
|
-
|
91
|
-
|
119
|
+
generate_user_metrics + generate_index_metrics
|
120
|
+
end
|
121
|
+
|
122
|
+
def generate_user_metrics
|
123
|
+
[
|
124
|
+
base.merge('metric_name' => 'user_query_count')
|
125
|
+
]
|
126
|
+
end
|
127
|
+
|
128
|
+
def generate_index_metrics
|
129
|
+
indices.inject([]) do |metrics, index|
|
130
|
+
metrics << base.merge(
|
131
|
+
'metric_name' => 'index_query_count',
|
132
|
+
"#{conf.metadata_prefix}index" => index
|
133
|
+
)
|
92
134
|
end
|
93
|
-
metrics
|
94
135
|
end
|
95
136
|
end
|
96
137
|
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
module Plugin
|
5
|
+
module ElasticLog
|
6
|
+
class MetricAccumulator
|
7
|
+
attr_reader :value_key
|
8
|
+
|
9
|
+
def initialize(value_key:)
|
10
|
+
@value_key = value_key
|
11
|
+
|
12
|
+
@data = []
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(metric)
|
16
|
+
@data << metric
|
17
|
+
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
def to_a
|
22
|
+
group.to_a
|
23
|
+
end
|
24
|
+
|
25
|
+
def group
|
26
|
+
grouped_data = @data.group_by { |metric| metric.reject { |k, _| k == value_key } }
|
27
|
+
grouped_data.map do |metric_base, metrics|
|
28
|
+
accumulated_value = metrics.sum { |metric| metric.fetch(value_key, 0) }
|
29
|
+
metric_base.update(
|
30
|
+
value_key => accumulated_value
|
31
|
+
)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -34,7 +34,9 @@ module Fluent
|
|
34
34
|
ALLOWED_CATEGORIES = %w[GRANTED_PRIVILEGES FAILED_LOGIN].freeze
|
35
35
|
# FAILED_LOGIN AUTHENTICATED MISSING_PRIVILEGES SSL_EXCEPTION
|
36
36
|
# OPENDISTRO_SECURITY_INDEX_ATTEMPT BAD_HEADERS
|
37
|
+
DEFAULT_CATEGORIES = %w[GRANTED_PRIVILEGES].freeze
|
37
38
|
|
39
|
+
CONFIGURATION_KEYS = %w[category layer request_type cluster user indices r_indices timestamp privilege].freeze
|
38
40
|
DEFAULT_CATEGORY_KEY = 'audit_category'
|
39
41
|
DEFAULT_LAYER_KEY = 'audit_request_layer'
|
40
42
|
DEFAULT_REQUEST_TYPE = 'audit_transport_request_type'
|
@@ -46,12 +48,16 @@ module Fluent
|
|
46
48
|
DEFAULT_REQUEST_BODY = 'audit_request_body'
|
47
49
|
DEFAULT_TIMESTAMP_KEY = '@timestamp'
|
48
50
|
DEFAULT_PRIVILEGE_KEY = 'audit_request_privilege'
|
49
|
-
|
51
|
+
|
52
|
+
DEFAULT_AGGREGATE_INDEX_CLEAN_SUFFIX = [].freeze
|
53
|
+
DEFAULT_AGGREGATE_INTERVAL = nil
|
54
|
+
|
55
|
+
DEFAULT_METADATA_PREFIX = ''
|
50
56
|
|
51
57
|
desc 'Tag to emit metric events on'
|
52
58
|
config_param :tag, :string, default: nil
|
53
59
|
desc 'Categories selected to be converted to metrics'
|
54
|
-
config_param :categories, :array, default:
|
60
|
+
config_param :categories, :array, default: DEFAULT_CATEGORIES, value_type: :string
|
55
61
|
|
56
62
|
desc 'Category key'
|
57
63
|
config_param :category_key, :string, default: DEFAULT_CATEGORY_KEY
|
@@ -78,10 +84,17 @@ module Fluent
|
|
78
84
|
|
79
85
|
desc 'Timestamp format'
|
80
86
|
config_param :timestamp_format, :enum, list: %i[iso epochmillis epochmillis_str], default: :iso
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
87
|
+
|
88
|
+
desc 'Metadata prefix'
|
89
|
+
config_param :metadata_prefix, :string, default: DEFAULT_METADATA_PREFIX
|
90
|
+
|
91
|
+
desc 'Index suffix to clean, to aggregate_index'
|
92
|
+
config_param :aggregate_index_clean_suffix, :array, value_type: :regexp,
|
93
|
+
default: DEFAULT_AGGREGATE_INDEX_CLEAN_SUFFIX
|
94
|
+
|
95
|
+
desc 'Aggregate interval, to aggregate metrics by time'
|
96
|
+
config_param :aggregate_interval, :integer, default: DEFAULT_AGGREGATE_INTERVAL
|
97
|
+
|
85
98
|
desc 'Events block size'
|
86
99
|
config_param :event_stream_size, :integer, default: 1000
|
87
100
|
|
@@ -105,7 +118,7 @@ module Fluent
|
|
105
118
|
end
|
106
119
|
|
107
120
|
def check_configuration_keys
|
108
|
-
keys =
|
121
|
+
keys = CONFIGURATION_KEYS
|
109
122
|
invalid_keys = keys.each_with_object([]) do |key, invalid|
|
110
123
|
key_label = "#{key}_key"
|
111
124
|
key_value = send(key_label)
|
@@ -115,11 +128,12 @@ module Fluent
|
|
115
128
|
raise Fluent::ConfigError, "#{NAME}: #{invalid_keys} are empty" if invalid_keys.any?
|
116
129
|
end
|
117
130
|
|
118
|
-
def process(
|
119
|
-
|
131
|
+
def process(es_tag, es)
|
132
|
+
time = Fluent::EventTime.now
|
133
|
+
metrics = metric_processor.process(es_tag, es) || []
|
120
134
|
metrics.each_slice(event_stream_size) do |metrics_slice|
|
121
135
|
metrics_es = MultiEventStream.new
|
122
|
-
metrics_slice.each { |
|
136
|
+
metrics_slice.each { |record| metrics_es.add(time, record) }
|
123
137
|
router.emit_stream(tag, metrics_es)
|
124
138
|
end
|
125
139
|
end
|
metadata
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elastic-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thomas Tych
|
8
|
-
autorequire:
|
9
8
|
bindir: bin
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 2025-03-20 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
13
|
name: bump
|
@@ -16,28 +15,34 @@ dependencies:
|
|
16
15
|
requirements:
|
17
16
|
- - "~>"
|
18
17
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.10
|
18
|
+
version: '0.10'
|
20
19
|
type: :development
|
21
20
|
prerelease: false
|
22
21
|
version_requirements: !ruby/object:Gem::Requirement
|
23
22
|
requirements:
|
24
23
|
- - "~>"
|
25
24
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.10
|
25
|
+
version: '0.10'
|
27
26
|
- !ruby/object:Gem::Dependency
|
28
27
|
name: bundler
|
29
28
|
requirement: !ruby/object:Gem::Requirement
|
30
29
|
requirements:
|
31
30
|
- - "~>"
|
32
31
|
- !ruby/object:Gem::Version
|
33
|
-
version: '2.
|
32
|
+
version: '2.6'
|
33
|
+
- - ">="
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: 2.6.5
|
34
36
|
type: :development
|
35
37
|
prerelease: false
|
36
38
|
version_requirements: !ruby/object:Gem::Requirement
|
37
39
|
requirements:
|
38
40
|
- - "~>"
|
39
41
|
- !ruby/object:Gem::Version
|
40
|
-
version: '2.
|
42
|
+
version: '2.6'
|
43
|
+
- - ">="
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 2.6.5
|
41
46
|
- !ruby/object:Gem::Dependency
|
42
47
|
name: byebug
|
43
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -58,76 +63,154 @@ dependencies:
|
|
58
63
|
- - ">="
|
59
64
|
- !ruby/object:Gem::Version
|
60
65
|
version: 11.1.3
|
66
|
+
- !ruby/object:Gem::Dependency
|
67
|
+
name: mocha
|
68
|
+
requirement: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - "~>"
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '2.7'
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 2.7.1
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '2.7'
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 2.7.1
|
61
86
|
- !ruby/object:Gem::Dependency
|
62
87
|
name: rake
|
63
88
|
requirement: !ruby/object:Gem::Requirement
|
64
89
|
requirements:
|
65
90
|
- - "~>"
|
66
91
|
- !ruby/object:Gem::Version
|
67
|
-
version: 13.
|
92
|
+
version: '13.2'
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: 13.2.1
|
68
96
|
type: :development
|
69
97
|
prerelease: false
|
70
98
|
version_requirements: !ruby/object:Gem::Requirement
|
71
99
|
requirements:
|
72
100
|
- - "~>"
|
73
101
|
- !ruby/object:Gem::Version
|
74
|
-
version: 13.
|
102
|
+
version: '13.2'
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: 13.2.1
|
75
106
|
- !ruby/object:Gem::Dependency
|
76
107
|
name: reek
|
77
108
|
requirement: !ruby/object:Gem::Requirement
|
78
109
|
requirements:
|
79
110
|
- - "~>"
|
80
111
|
- !ruby/object:Gem::Version
|
81
|
-
version: 6.
|
112
|
+
version: '6.4'
|
82
113
|
type: :development
|
83
114
|
prerelease: false
|
84
115
|
version_requirements: !ruby/object:Gem::Requirement
|
85
116
|
requirements:
|
86
117
|
- - "~>"
|
87
118
|
- !ruby/object:Gem::Version
|
88
|
-
version: 6.
|
119
|
+
version: '6.4'
|
89
120
|
- !ruby/object:Gem::Dependency
|
90
121
|
name: rubocop
|
91
122
|
requirement: !ruby/object:Gem::Requirement
|
92
123
|
requirements:
|
93
124
|
- - "~>"
|
94
125
|
- !ruby/object:Gem::Version
|
95
|
-
version: 1.
|
126
|
+
version: '1.73'
|
127
|
+
- - ">="
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: 1.73.1
|
96
130
|
type: :development
|
97
131
|
prerelease: false
|
98
132
|
version_requirements: !ruby/object:Gem::Requirement
|
99
133
|
requirements:
|
100
134
|
- - "~>"
|
101
135
|
- !ruby/object:Gem::Version
|
102
|
-
version: 1.
|
136
|
+
version: '1.73'
|
137
|
+
- - ">="
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: 1.73.1
|
103
140
|
- !ruby/object:Gem::Dependency
|
104
141
|
name: rubocop-rake
|
105
142
|
requirement: !ruby/object:Gem::Requirement
|
106
143
|
requirements:
|
107
144
|
- - "~>"
|
108
145
|
- !ruby/object:Gem::Version
|
109
|
-
version: 0.
|
146
|
+
version: '0.7'
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 0.7.1
|
110
150
|
type: :development
|
111
151
|
prerelease: false
|
112
152
|
version_requirements: !ruby/object:Gem::Requirement
|
113
153
|
requirements:
|
114
154
|
- - "~>"
|
115
155
|
- !ruby/object:Gem::Version
|
116
|
-
version: 0.
|
156
|
+
version: '0.7'
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: 0.7.1
|
160
|
+
- !ruby/object:Gem::Dependency
|
161
|
+
name: simplecov
|
162
|
+
requirement: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0.22'
|
167
|
+
type: :development
|
168
|
+
prerelease: false
|
169
|
+
version_requirements: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - "~>"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0.22'
|
117
174
|
- !ruby/object:Gem::Dependency
|
118
175
|
name: test-unit
|
119
176
|
requirement: !ruby/object:Gem::Requirement
|
120
177
|
requirements:
|
121
178
|
- - "~>"
|
122
179
|
- !ruby/object:Gem::Version
|
123
|
-
version: 3.
|
180
|
+
version: '3.6'
|
181
|
+
- - ">="
|
182
|
+
- !ruby/object:Gem::Version
|
183
|
+
version: 3.6.7
|
184
|
+
type: :development
|
185
|
+
prerelease: false
|
186
|
+
version_requirements: !ruby/object:Gem::Requirement
|
187
|
+
requirements:
|
188
|
+
- - "~>"
|
189
|
+
- !ruby/object:Gem::Version
|
190
|
+
version: '3.6'
|
191
|
+
- - ">="
|
192
|
+
- !ruby/object:Gem::Version
|
193
|
+
version: 3.6.7
|
194
|
+
- !ruby/object:Gem::Dependency
|
195
|
+
name: timecop
|
196
|
+
requirement: !ruby/object:Gem::Requirement
|
197
|
+
requirements:
|
198
|
+
- - "~>"
|
199
|
+
- !ruby/object:Gem::Version
|
200
|
+
version: '0.9'
|
201
|
+
- - ">="
|
202
|
+
- !ruby/object:Gem::Version
|
203
|
+
version: 0.9.10
|
124
204
|
type: :development
|
125
205
|
prerelease: false
|
126
206
|
version_requirements: !ruby/object:Gem::Requirement
|
127
207
|
requirements:
|
128
208
|
- - "~>"
|
129
209
|
- !ruby/object:Gem::Version
|
130
|
-
version:
|
210
|
+
version: '0.9'
|
211
|
+
- - ">="
|
212
|
+
- !ruby/object:Gem::Version
|
213
|
+
version: 0.9.10
|
131
214
|
- !ruby/object:Gem::Dependency
|
132
215
|
name: fluentd
|
133
216
|
requirement: !ruby/object:Gem::Requirement
|
@@ -148,7 +231,6 @@ dependencies:
|
|
148
231
|
- - "<"
|
149
232
|
- !ruby/object:Gem::Version
|
150
233
|
version: '2'
|
151
|
-
description:
|
152
234
|
email:
|
153
235
|
- thomas.tych@gmail.com
|
154
236
|
executables: []
|
@@ -156,7 +238,6 @@ extensions: []
|
|
156
238
|
extra_rdoc_files: []
|
157
239
|
files:
|
158
240
|
- ".rubocop.yml"
|
159
|
-
- ".ruby-version"
|
160
241
|
- Gemfile
|
161
242
|
- LICENSE
|
162
243
|
- README.md
|
@@ -165,13 +246,13 @@ files:
|
|
165
246
|
- lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb
|
166
247
|
- lib/fluent/plugin/elastic_log/failed_login_metric.rb
|
167
248
|
- lib/fluent/plugin/elastic_log/granted_privileges_metric.rb
|
249
|
+
- lib/fluent/plugin/elastic_log/metric_accumulator.rb
|
168
250
|
- lib/fluent/plugin/out_elastic_audit_log_metric.rb
|
169
251
|
homepage: https://gitlab.com/ttych/fluent-plugin-elastic-log
|
170
252
|
licenses:
|
171
253
|
- Apache-2.0
|
172
254
|
metadata:
|
173
255
|
rubygems_mfa_required: 'true'
|
174
|
-
post_install_message:
|
175
256
|
rdoc_options: []
|
176
257
|
require_paths:
|
177
258
|
- lib
|
@@ -179,15 +260,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
179
260
|
requirements:
|
180
261
|
- - ">="
|
181
262
|
- !ruby/object:Gem::Version
|
182
|
-
version: 2.
|
263
|
+
version: 2.7.0
|
183
264
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
184
265
|
requirements:
|
185
266
|
- - ">="
|
186
267
|
- !ruby/object:Gem::Version
|
187
268
|
version: '0'
|
188
269
|
requirements: []
|
189
|
-
rubygems_version: 3.5
|
190
|
-
signing_key:
|
270
|
+
rubygems_version: 3.6.5
|
191
271
|
specification_version: 4
|
192
272
|
summary: fluentd plugins to process elastic logs.'
|
193
273
|
test_files: []
|
data/.ruby-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
2.4.10
|