fluent-plugin-elastic-log 0.4.0 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 89de05403ee1b497031fd3d6a833dadf22c47d5d78fdcabed897be853cebd8e5
|
|
4
|
+
data.tar.gz: f71df586cf16bc07db17f7fc0ed4b24ee08458ad3b02d9e62152e94cebe5f647
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4b749a87135558490c9fa2fd8475e03a24870c602f31b049548ea87907dfd092a37c6024506c4240055173993793a9368d3e256bc69dca97587168f758a4402a
|
|
7
|
+
data.tar.gz: c9b796602e030148fb46c50c981b09d333c32897025da0afd5bd977a21080a030d8e0de177924f455a952d1ae4aac4b02c9304ff46a48968b57ea76bb97f82d4
|
data/README.md
CHANGED
|
@@ -1,40 +1,65 @@
|
|
|
1
1
|
# fluent-plugin-elastic-log
|
|
2
2
|
|
|
3
|
-
[Fluentd](https://fluentd.org/) filter plugin to
|
|
3
|
+
[Fluentd](https://fluentd.org/) filter plugin to process elastic logs.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## plugins
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
### out - elastic_audit_log_metric
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
process audit logs and transform to metrics.
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
Example:
|
|
12
|
+
|
|
13
|
+
``` conf
|
|
14
|
+
<match my_tag_pattern>
|
|
15
|
+
@type elastic_audit_log_metric
|
|
16
|
+
|
|
17
|
+
tag elastic_audit_log_metric
|
|
18
|
+
timestamp_key timestamp
|
|
19
|
+
timestamp_format epochmillis
|
|
20
|
+
prefix tags_
|
|
21
|
+
</match>
|
|
13
22
|
```
|
|
14
23
|
|
|
15
|
-
|
|
24
|
+
parameters are:
|
|
25
|
+
* tag : Tag to emit metric events
|
|
26
|
+
|
|
27
|
+
parameters for input record:
|
|
28
|
+
* categories: Categories selected to be converted to metrics
|
|
29
|
+
* category_key: Category key in input record
|
|
30
|
+
* layer_key: Layer key in input record
|
|
31
|
+
* request_type_key: Request type key in input record
|
|
32
|
+
* cluster_key: Cluster key in input record
|
|
33
|
+
* user_key: Request user key in input record
|
|
34
|
+
* indices_key: Indices key in input record
|
|
35
|
+
* r_indices_key: Resolved indices key in input record
|
|
36
|
+
* timestamp_key: Timestamp key in input record
|
|
37
|
+
* privilege_key: Request privilege key in input record
|
|
38
|
+
|
|
39
|
+
parameters for output metric:
|
|
40
|
+
* timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
|
|
41
|
+
* prefix: Attribute prefix for output metric
|
|
42
|
+
* aggregate_ilm: Aggregate ILM on resolved indices
|
|
43
|
+
|
|
44
|
+
More details from the
|
|
45
|
+
[elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
|
|
16
46
|
|
|
17
|
-
|
|
47
|
+
## Installation
|
|
18
48
|
|
|
19
|
-
```ruby
|
|
20
|
-
gem "fluent-plugin-elastic-log"
|
|
21
|
-
```
|
|
22
49
|
|
|
23
|
-
|
|
50
|
+
Manual install, by executing:
|
|
24
51
|
|
|
25
|
-
|
|
26
|
-
$ bundle
|
|
27
|
-
```
|
|
52
|
+
$ gem install fluent-plugin-elastic-log
|
|
28
53
|
|
|
29
|
-
|
|
54
|
+
Add to Gemfile with:
|
|
30
55
|
|
|
31
|
-
|
|
56
|
+
$ bundle add fluent-plugin-elastic-log
|
|
32
57
|
|
|
33
|
-
|
|
34
|
-
$ fluent-plugin-config-format filter elastic-log
|
|
35
|
-
```
|
|
58
|
+
## Compatibility
|
|
36
59
|
|
|
37
|
-
|
|
60
|
+
plugin in 1.x.x will work with:
|
|
61
|
+
- ruby >= 2.4.10
|
|
62
|
+
- td-agent >= 3.8.1-0
|
|
38
63
|
|
|
39
64
|
## Copyright
|
|
40
65
|
|
|
@@ -14,7 +14,7 @@ module Fluent
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def process(_tag, log_es)
|
|
17
|
-
metric_es =
|
|
17
|
+
metric_es = []
|
|
18
18
|
|
|
19
19
|
log_es.each do |time, record|
|
|
20
20
|
next unless record
|
|
@@ -22,7 +22,7 @@ module Fluent
|
|
|
22
22
|
next unless conf.categories.include? category
|
|
23
23
|
|
|
24
24
|
new_records = send("generate_#{category.downcase}_metrics_for", record)
|
|
25
|
-
new_records
|
|
25
|
+
new_records&.each { |new_record| metric_es << [time, new_record] }
|
|
26
26
|
end
|
|
27
27
|
metric_es
|
|
28
28
|
end
|
|
@@ -31,7 +31,7 @@ module Fluent
|
|
|
31
31
|
|
|
32
32
|
# rubocop:disable Metrics/AbcSize
|
|
33
33
|
def generate_granted_privileges_metrics_for(record)
|
|
34
|
-
return unless record[conf.privilege_key]
|
|
34
|
+
return [] unless record[conf.privilege_key]
|
|
35
35
|
|
|
36
36
|
GrantedPrivilegesMetric.new(
|
|
37
37
|
record: {
|
|
@@ -76,6 +76,8 @@ module Fluent
|
|
|
76
76
|
config_param :prefix, :string, default: DEFAULT_PREFIX
|
|
77
77
|
desc 'Aggregate ILM'
|
|
78
78
|
config_param :aggregate_ilm, :bool, default: true
|
|
79
|
+
desc 'Events block size'
|
|
80
|
+
config_param :event_stream_size, :integer, default: 1000
|
|
79
81
|
|
|
80
82
|
attr_reader :metric_processor
|
|
81
83
|
|
|
@@ -109,7 +111,11 @@ module Fluent
|
|
|
109
111
|
|
|
110
112
|
def process(_tag, es)
|
|
111
113
|
metrics = metric_processor.process(tag, es) || []
|
|
112
|
-
|
|
114
|
+
metrics.each_slice(event_stream_size) do |metrics_slice|
|
|
115
|
+
metrics_es = MultiEventStream.new
|
|
116
|
+
metrics_slice.each { |time, record| metrics_es.add(time, record) }
|
|
117
|
+
router.emit_stream(tag, metrics_es)
|
|
118
|
+
end
|
|
113
119
|
end
|
|
114
120
|
end
|
|
115
121
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-elastic-log
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Thomas Tych
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2023-06-
|
|
11
|
+
date: 2023-06-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bump
|