fluent-plugin-elastic-log 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 89de05403ee1b497031fd3d6a833dadf22c47d5d78fdcabed897be853cebd8e5
|
|
4
|
+
data.tar.gz: f71df586cf16bc07db17f7fc0ed4b24ee08458ad3b02d9e62152e94cebe5f647
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4b749a87135558490c9fa2fd8475e03a24870c602f31b049548ea87907dfd092a37c6024506c4240055173993793a9368d3e256bc69dca97587168f758a4402a
|
|
7
|
+
data.tar.gz: c9b796602e030148fb46c50c981b09d333c32897025da0afd5bd977a21080a030d8e0de177924f455a952d1ae4aac4b02c9304ff46a48968b57ea76bb97f82d4
|
data/README.md
CHANGED
|
@@ -1,40 +1,65 @@
|
|
|
1
1
|
# fluent-plugin-elastic-log
|
|
2
2
|
|
|
3
|
-
[Fluentd](https://fluentd.org/) filter plugin to
|
|
3
|
+
[Fluentd](https://fluentd.org/) filter plugin to process elastic logs.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## plugins
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
### out - elastic_audit_log_metric
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
process audit logs and transform to metrics.
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
Example:
|
|
12
|
+
|
|
13
|
+
``` conf
|
|
14
|
+
<match my_tag_pattern>
|
|
15
|
+
@type elastic_audit_log_metric
|
|
16
|
+
|
|
17
|
+
tag elastic_audit_log_metric
|
|
18
|
+
timestamp_key timestamp
|
|
19
|
+
timestamp_format epochmillis
|
|
20
|
+
prefix tags_
|
|
21
|
+
</match>
|
|
13
22
|
```
|
|
14
23
|
|
|
15
|
-
|
|
24
|
+
parameters are:
|
|
25
|
+
* tag : Tag to emit metric events
|
|
26
|
+
|
|
27
|
+
parameters for input record:
|
|
28
|
+
* categories: Categories selected to be converted to metrics
|
|
29
|
+
* category_key: Category key in input record
|
|
30
|
+
* layer_key: Layer key in input record
|
|
31
|
+
* request_type_key: Request type key in input record
|
|
32
|
+
* cluster_key: Cluster key in input record
|
|
33
|
+
* user_key: Request user key in input record
|
|
34
|
+
* indices_key: Indices key in input record
|
|
35
|
+
* r_indices_key: Resolved indices key in input record
|
|
36
|
+
* timestamp_key: Timestamp key in input record
|
|
37
|
+
* privilege_key: Request privilege key in input record
|
|
38
|
+
|
|
39
|
+
parameters for output metric:
|
|
40
|
+
* timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
|
|
41
|
+
* prefix: Attribute prefix for output metric
|
|
42
|
+
* aggregate_ilm: Aggregate ILM on resolved indices
|
|
43
|
+
|
|
44
|
+
More details from the
|
|
45
|
+
[elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
|
|
16
46
|
|
|
17
|
-
|
|
47
|
+
## Installation
|
|
18
48
|
|
|
19
|
-
```ruby
|
|
20
|
-
gem "fluent-plugin-elastic-log"
|
|
21
|
-
```
|
|
22
49
|
|
|
23
|
-
|
|
50
|
+
Manual install, by executing:
|
|
24
51
|
|
|
25
|
-
|
|
26
|
-
$ bundle
|
|
27
|
-
```
|
|
52
|
+
$ gem install fluent-plugin-elastic-log
|
|
28
53
|
|
|
29
|
-
|
|
54
|
+
Add to Gemfile with:
|
|
30
55
|
|
|
31
|
-
|
|
56
|
+
$ bundle add fluent-plugin-elastic-log
|
|
32
57
|
|
|
33
|
-
|
|
34
|
-
$ fluent-plugin-config-format filter elastic-log
|
|
35
|
-
```
|
|
58
|
+
## Compatibility
|
|
36
59
|
|
|
37
|
-
|
|
60
|
+
plugin in 1.x.x will work with:
|
|
61
|
+
- ruby >= 2.4.10
|
|
62
|
+
- td-agent >= 3.8.1-0
|
|
38
63
|
|
|
39
64
|
## Copyright
|
|
40
65
|
|
|
@@ -14,7 +14,7 @@ module Fluent
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def process(_tag, log_es)
|
|
17
|
-
metric_es =
|
|
17
|
+
metric_es = []
|
|
18
18
|
|
|
19
19
|
log_es.each do |time, record|
|
|
20
20
|
next unless record
|
|
@@ -22,7 +22,7 @@ module Fluent
|
|
|
22
22
|
next unless conf.categories.include? category
|
|
23
23
|
|
|
24
24
|
new_records = send("generate_#{category.downcase}_metrics_for", record)
|
|
25
|
-
new_records&.each { |new_record| metric_es
|
|
25
|
+
new_records&.each { |new_record| metric_es << [time, new_record] }
|
|
26
26
|
end
|
|
27
27
|
metric_es
|
|
28
28
|
end
|
|
@@ -76,6 +76,8 @@ module Fluent
|
|
|
76
76
|
config_param :prefix, :string, default: DEFAULT_PREFIX
|
|
77
77
|
desc 'Aggregate ILM'
|
|
78
78
|
config_param :aggregate_ilm, :bool, default: true
|
|
79
|
+
desc 'Events block size'
|
|
80
|
+
config_param :event_stream_size, :integer, default: 1000
|
|
79
81
|
|
|
80
82
|
attr_reader :metric_processor
|
|
81
83
|
|
|
@@ -109,7 +111,11 @@ module Fluent
|
|
|
109
111
|
|
|
110
112
|
def process(_tag, es)
|
|
111
113
|
metrics = metric_processor.process(tag, es) || []
|
|
112
|
-
|
|
114
|
+
metrics.each_slice(event_stream_size) do |metrics_slice|
|
|
115
|
+
metrics_es = MultiEventStream.new
|
|
116
|
+
metrics_slice.each { |time, record| metrics_es.add(time, record) }
|
|
117
|
+
router.emit_stream(tag, metrics_es)
|
|
118
|
+
end
|
|
113
119
|
end
|
|
114
120
|
end
|
|
115
121
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-elastic-log
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Thomas Tych
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2023-06-
|
|
11
|
+
date: 2023-06-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bump
|