fluent-plugin-elastic-log 0.4.0 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '089f4560a3510e19726b2236b1085bfd9d2bcc751ef9fbdb3200385348eb2505'
4
- data.tar.gz: 0b09fcff0fd33999c05a55191a766161ebf09464fc130a4ab80271352ec896e0
3
+ metadata.gz: 89de05403ee1b497031fd3d6a833dadf22c47d5d78fdcabed897be853cebd8e5
4
+ data.tar.gz: f71df586cf16bc07db17f7fc0ed4b24ee08458ad3b02d9e62152e94cebe5f647
5
5
  SHA512:
6
- metadata.gz: 93fbd475a195e8cd124160cbbf3ffb76f77023e4926bae097e533398950b27a68d99e9d13515b872963b6541f753941041b2b9a09a5c195cbbffc7d2cad1f095
7
- data.tar.gz: 608e7042c5ec2facaf5a5ae58fc9cd718ff2aa010dcfee4a12e785e22ce685100571389e97107f16f1cf144d7e6d2b01d93390b36ad58243963e69a07ec17cc5
6
+ metadata.gz: 4b749a87135558490c9fa2fd8475e03a24870c602f31b049548ea87907dfd092a37c6024506c4240055173993793a9368d3e256bc69dca97587168f758a4402a
7
+ data.tar.gz: c9b796602e030148fb46c50c981b09d333c32897025da0afd5bd977a21080a030d8e0de177924f455a952d1ae4aac4b02c9304ff46a48968b57ea76bb97f82d4
data/README.md CHANGED
@@ -1,40 +1,65 @@
1
1
  # fluent-plugin-elastic-log
2
2
 
3
- [Fluentd](https://fluentd.org/) filter plugin to do something.
3
+ [Fluentd](https://fluentd.org/) filter plugin to process elastic logs.
4
4
 
5
- TODO: write description for you plugin.
5
+ ## plugins
6
6
 
7
- ## Installation
7
+ ### out - elastic_audit_log_metric
8
8
 
9
- ### RubyGems
9
+ process audit logs and transform to metrics.
10
10
 
11
- ```
12
- $ gem install fluent-plugin-elastic-log
11
+ Example:
12
+
13
+ ``` conf
14
+ <match my_tag_pattern>
15
+ @type elastic_audit_log_metric
16
+
17
+ tag elastic_audit_log_metric
18
+ timestamp_key timestamp
19
+ timestamp_format epochmillis
20
+ prefix tags_
21
+ </match>
13
22
  ```
14
23
 
15
- ### Bundler
24
+ parameters are:
25
+ * tag : Tag to emit metric events
26
+
27
+ parameters for input record:
28
+ * categories: Categories selected to be converted to metrics
29
+ * category_key: Category key in input record
30
+ * layer_key: Layer key in input record
31
+ * request_type_key: Request type key in input record
32
+ * cluster_key: Cluster key in input record
33
+ * user_key: Request user key in input record
34
+ * indices_key: Indices key in input record
35
+ * r_indices_key: Resolved indices key in input record
36
+ * timestamp_key: Timestamp key in input record
37
+ * privilege_key: Request privilege key in input record
38
+
39
+ parameters for output metric:
40
+ * timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
41
+ * prefix: Attribute prefix for output metric
42
+ * aggregate_ilm: Aggregate ILM on resolved indices
43
+
44
+ More details from the
45
+ [elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
16
46
 
17
- Add following line to your Gemfile:
47
+ ## Installation
18
48
 
19
- ```ruby
20
- gem "fluent-plugin-elastic-log"
21
- ```
22
49
 
23
- And then execute:
50
+ Manual install, by executing:
24
51
 
25
- ```
26
- $ bundle
27
- ```
52
+ $ gem install fluent-plugin-elastic-log
28
53
 
29
- ## Configuration
54
+ Add to Gemfile with:
30
55
 
31
- You can generate configuration template:
56
+ $ bundle add fluent-plugin-elastic-log
32
57
 
33
- ```
34
- $ fluent-plugin-config-format filter elastic-log
35
- ```
58
+ ## Compatibility
36
59
 
37
- You can copy and paste generated documents here.
60
+ plugin in 1.x.x will work with:
61
+ - ruby >= 2.4.10
62
+ - td-agent >= 3.8.1-0
38
63
 
39
64
  ## Copyright
40
65
 
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'fluent-plugin-elastic-log'
8
- spec.version = '0.4.0'
8
+ spec.version = '0.4.2'
9
9
  spec.authors = ['Thomas Tych']
10
10
  spec.email = ['thomas.tych@gmail.com']
11
11
 
@@ -14,7 +14,7 @@ module Fluent
14
14
  end
15
15
 
16
16
  def process(_tag, log_es)
17
- metric_es = MultiEventStream.new
17
+ metric_es = []
18
18
 
19
19
  log_es.each do |time, record|
20
20
  next unless record
@@ -22,7 +22,7 @@ module Fluent
22
22
  next unless conf.categories.include? category
23
23
 
24
24
  new_records = send("generate_#{category.downcase}_metrics_for", record)
25
- new_records.each { |new_record| metric_es.add(time, new_record) }
25
+ new_records&.each { |new_record| metric_es << [time, new_record] }
26
26
  end
27
27
  metric_es
28
28
  end
@@ -31,7 +31,7 @@ module Fluent
31
31
 
32
32
  # rubocop:disable Metrics/AbcSize
33
33
  def generate_granted_privileges_metrics_for(record)
34
- return unless record[conf.privilege_key]
34
+ return [] unless record[conf.privilege_key]
35
35
 
36
36
  GrantedPrivilegesMetric.new(
37
37
  record: {
@@ -76,6 +76,8 @@ module Fluent
76
76
  config_param :prefix, :string, default: DEFAULT_PREFIX
77
77
  desc 'Aggregate ILM'
78
78
  config_param :aggregate_ilm, :bool, default: true
79
+ desc 'Events block size'
80
+ config_param :event_stream_size, :integer, default: 1000
79
81
 
80
82
  attr_reader :metric_processor
81
83
 
@@ -109,7 +111,11 @@ module Fluent
109
111
 
110
112
  def process(_tag, es)
111
113
  metrics = metric_processor.process(tag, es) || []
112
- router.emit_stream(tag, metrics) if metrics
114
+ metrics.each_slice(event_stream_size) do |metrics_slice|
115
+ metrics_es = MultiEventStream.new
116
+ metrics_slice.each { |time, record| metrics_es.add(time, record) }
117
+ router.emit_stream(tag, metrics_es)
118
+ end
113
119
  end
114
120
  end
115
121
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elastic-log
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Tych
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-06-14 00:00:00.000000000 Z
11
+ date: 2023-06-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bump