fluent-plugin-elastic-log 0.3.0 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f99628e3afa35a188a2f1b94cf56b44a1cbac89f541852cd1791d4a635b6e658
4
- data.tar.gz: ae773fc1151757b4b1ebdc5ef463c05c722ab1d50e89d7e5b01f8c5187505252
3
+ metadata.gz: '08f18d938d7e838acc560c6857affc1821290a66bb4c9a203e039ca75de0b1db'
4
+ data.tar.gz: 3be561a99555350c031f26d0b1d97de9162a8deee8d32eb2dfb3c41e4008957a
5
5
  SHA512:
6
- metadata.gz: e9b473e3db2182d81103f0b5a39c8c7a240c2bcc00dd2e01f852baa55aca426c3edc466595c6feb878ead856cb3f787c9f0cd7f2d182bfb0dcc38d9d5b0498f7
7
- data.tar.gz: 544ddb9b34eed81fe9e758b65736ab4335b975ee575afddf36083ec91a026f506b1edea1d6f6a68fb914a88b12e234b375e58dcf84db1f89cd5615088a115ae4
6
+ metadata.gz: 41dc18b2e54f6d852caca69fffacce7ffed2cdf8a46dcbde37f4b9b0120940fd680a535178fbef331b669b9c60ba0154b87596bba741b35fa6d9abce22a361fa
7
+ data.tar.gz: b344dc9bf21982ff52a81b0ad0e395969f5e1d9e52efee63e75042d9a0b9a7c29313410a91c19f62d00fe15e5921ee9f1e0a47668017aa4f034be50393a9ad16
data/.rubocop.yml CHANGED
@@ -17,6 +17,10 @@ Metrics/ClassLength:
17
17
  Metrics/MethodLength:
18
18
  Max: 20
19
19
 
20
+ Metrics/ParameterLists:
21
+ Exclude:
22
+ - test/helper.rb
23
+
20
24
  Naming/MethodParameterName:
21
25
  Exclude:
22
26
  - lib/fluent/plugin/out_elastic_audit_log_metric.rb
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'fluent-plugin-elastic-log'
8
- spec.version = '0.3.0'
8
+ spec.version = '0.4.1'
9
9
  spec.authors = ['Thomas Tych']
10
10
  spec.email = ['thomas.tych@gmail.com']
11
11
 
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'granted_privileges_metric'
4
+
5
+ module Fluent
6
+ module Plugin
7
+ module ElasticLog
8
+ # convert audit log event stream to metric event stream
9
+ class AuditLogToMetricProcessor
10
+ attr_reader :conf
11
+
12
+ def initialize(conf:)
13
+ @conf = conf
14
+ end
15
+
16
+ def process(_tag, log_es)
17
+ metric_es = MultiEventStream.new
18
+
19
+ log_es.each do |time, record|
20
+ next unless record
21
+ next unless (category = record[conf.category_key])
22
+ next unless conf.categories.include? category
23
+
24
+ new_records = send("generate_#{category.downcase}_metrics_for", record)
25
+ new_records&.each { |new_record| metric_es.add(time, new_record) }
26
+ end
27
+ metric_es
28
+ end
29
+
30
+ private
31
+
32
+ # rubocop:disable Metrics/AbcSize
33
+ def generate_granted_privileges_metrics_for(record)
34
+ return [] unless record[conf.privilege_key]
35
+
36
+ GrantedPrivilegesMetric.new(
37
+ record: {
38
+ timestamp: record[conf.timestamp_key],
39
+ privilege: record[conf.privilege_key],
40
+ user: record[conf.user_key],
41
+ cluster: record[conf.cluster_key],
42
+ indices: record[conf.indices_key],
43
+ r_indices: record[conf.r_indices_key],
44
+ layer: record[conf.layer_key],
45
+ request_type: record[conf.request_type_key]
46
+ },
47
+ conf: conf
48
+ ).generate_metrics
49
+ end
50
+ # rubocop:enable Metrics/AbcSize
51
+ end
52
+ end
53
+ end
54
+ end
@@ -3,8 +3,6 @@
3
3
  require 'set'
4
4
  require 'time'
5
5
 
6
- require 'fluent/event'
7
-
8
6
  module Fluent
9
7
  module Plugin
10
8
  module ElasticLog
@@ -23,43 +21,39 @@ module Fluent
23
21
  PRIVILEGE_MAP = {
24
22
  'cluster:admin/' => 'admin',
25
23
  'cluster:monitor/' => 'monitor',
24
+ 'indices:admin/delete' => 'destroy',
26
25
  'indices:admin/' => 'admin',
27
26
  'indices:data/read/' => 'read',
27
+ 'indices:data/write/delete' => 'delete',
28
28
  'indices:data/write/' => 'write',
29
29
  'indices:monitor/' => 'monitor'
30
30
  }.freeze
31
31
 
32
32
  ILM_PATTERN = /^(.*)-\d{6}$/.freeze
33
33
 
34
- attr_reader :time, :record, :conf, :prefix
34
+ attr_reader :record, :conf
35
35
 
36
- def initialize(time:, record:, conf:, prefix: '')
37
- @time = time
36
+ def initialize(record:, conf:)
38
37
  @record = record
39
38
  @conf = conf
40
- @prefix = prefix
41
39
  end
42
40
 
43
- # rubocop:disable Metrics/AbcSize
44
41
  def timestamp
45
- begin
46
- timestamp = Time.parse(record[:timestamp])
47
- rescue StandardError
48
- timestamp = time.to_time
49
- end
42
+ timestamp = Time.parse(record[:timestamp])
50
43
 
51
44
  return (timestamp.utc.to_f * 1000).to_i if conf.timestamp_format == :epochmillis
52
45
  return timestamp.utc.strftime('%s%3N') if conf.timestamp_format == :epochmillis_str
53
46
 
54
47
  timestamp.utc.iso8601(3)
48
+ rescue StandardError
49
+ nil
55
50
  end
56
- # rubocop:enable Metrics/AbcSize
57
51
 
58
52
  def query_type
59
53
  PRIVILEGE_MAP.each do |pattern, name|
60
54
  return name if record[:privilege].to_s.start_with?(pattern)
61
55
  end
62
- 'unknown_count'
56
+ 'unknown'
63
57
  end
64
58
 
65
59
  def base
@@ -67,9 +61,9 @@ module Fluent
67
61
  'timestamp' => timestamp,
68
62
  'metric_name' => 'query_count',
69
63
  'metric_value' => 1,
70
- "#{prefix}user" => record[:user],
71
- "#{prefix}cluster" => record[:cluster],
72
- "#{prefix}query_type" => query_type
64
+ "#{conf.prefix}user" => record[:user],
65
+ "#{conf.prefix}cluster" => record[:cluster],
66
+ "#{conf.prefix}query_type" => query_type
73
67
  }
74
68
  end
75
69
 
@@ -77,19 +71,19 @@ module Fluent
77
71
  indices = record[:r_indices] || record[:indices] || [nil]
78
72
  if conf.aggregate_ilm
79
73
  indices = indices.inject(Set.new) do |acc, index|
80
- aggregated_format = index[ILM_PATTERN, 1]
74
+ aggregated_format = index && index[ILM_PATTERN, 1]
81
75
  acc << (aggregated_format || index)
82
76
  end.to_a
83
77
  end
84
78
  indices
85
79
  end
86
80
 
87
- def generate_event_stream
88
- metric_es = MultiEventStream.new
81
+ def generate_metrics
82
+ metrics = []
89
83
  indices.each do |indice|
90
- metric_es.add(time, base.merge("#{prefix}technical_name" => indice))
84
+ metrics << base.merge("#{conf.prefix}technical_name" => indice)
91
85
  end
92
- metric_es
86
+ metrics
93
87
  end
94
88
  end
95
89
  end
@@ -16,7 +16,9 @@
16
16
  # limitations under the License.
17
17
 
18
18
  require 'fluent/plugin/output'
19
- require 'fluent/plugin/elastic_log/granted_privileges_metric'
19
+ require 'fluent/event'
20
+
21
+ require_relative 'elastic_log/audit_log_to_metric_processor'
20
22
 
21
23
  module Fluent
22
24
  module Plugin
@@ -44,16 +46,6 @@ module Fluent
44
46
  DEFAULT_PRIVILEGE_KEY = 'audit_request_privilege'
45
47
  DEFAULT_PREFIX = ''
46
48
 
47
- # REQUEST PRIVILEGE:
48
- # cluster:
49
- # admin/* => admin
50
- # monitor/* => monitor
51
- # indices:
52
- # admin/* => admin
53
- # data/read/* => read
54
- # data/write/* => write
55
- # monitor/* => monitor
56
-
57
49
  desc 'Tag to emit metric events on'
58
50
  config_param :tag, :string, default: nil
59
51
  desc 'Categories selected to be converted to metrics'
@@ -85,6 +77,8 @@ module Fluent
85
77
  desc 'Aggregate ILM'
86
78
  config_param :aggregate_ilm, :bool, default: true
87
79
 
80
+ attr_reader :metric_processor
81
+
88
82
  def configure(conf)
89
83
  super
90
84
  raise Fluent::ConfigError, "#{NAME}: tag is mandatory" if !tag || tag.to_s.empty?
@@ -97,6 +91,8 @@ module Fluent
97
91
  @categories = categories - unsupported_categories
98
92
  end
99
93
 
94
+ @metric_processor = ElasticLog::AuditLogToMetricProcessor.new(conf: self)
95
+
100
96
  true
101
97
  end
102
98
 
@@ -112,43 +108,9 @@ module Fluent
112
108
  end
113
109
 
114
110
  def process(_tag, es)
115
- es.each do |time, record|
116
- next unless record
117
- next unless (category = record[category_key])
118
- next unless ALLOWED_CATEGORIES.include? category
119
-
120
- event_time = Fluent::EventTime.from_time(time)
121
- metric_es = send("generate_#{category.downcase}_metrics_for", event_time, record)
122
- router.emit_stream(tag, metric_es) if metric_es
123
- end
124
- end
125
-
126
- # es = Fluent::MultiEventStream.new
127
- # router.emit_stream(tag, es)
128
-
129
- private
130
-
131
- # rubocop:disable Metrics/AbcSize
132
- def generate_granted_privileges_metrics_for(time, record)
133
- return unless record[privilege_key]
134
-
135
- Fluent::Plugin::ElasticLog::GrantedPrivilegesMetric.new(
136
- time: time,
137
- record: {
138
- timestamp: record[timestamp_key],
139
- privilege: record[privilege_key],
140
- user: record[user_key],
141
- cluster: record[cluster_key],
142
- indices: record[indices_key],
143
- r_indices: record[r_indices_key],
144
- layer: record[layer_key],
145
- request_type: record[request_type_key]
146
- },
147
- conf: self,
148
- prefix: prefix
149
- ).generate_event_stream
111
+ metrics = metric_processor.process(tag, es) || []
112
+ router.emit_stream(tag, metrics) if metrics
150
113
  end
151
- # rubocop:enable Metrics/AbcSize
152
114
  end
153
115
  end
154
116
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elastic-log
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Tych
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-06-13 00:00:00.000000000 Z
11
+ date: 2023-06-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bump
@@ -162,6 +162,7 @@ files:
162
162
  - README.md
163
163
  - Rakefile
164
164
  - fluent-plugin-elastic-log.gemspec
165
+ - lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb
165
166
  - lib/fluent/plugin/elastic_log/granted_privileges_metric.rb
166
167
  - lib/fluent/plugin/out_elastic_audit_log_metric.rb
167
168
  homepage: https://gitlab.com/ttych/fluent-plugin-elastic-log