fluent-plugin-elastic-log 0.4.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '089f4560a3510e19726b2236b1085bfd9d2bcc751ef9fbdb3200385348eb2505'
-  data.tar.gz: 0b09fcff0fd33999c05a55191a766161ebf09464fc130a4ab80271352ec896e0
+  metadata.gz: 89de05403ee1b497031fd3d6a833dadf22c47d5d78fdcabed897be853cebd8e5
+  data.tar.gz: f71df586cf16bc07db17f7fc0ed4b24ee08458ad3b02d9e62152e94cebe5f647
 SHA512:
-  metadata.gz: 93fbd475a195e8cd124160cbbf3ffb76f77023e4926bae097e533398950b27a68d99e9d13515b872963b6541f753941041b2b9a09a5c195cbbffc7d2cad1f095
-  data.tar.gz: 608e7042c5ec2facaf5a5ae58fc9cd718ff2aa010dcfee4a12e785e22ce685100571389e97107f16f1cf144d7e6d2b01d93390b36ad58243963e69a07ec17cc5
+  metadata.gz: 4b749a87135558490c9fa2fd8475e03a24870c602f31b049548ea87907dfd092a37c6024506c4240055173993793a9368d3e256bc69dca97587168f758a4402a
+  data.tar.gz: c9b796602e030148fb46c50c981b09d333c32897025da0afd5bd977a21080a030d8e0de177924f455a952d1ae4aac4b02c9304ff46a48968b57ea76bb97f82d4

data/README.md

@@ -1,40 +1,65 @@
 # fluent-plugin-elastic-log
 
-[Fluentd](https://fluentd.org/) filter plugin to do something.
+[Fluentd](https://fluentd.org/) filter plugin to process elastic logs.
 
-TODO: write description for you plugin.
+## plugins
 
-## Installation
+### out - elastic_audit_log_metric
 
-### RubyGems
+process audit logs and transform to metrics.
 
-```
-$ gem install fluent-plugin-elastic-log
+Example:
+
+``` conf
+<match my_tag_pattern>
+  @type elastic_audit_log_metric
+
+  tag elastic_audit_log_metric
+  timestamp_key timestamp
+  timestamp_format epochmillis
+  prefix tags_
+</match>
 ```
 
-### Bundler
+parameters are:
+* tag : Tag to emit metric events
+
+parameters for input record:
+* categories: Categories selected to be converted to metrics
+* category_key: Category key in input record
+* layer_key: Layer key in input record
+* request_type_key: Request type key in input record
+* cluster_key: Cluster key in input record
+* user_key: Request user key in input record
+* indices_key: Indices key in input record
+* r_indices_key: Resolved indices key in input record
+* timestamp_key: Timestamp key in input record
+* privilege_key: Request privilege key in input record
+
+parameters for output metric:
+* timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
+* prefix: Attribute prefix for output metric
+* aggregate_ilm: Aggregate ILM on resolved indices
+
+More details from the
+[elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
 
-Add following line to your Gemfile:
+## Installation
 
-```ruby
-gem "fluent-plugin-elastic-log"
-```
 
-And then execute:
+Manual install, by executing:
 
-```
-$ bundle
-```
+    $ gem install fluent-plugin-elastic-log
 
-## Configuration
+Add to Gemfile with:
 
-You can generate configuration template:
+    $ bundle add fluent-plugin-elastic-log
 
-```
-$ fluent-plugin-config-format filter elastic-log
-```
+## Compatibility
 
-You can copy and paste generated documents here.
+plugin in 1.x.x will work with:
+- ruby >= 2.4.10
+- td-agent >= 3.8.1-0
 
 ## Copyright
 

data/fluent-plugin-elastic-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-elastic-log'
-  spec.version = '0.4.0'
+  spec.version = '0.4.2'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb

@@ -14,7 +14,7 @@ module Fluent
         end
 
         def process(_tag, log_es)
-          metric_es = MultiEventStream.new
+          metric_es = []
 
           log_es.each do |time, record|
             next unless record
@@ -22,7 +22,7 @@ module Fluent
             next unless conf.categories.include? category
 
             new_records = send("generate_#{category.downcase}_metrics_for", record)
-            new_records.each { |new_record| metric_es.add(time, new_record) }
+            new_records&.each { |new_record| metric_es << [time, new_record] }
           end
           metric_es
         end
@@ -31,7 +31,7 @@ module Fluent
 
         # rubocop:disable Metrics/AbcSize
         def generate_granted_privileges_metrics_for(record)
-          return unless record[conf.privilege_key]
+          return [] unless record[conf.privilege_key]
 
           GrantedPrivilegesMetric.new(
             record: {

data/lib/fluent/plugin/out_elastic_audit_log_metric.rb

@@ -76,6 +76,8 @@ module Fluent
       config_param :prefix, :string, default: DEFAULT_PREFIX
       desc 'Aggregate ILM'
       config_param :aggregate_ilm, :bool, default: true
+      desc 'Events block size'
+      config_param :event_stream_size, :integer, default: 1000
 
       attr_reader :metric_processor
 
@@ -109,7 +111,11 @@ module Fluent
 
       def process(_tag, es)
         metrics = metric_processor.process(tag, es) || []
-        router.emit_stream(tag, metrics) if metrics
+        metrics.each_slice(event_stream_size) do |metrics_slice|
+          metrics_es = MultiEventStream.new
+          metrics_slice.each { |time, record| metrics_es.add(time, record) }
+          router.emit_stream(tag, metrics_es)
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elastic-log
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-14 00:00:00.000000000 Z
+date: 2023-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump