fluent-plugin-elastic-log 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08f18d938d7e838acc560c6857affc1821290a66bb4c9a203e039ca75de0b1db'
-  data.tar.gz: 3be561a99555350c031f26d0b1d97de9162a8deee8d32eb2dfb3c41e4008957a
+  metadata.gz: 89de05403ee1b497031fd3d6a833dadf22c47d5d78fdcabed897be853cebd8e5
+  data.tar.gz: f71df586cf16bc07db17f7fc0ed4b24ee08458ad3b02d9e62152e94cebe5f647
 SHA512:
-  metadata.gz: 41dc18b2e54f6d852caca69fffacce7ffed2cdf8a46dcbde37f4b9b0120940fd680a535178fbef331b669b9c60ba0154b87596bba741b35fa6d9abce22a361fa
-  data.tar.gz: b344dc9bf21982ff52a81b0ad0e395969f5e1d9e52efee63e75042d9a0b9a7c29313410a91c19f62d00fe15e5921ee9f1e0a47668017aa4f034be50393a9ad16
+  metadata.gz: 4b749a87135558490c9fa2fd8475e03a24870c602f31b049548ea87907dfd092a37c6024506c4240055173993793a9368d3e256bc69dca97587168f758a4402a
+  data.tar.gz: c9b796602e030148fb46c50c981b09d333c32897025da0afd5bd977a21080a030d8e0de177924f455a952d1ae4aac4b02c9304ff46a48968b57ea76bb97f82d4

data/README.md

@@ -1,40 +1,65 @@
 # fluent-plugin-elastic-log
 
-[Fluentd](https://fluentd.org/) filter plugin to do something.
+[Fluentd](https://fluentd.org/) filter plugin to process elastic logs.
 
-TODO: write description for you plugin.
+## plugins
 
-## Installation
+### out - elastic_audit_log_metric
 
-### RubyGems
+process audit logs and transform to metrics.
 
-```
-$ gem install fluent-plugin-elastic-log
+Example:
+
+``` conf
+<match my_tag_pattern>
+  @type elastic_audit_log_metric
+
+  tag elastic_audit_log_metric
+  timestamp_key timestamp
+  timestamp_format epochmillis
+  prefix tags_
+</match>
 ```
 
-### Bundler
+parameters are:
+* tag : Tag to emit metric events
+
+parameters for input record:
+* categories: Categories selected to be converted to metrics
+* category_key: Category key in input record
+* layer_key: Layer key in input record
+* request_type_key: Request type key in input record
+* cluster_key: Cluster key in input record
+* user_key: Request user key in input record
+* indices_key: Indices key in input record
+* r_indices_key: Resolved indices key in input record
+* timestamp_key: Timestamp key in input record
+* privilege_key: Request privilege key in input record
+
+parameters for output metric:
+* timestamp_format: Timestamp format (iso, epochmillis, epochmillis_str)
+* prefix: Attribute prefix for output metric
+* aggregate_ilm: Aggregate ILM on resolved indices
+
+More details from the
+[elastic_audit_log_metric output plugin code](lib/fluent/plugin/out_elastic_audit_log_metric.rb#L49)
 
-Add following line to your Gemfile:
+## Installation
 
-```ruby
-gem "fluent-plugin-elastic-log"
-```
 
-And then execute:
+Manual install, by executing:
 
-```
-$ bundle
-```
+    $ gem install fluent-plugin-elastic-log
 
-## Configuration
+Add to Gemfile with:
 
-You can generate configuration template:
+    $ bundle add fluent-plugin-elastic-log
 
-```
-$ fluent-plugin-config-format filter elastic-log
-```
+## Compatibility
 
-You can copy and paste generated documents here.
+plugin in 1.x.x will work with:
+- ruby >= 2.4.10
+- td-agent >= 3.8.1-0
 
 ## Copyright
 

data/fluent-plugin-elastic-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-elastic-log'
-  spec.version = '0.4.1'
+  spec.version = '0.4.2'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/elastic_log/audit_log_to_metric_processor.rb

@@ -14,7 +14,7 @@ module Fluent
         end
 
         def process(_tag, log_es)
-          metric_es = MultiEventStream.new
+          metric_es = []
 
           log_es.each do |time, record|
             next unless record
@@ -22,7 +22,7 @@ module Fluent
             next unless conf.categories.include? category
 
             new_records = send("generate_#{category.downcase}_metrics_for", record)
-            new_records&.each { |new_record| metric_es.add(time, new_record) }
+            new_records&.each { |new_record| metric_es << [time, new_record] }
           end
           metric_es
         end

data/lib/fluent/plugin/out_elastic_audit_log_metric.rb

@@ -76,6 +76,8 @@ module Fluent
       config_param :prefix, :string, default: DEFAULT_PREFIX
       desc 'Aggregate ILM'
       config_param :aggregate_ilm, :bool, default: true
+      desc 'Events block size'
+      config_param :event_stream_size, :integer, default: 1000
 
       attr_reader :metric_processor
 
@@ -109,7 +111,11 @@ module Fluent
 
       def process(_tag, es)
         metrics = metric_processor.process(tag, es) || []
-        router.emit_stream(tag, metrics) if metrics
+        metrics.each_slice(event_stream_size) do |metrics_slice|
+          metrics_es = MultiEventStream.new
+          metrics_slice.each { |time, record| metrics_es.add(time, record) }
+          router.emit_stream(tag, metrics_es)
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elastic-log
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Thomas Tych
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-15 00:00:00.000000000 Z
+date: 2023-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump