fluent-plugin-aggregate 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b58e7447b1b2957f0bc50b0f1266a0773b9865e894a7c0ec7284adf8f1755ed9
+  data.tar.gz: 42692ba88949646f48152a22d2f83155df34f5af9e90dd446c7796acbeb082ea
+SHA512:
+  metadata.gz: f1b7d30f0c24d17a3f9a73deb82bc17fe9b476d6e55346795b4dc2f7f90859b0365b05c1f1be1d7439887d0a926904dce4fd2f6a27af9642b395ae1700fdac2a
+  data.tar.gz: c2110a4e5a2917041ae110f88a079f9b4cd4f4131a5fb08614bbf2aa0a3a05d0601931fdea64a1f062c63df22e9b5b675edba03b2c1e8ffcdc243c7551e15c01

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 superguillen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# fluent-plugin-aggregate , a plugin for [Fluentd](http://fluentd.org)
+
+A fluentd plugin to aggregate events by fields over time.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-aggregate'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-aggregate --no-document
+
+## Requirements
+
+- Ruby 2.1 or later
+- fluentd v0.12 or later
+- aggregate v0.0.1 or later
+
+## Usage
+### Filter plugin (@type 'aggregate')
+
+Aggregate events grouping by fields over time.
+
+```
+<filter>
+  @type aggregate
+  intervals 5s
+  keep_interval 1s
+  group_fields field_group1,field_group2
+  aggregate_fields numeric_field1, numeric_field2
+  aggregations mean,median
+</filter>
+```
+### Common parameters
+### intervals
+Intervals for the aggregatios, this plugin support multi interval aggregatios
+```
+intervals 5s,10s,20s
+```
+### keep_interval
+Additional time to wait fof arrive events (used when events has a delay in the origin)
+```
+keep_interval 5s
+```
+### group_fields
+Fields to group events (like group by in SQL)
+```
+group_fields tx,region
+```
+### aggregate_fields
+Fields to apply aggregation funtions (like mean, median, sum, etc), this plugin support multiple aggregations fields.
+```
+aggregate_fields response_time,pressure
+```
+### aggregations
+Aggregate funtions to apply, this plugin support multiple aggregations fields.
+```
+aggregations sum,min,max,mean,median,variance,standard_deviation
+```

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/lib/fluent/plugin/filter_aggregate.rb

@@ -0,0 +1,176 @@
+module Fluent
+  class FilterAggregate < Filter
+    Fluent::Plugin.register_filter('aggregate', self)
+    config_param :aggregator_suffix_name, :string, :default => nil
+    config_param :time_format, :string, :default =>'%Y-%m-%dT%H:%M:%S.%L%:z'
+    config_param :output_time_format, :string, :default =>'%Y-%m-%dT%H:%M:%S.%L%z'
+    config_param :intervals, :array, :default =>[5], value_type: :time,
+	         :desc => 'Interval for accumulative  aggregation'
+    config_param :flush_interval, :time, :default =>5,
+	         :desc => 'Interval for emmit  aggregation events'
+    config_param :keep_interval, :time, :default =>10,
+	         :desc => 'Interval to wait for events to aggregate'
+    config_param :group_fields, :string, :default =>'field1,field2'
+    config_param :aggregate_fields, :string, :default =>'aggregate_field1,aggregate_field2'
+    config_param :time_field, :string, :default =>'timestamp'
+    config_param :field_no_data_value, :string, :default =>'no_data'
+    config_param :emit_original_message, :bool, :default => true
+    config_param :aggregate_event_tag, :string, :default => 'aggregate'
+    config_param :aggregations, :string, :default => 'sum,min,max,mean,median,variance,standard_deviation'
+    config_param :temporary_status_file_path, :string, :default => nil, :desc => 'File to store aggregate information when the agent down'
+    config_param :load_temporarystatus_file_enabled, :bool, :default => true, :desc => 'Enable load saved data from file (if exist status file)'
+
+    VALID_AGGREGATIONS = ['sum','min','max','mean','median','variance','standard_deviation']
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+
+      require 'dataoperations-aggregate'
+
+      @hash_time_format = "%Y-%m-%dT%H"
+      @interval_seconds = 3600
+      @intervals[1..-1].each{|interval|
+        case interval
+        when 1,5,10,20,30,60,120,180,240,300,600,900,1200,1800,3600
+    	  if ! (interval % @intervals[0]) == 0
+            raise Fluent::ConfigError, "interval must be multiple of default_aggregate_interval(#{@default_aggregate_interval}s)"
+	  end
+        else
+          raise Fluent::ConfigError, "interval must set to 1s,5s,10s,20s,30s,1m,5m,10m"
+        end
+      }
+
+      @group_field_names = @group_fields.split(",")
+      @aggregate_field_names = @aggregate_fields.split(",")
+      @aggregation_names = @aggregations.split(",")
+      @aggregator_name = "#{Socket.gethostname}"
+      @aggregator_name = "#{@aggregator_name}-#{@aggregator_suffix_name}" unless @aggregator_suffix_name.nil?
+
+      @aggregation_names.each {|operation|
+        if ! VALID_AGGREGATIONS.include?(operation)
+          raise Fluent::ConfigError, "aggregations must set any combination of sum,min,max,mean,median,variance,standard_deviation "
+        end
+      }
+
+      @aggregator = {}
+      if load_temporarystatus_file_enabled && ! @temporary_status_file_path.nil? && File.exist?(@temporary_status_file_path) && file_status = File.open(@temporary_status_file_path,'r')
+        begin
+          @aggregator=eval(file_status.read)
+          file_status.close
+          File.delete(@temporary_status_file_path)
+          log.info "Temporary information loaded from temporary_status_file_path:#{@temporary_status_file_path} before startup"
+        rescue Exception => e
+          log.warn "Failed to load temporary_status_file_path:#{@temporary_status_file_path}"
+          log.warn e.message
+          log.warn e.backtrace.inspect
+        end
+      end
+
+      @aggregator = {} unless @aggregator.is_a?(Hash)
+
+      log.warn "temporary_status_file_path is empty, is recomended using to avoid lost statistic information beetween restarts." if @temporary_status_file_path.nil?
+      @aggregator_mutex = Mutex.new
+      @data_operations = DataOperations::Aggregate.new(aggregator: @aggregator,
+                          time_format: @time_format,
+                          time_field: @time_field,
+                          output_time_format: @output_time_format,
+                          intervals: @intervals,
+                          flush_interval: @flush_interval,
+                          keep_interval: @keep_interval,
+                          field_no_data_value: @field_no_data_value,
+                          processing_mode: :online,
+                          log: log,
+                          aggregation_names: @aggregation_names,
+                          group_field_names: @group_field_names,
+                          aggregate_field_names: @aggregate_field_names)
+    end
+
+    def filter(tag, time, record)
+      result = nil
+      begin
+        result = record unless ! emit_original_message
+
+        @data_operations.add_events(result)
+        result
+      rescue => e
+        log.warn "failed to filter events", :error_class => e.class, :error => e.message
+        log.warn_backtrace
+      end
+      result
+    end
+
+    def start
+      super
+      @loop = Coolio::Loop.new
+      tw = TimerWatcher.new(@flush_interval, true, @log, &method(:aggregate_events))
+      tw.attach(@loop)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def run
+      @loop.run
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+
+    def shutdown
+      @loop.stop
+      @thread.join
+
+      if load_temporarystatus_file_enabled && ! @temporary_status_file_path.nil? && file_status = File.open(@temporary_status_file_path,'w')
+        begin
+          file_status.write @aggregator
+          file_status.close
+          log.info "Temporary information stored in temporary_status_file_path:#{@temporary_status_file_path} before shutdown"
+        rescue Exception => e
+          log.warn "Failed to load temporary_status_file_path:#{@temporary_status_file_path}"
+          log.warn e.message
+          log.warn e.backtrace.inspect
+        end
+      end
+
+      super
+    end
+
+    def aggregate_events
+
+      #log.trace @aggregator
+      @aggregator_mutex.synchronize do
+	data_aggregated = @data_operations.aggregate_events
+        data_aggregated.each{|s_interval,data_aggregated|
+          #log.trace data_aggregated
+          es = MultiEventStream.new
+          data_aggregated.each {|item|
+             es.add(item['time'], item)
+          }
+          unless es.empty?
+            tag="#{@aggregate_event_tag}.#{s_interval}"
+            router.emit_stream(tag, es)
+          end
+        } if data_aggregated
+      end
+
+    #rescue Exception => e
+    #  $log.error e
+    end
+
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, repeat, log, &callback)
+        @callback = callback
+        @log = log
+        super(interval, repeat)
+      end
+      def on_timer
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-aggregate
+version: !ruby/object:Gem::Version
+  version: '1.0'
+platform: ruby
+authors:
+- superguillen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-04-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: dataoperations-aggregate
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Filter aggregtation plugin for Fluent
+email: superguillen.public@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- lib/fluent/plugin/filter_aggregate.rb
+homepage: https://github.com/superguillen/fluent-plugin-aggregate
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Filter aggregtation plugin for Fluent
+test_files: []