dataoperations-aggregate 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 641acf44ae266d4790320ae506cc968b20fe9fdcbc63b7fb50b0cb41c13a77b6
-  data.tar.gz: adeed75be0d03687fa1700e3ee5c8048885a904b0e8ad5667969328e8a4472a0
+  metadata.gz: 88229d7d1a3eb43c59f593c15ed5350189f534927d849b8bb75e70583610bdb6
+  data.tar.gz: 812bea576d59f4211e659c6e6da8f37cdabbac7a01faadc9e8f09e3512f0ea7b
 SHA512:
-  metadata.gz: 3b7f039bc8f7046051a8150576eb6a29ba10edb6287251bd0980a211eecec41b69bcda10dde9cf8d920f5dcf07c7be1752b2b211dda5419f7fd7c5686cfd0195
-  data.tar.gz: 12f5981b5fc696c4cb0b36a68dd38a5ad814085bf8f3aded30cfb2e2446607f0971a6adc9220cb29f3615ca44a9673c8dcac18f8d825bd1a28755bdb4e92a3cd
+  metadata.gz: dabc7016ecefa788042959f1184b258fe70d2468ed5e72e9a675902c1f2f53941559fe5e5083887f131662daf65e4f6339bf788269509f41da3093dd090b68e2
+  data.tar.gz: d0da00e819b18b73d0a942e2b8b871af276300936043cb09ebec48e0277bd1eb283ffbc57fdee3e73c1d400aae0149d7cfc9634dc45bf7533a9f68daa18f730f

data/lib/dataoperations-aggregate.rb

@@ -9,9 +9,10 @@ module DataOperations
     DEFAULT_INTERVALS = [10].freeze
     DEFAULT_FLUSH_INTERVAL = 5
     DEFAULT_PROCESSING_MODE = :batch
+    DEFAULT_TIME_STARTED_MODE = :first_message
     DEFAULT_FIELD_NO_DATA_VALUE = 'no_data'.freeze
     DEFAULT_AGGREGATIONS = %w[sum min max mean median variance standard_deviation].freeze
-    VALID_AGGREGATIONS = %w[sum min max mean median variance standard_deviation].freeze
+    VALID_AGGREGATIONS = %w[sum min max mean median variance standard_deviation bucket].freeze
     DEFAULT_HASH_TIME_FORMAT = '%Y-%m-%dT%H'.freeze
     DEFAULT_INERVAL_SECONDS = 3600
 
@@ -24,11 +25,14 @@ module DataOperations
                    keep_interval: DEFAULT_KEEP_INTERVAL,
                    field_no_data_value: DEFAULT_FIELD_NO_DATA_VALUE,
                    processing_mode: DEFAULT_PROCESSING_MODE,
+                   time_started_mode: DEFAULT_TIME_STARTED_MODE,
                    aggregator_name: nil,
                    log: Logger.new(STDOUT),
                    aggregation_names:,
                    group_field_names:,
-                   aggregate_field_names:
+                   aggregate_field_names:,
+                   buckets:[],
+                   bucket_metrics:[]
                 )
       @aggregator = aggregator
       @time_format = time_format
@@ -39,7 +43,10 @@ module DataOperations
       @keep_interval = keep_interval
       @field_no_data_value = field_no_data_value
       @processing_mode = processing_mode
+      @time_started_mode = time_started_mode
       @aggregator_name = aggregator_name
+      @buckets = buckets
+      @bucket_metrics = bucket_metrics
 
 
       if aggregation_names.nil? || !aggregation_names.is_a?(Array)
@@ -117,14 +124,14 @@ module DataOperations
         aggregator_item['aggregate_fields'] = aggregate_detail
         aggregator_item['intervals'] = interval_detail
 
-        @aggregator_mutex.synchronize {@aggregator[hash_group_key] = aggregator_item}
+	@aggregator_mutex.synchronize {@aggregator[hash_group_key] = aggregator_item}
       end
 
       if !aggregator_item['aggregate_fields'].key?(aggregator_hash_key)
         hash_aggregator = {}
         hash_aggregator[:time_started] = Time.now.to_i
         hash_aggregator['processed'] = 1
-        @aggregator_mutex.synchronize {aggregator_item['aggregate_fields'][aggregator_hash_key] = hash_aggregator}
+	@aggregator_mutex.synchronize {aggregator_item['aggregate_fields'][aggregator_hash_key] = hash_aggregator}
       else
         aggregator_item['aggregate_fields'][aggregator_hash_key]['processed'] += 1
       end
@@ -216,16 +223,51 @@ module DataOperations
       # Aggregate data
       if aggregate_field_value.is_a?(Array)
         @aggregation_names.each do |operation|
-          data = aggregate_field_value.method(operation).call
-          aggregator_data["#{aggregate_field_key}_#{operation}"] = data
-
-          # Add aggregated data to interval
-          group_item_value['intervals'].keys[1..-1].each do |interval_secs|
-            interval_aggregator_item_key = (aggregator_item_key / interval_secs.to_i) * interval_secs.to_i
-            interval_aggregator_item_value = group_item_value['intervals'][interval_secs][interval_aggregator_item_key]
-            interval_aggregator_item_value['aggregate_fields'][aggregate_field_key][operation] << data
+          
+          #If bucket, calculate bucket for metric
+          if operation == 'bucket'
+            #If set buckets and set metrics to calculate (bucket values depends of ranges based in metric activity)
+            if !@buckets.nil? && !@bucket_metrics.nil? && @bucket_metrics.include?(aggregate_field_key)
+              data_bucket = calculate_buckets(aggregate_field_value, @buckets)
+             
+              data_bucket.each {|bucket,bucket_count|
+                #@log.info("#{aggregate_field_key}_#{bucket} = #{bucket_count}")
+                aggregator_data["#{aggregate_field_key}_bucket#{bucket}"] = bucket_count
+              }
+
+              # Add aggregated data to interval
+              group_item_value['intervals'].keys[1..-1].each do |interval_secs|
+
+                interval_aggregator_item_key = (aggregator_item_key / interval_secs.to_i) * interval_secs.to_i
+                interval_aggregator_item_value = group_item_value['intervals'][interval_secs][interval_aggregator_item_key]
+                data_bucket.each {|bucket,bucket_count|
+                  #@log.info("#{aggregate_field_key}_#{bucket} = #{bucket_count}")
+                  interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["bucket#{bucket}"] = [] if interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["bucket#{bucket}"].nil?
+                  interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["bucket#{bucket}"] << bucket_count
+                }
+                  
+              end
+
+            end
+          else
+            data = aggregate_field_value.method(operation).call
+            aggregator_data["#{aggregate_field_key}_#{operation}"] = data
+
+            # Add aggregated data to interval
+            group_item_value['intervals'].keys[1..-1].each do |interval_secs|
+              interval_aggregator_item_key = (aggregator_item_key / interval_secs.to_i) * interval_secs.to_i
+              interval_aggregator_item_value = group_item_value['intervals'][interval_secs][interval_aggregator_item_key]
+              interval_aggregator_item_value['aggregate_fields'][aggregate_field_key][operation] << data
+            end
           end
+
+
+        end
+
+        if !@buckets.nil? && ! @bucket_metrics.nil?
+          #data = calculate_buckets(data, @buckets)
         end
+
       end
     end
 
@@ -241,6 +283,9 @@ module DataOperations
         @aggregation_names.each do |operation|
           interval_aggregator_item_value['aggregate_fields'][aggregate_field_key][operation] = []
         end
+
+        ##Add buckets metadata (empty hash)
+        ##interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]['buckets']={}
       end
     end
 
@@ -249,7 +294,9 @@ module DataOperations
       # @log.debug  "interval_aggregator_item_key: #{interval_aggregator_item_key}"
 
       if interval_aggregator_item_value = group_item_value['intervals'][interval_secs][interval_aggregator_item_key]
-        if interval_aggregator_item_value[:time_started] < aggregator_item_value[:time_started]
+        if @time_started_mode == :first_event && aggregator_item_value[:time_started] < interval_aggregator_item_value[:time_started]
+          interval_aggregator_item_value[:time_started] = aggregator_item_value[:time_started]
+        elseif @time_started_mode == :last_event && aggregator_item_value[:time_started] > interval_aggregator_item_value[:time_started]
           interval_aggregator_item_value[:time_started] = aggregator_item_value[:time_started]
         end
         interval_aggregator_item_value['processed'] += aggregator_item_value['processed']
@@ -292,10 +339,17 @@ module DataOperations
             case operation
             when 'max', 'min', 'mean', 'median'
               data = vector.method(operation).call
+            when 'bucket'
+              #Bucket operation generate bucket[\d]+
+              data = nil
+            when /^bucket[\d]+/
+              #For buckets sum accumulations for internvals
+              data = vector.method('sum').call
             else
               data = vector.median
             end
-            aggregator_data["#{field_name}_#{operation}"] = data
+            #Nil data is avoid (for example for 'bucket' name operation)
+            aggregator_data["#{field_name}_#{operation}"] = data unless data.nil?
           end
         end
         # @log.debug aggregator_item_value
@@ -305,5 +359,24 @@ module DataOperations
         aggregate_data[s_interval] << aggregator_data
       end
     end
+    
+    #Return Array with count by each bucket
+    def calculate_buckets(data, buckets_config)
+      buckets_config.sort!.uniq!
+      buckets = {}
+    
+      buckets_config.each {|bucket| buckets[bucket] = 0}
+    
+      data.each {|item|
+        buckets_config.each {|bucket|
+          if item <= bucket
+            buckets[bucket] += 1
+            next
+          end
+        }
+      }
+      return buckets
+    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataoperations-aggregate
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Victor Guillen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-26 00:00:00.000000000 Z
+date: 2022-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: descriptive_statistics
@@ -50,7 +50,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Aggregate data