dataoperations-aggregate 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/dataoperations-aggregate.rb +64 -28
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e69958cbf826f7524a31c8621afd7c554c61ceeead22b62460c464d0d584e79
|
4
|
+
data.tar.gz: c8451ad56f5cae4c5b4b4ff26d02bf9445f3fc925c02a2cf843a551f3006620a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a85e773496690078a0ad56971ea0a17b15c0656abc2f9d28eefbfbeec86b867f378b4bc3175eb84462d75b4035a5249e356fc969f80302ed1b691f00be569e20
|
7
|
+
data.tar.gz: 0f1dd3bccb61205b3decbce1a2321ba5b2326978d9584e200cd3077b5f9eebe077015a8460f2261fc4583ec9850f3178f79171dc1f90a0ca622e47961dc2edb4
|
@@ -12,7 +12,10 @@ module DataOperations
|
|
12
12
|
DEFAULT_TIME_STARTED_MODE = :first_message
|
13
13
|
DEFAULT_FIELD_NO_DATA_VALUE = 'no_data'.freeze
|
14
14
|
DEFAULT_AGGREGATIONS = %w[sum min max mean median variance standard_deviation].freeze
|
15
|
-
VALID_AGGREGATIONS = %w[sum min max mean median variance standard_deviation
|
15
|
+
VALID_AGGREGATIONS = %w[sum min max mean median variance standard_deviation histogram].freeze
|
16
|
+
DEFAULT_HISTOGRAM_CUMULATIVE = false
|
17
|
+
DEFAULT_HISTOGRAM_BUCKET_INFINITE_ENABLED = true
|
18
|
+
DEFAULT_HISTOGRAM_BUCKET_COMPARATION = :less_or_equal
|
16
19
|
DEFAULT_HASH_TIME_FORMAT = '%Y-%m-%dT%H'.freeze
|
17
20
|
DEFAULT_INERVAL_SECONDS = 3600
|
18
21
|
|
@@ -31,8 +34,11 @@ module DataOperations
|
|
31
34
|
aggregation_names:,
|
32
35
|
group_field_names:,
|
33
36
|
aggregate_field_names:,
|
34
|
-
|
35
|
-
|
37
|
+
histogram_cumulative: DEFAULT_HISTOGRAM_CUMULATIVE,
|
38
|
+
histogram_bucket_infinite_enabled: DEFAULT_HISTOGRAM_BUCKET_INFINITE_ENABLED,
|
39
|
+
histogram_bucket_comparation: DEFAULT_HISTOGRAM_BUCKET_COMPARATION,
|
40
|
+
histogram_buckets:[],
|
41
|
+
histogram_fields:[]
|
36
42
|
)
|
37
43
|
@aggregator = aggregator
|
38
44
|
@time_format = time_format
|
@@ -45,8 +51,11 @@ module DataOperations
|
|
45
51
|
@processing_mode = processing_mode
|
46
52
|
@time_started_mode = time_started_mode
|
47
53
|
@aggregator_name = aggregator_name
|
48
|
-
@
|
49
|
-
@
|
54
|
+
@histogram_cumulative = histogram_cumulative
|
55
|
+
@histogram_bucket_infinite_enabled = histogram_bucket_infinite_enabled
|
56
|
+
@histogram_bucket_comparation = histogram_bucket_comparation
|
57
|
+
@histogram_buckets = histogram_buckets
|
58
|
+
@histogram_fields = histogram_fields
|
50
59
|
|
51
60
|
|
52
61
|
if aggregation_names.nil? || !aggregation_names.is_a?(Array)
|
@@ -224,15 +233,20 @@ module DataOperations
|
|
224
233
|
if aggregate_field_value.is_a?(Array)
|
225
234
|
@aggregation_names.each do |operation|
|
226
235
|
|
227
|
-
#If
|
228
|
-
if operation == '
|
236
|
+
#If histogram, calculate bucket for metric
|
237
|
+
if operation == 'histogram'
|
238
|
+
if @histogram_bucket_comparation == :less_or_equal
|
239
|
+
bucket_comparation = "le"
|
240
|
+
else
|
241
|
+
bucket_comparation = "ge"
|
242
|
+
end
|
229
243
|
#If set buckets and set metrics to calculate (bucket values depends of ranges based in metric activity)
|
230
|
-
if !@
|
231
|
-
data_bucket =
|
244
|
+
if !@histogram_buckets.nil? && !@histogram_fields.nil? && @histogram_fields.include?(aggregate_field_key)
|
245
|
+
data_bucket = calculate_histogram_buckets(aggregate_field_value, @histogram_buckets)
|
232
246
|
|
233
247
|
data_bucket.each {|bucket,bucket_count|
|
234
248
|
#@log.info("#{aggregate_field_key}_#{bucket} = #{bucket_count}")
|
235
|
-
aggregator_data["#{aggregate_field_key}
|
249
|
+
aggregator_data["#{aggregate_field_key}_bucket_#{bucket_comparation}_#{bucket}"] = bucket_count
|
236
250
|
}
|
237
251
|
|
238
252
|
# Add aggregated data to interval
|
@@ -242,8 +256,8 @@ module DataOperations
|
|
242
256
|
interval_aggregator_item_value = group_item_value['intervals'][interval_secs][interval_aggregator_item_key]
|
243
257
|
data_bucket.each {|bucket,bucket_count|
|
244
258
|
#@log.info("#{aggregate_field_key}_#{bucket} = #{bucket_count}")
|
245
|
-
interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["
|
246
|
-
interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["
|
259
|
+
interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["bucket_#{bucket_comparation}_#{bucket}"] = [] if interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["bucket#{bucket}"].nil?
|
260
|
+
interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]["bucket_#{bucket_comparation}_#{bucket}"] << bucket_count
|
247
261
|
}
|
248
262
|
|
249
263
|
end
|
@@ -264,10 +278,6 @@ module DataOperations
|
|
264
278
|
|
265
279
|
end
|
266
280
|
|
267
|
-
if !@buckets.nil? && ! @bucket_metrics.nil?
|
268
|
-
#data = calculate_buckets(data, @buckets)
|
269
|
-
end
|
270
|
-
|
271
281
|
end
|
272
282
|
end
|
273
283
|
|
@@ -284,8 +294,6 @@ module DataOperations
|
|
284
294
|
interval_aggregator_item_value['aggregate_fields'][aggregate_field_key][operation] = []
|
285
295
|
end
|
286
296
|
|
287
|
-
##Add buckets metadata (empty hash)
|
288
|
-
##interval_aggregator_item_value['aggregate_fields'][aggregate_field_key]['buckets']={}
|
289
297
|
end
|
290
298
|
end
|
291
299
|
|
@@ -339,10 +347,10 @@ module DataOperations
|
|
339
347
|
case operation
|
340
348
|
when 'max', 'min', 'mean', 'median'
|
341
349
|
data = vector.method(operation).call
|
342
|
-
when '
|
350
|
+
when 'histogram'
|
343
351
|
#Bucket operation generate bucket[\d]+
|
344
352
|
data = nil
|
345
|
-
when /^bucket
|
353
|
+
when /^bucket.+/
|
346
354
|
#For buckets sum accumulations for internvals
|
347
355
|
data = vector.method('sum').call
|
348
356
|
else
|
@@ -361,20 +369,48 @@ module DataOperations
|
|
361
369
|
end
|
362
370
|
|
363
371
|
#Return Array with count by each bucket
|
364
|
-
def
|
365
|
-
|
372
|
+
def calculate_histogram_buckets(data, buckets_config)
|
373
|
+
|
374
|
+
if @histogram_bucket_comparation == :less_or_equal
|
375
|
+
buckets_config.sort!.uniq!
|
376
|
+
else
|
377
|
+
buckets_config.reverse!.uniq!
|
378
|
+
end
|
379
|
+
|
366
380
|
buckets = {}
|
367
381
|
|
368
382
|
buckets_config.each {|bucket| buckets[bucket] = 0}
|
369
383
|
|
370
|
-
|
371
|
-
|
372
|
-
|
384
|
+
#By default add Inf bucket
|
385
|
+
buckets["Inf"] = 0 if @histogram_bucket_infinite_enabled
|
386
|
+
|
387
|
+
data.each do|item|
|
388
|
+
item_in_bucket = false
|
389
|
+
buckets_config.each do|bucket|
|
390
|
+
if @histogram_bucket_comparation == :less_or_equal
|
391
|
+
if item <= bucket
|
392
|
+
item_in_bucket = true
|
393
|
+
end
|
394
|
+
elsif item >= bucket
|
395
|
+
item_in_bucket = true
|
396
|
+
end
|
397
|
+
|
398
|
+
if item_in_bucket
|
373
399
|
buckets[bucket] += 1
|
374
|
-
|
400
|
+
#Not accumule if histogram cumulative is false
|
401
|
+
break if !@histogram_cumulative
|
375
402
|
end
|
376
|
-
|
377
|
-
|
403
|
+
end
|
404
|
+
|
405
|
+
#Only count Inf bucket if enabled
|
406
|
+
if @histogram_bucket_infinite_enabled
|
407
|
+
if !item_in_bucket
|
408
|
+
buckets["Inf"] += 1
|
409
|
+
elsif @histogram_cumulative
|
410
|
+
buckets["Inf"] += 1
|
411
|
+
end
|
412
|
+
end
|
413
|
+
end
|
378
414
|
return buckets
|
379
415
|
end
|
380
416
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dataoperations-aggregate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Guillen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-04-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: descriptive_statistics
|