logstash-output-scalyr 0.1.20.beta → 0.1.21.beta

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 23ede2e07ed42483146d4a267c8517fed2f8639fb27998e3f37e8732bd50a45d
4
- data.tar.gz: 4d3594dd94a908d5852d58fc6e5214982ae731ef8b3de3eb3707455bfa77bd3d
3
+ metadata.gz: 7dd0297a3ddf2f2acfdad08c410822b8a4ee808800e3cfeffc5df4ec8a991348
4
+ data.tar.gz: 855b7e3a655656442cf3de76e15ad3d0f90ca460421d4a4f86df46056a4b05a9
5
5
  SHA512:
6
- metadata.gz: b11f8aaeea8530b316de904d2fd4b654b8b2a33c2c13ca10614ca3cb346a154e494d9ec7c831a1cf0a83449101ed2bb59c97cc5506870bd252c31cbf9dbc9052
7
- data.tar.gz: dc6a23b5eae108b7ac8818cfd018c9a9670f69c31fb582b7c8568ebb33720fef19564b54420b2cfc4e7fc04e4a742836bdd2eefc013b15e240379726858fe15f
6
+ metadata.gz: 8c3024576c03a088008c5b1acc54d02be67be701921392d91f5991ac4482f85f62ea3b07255e38a972735182feccb1520d558f5cb065538dabe35cc86e1ad719
7
+ data.tar.gz: c807baca5797bbf47cf69c8816a40300a515162617d4a714e719673572f1fe29fe3fa9ab73fd51a711af3953e212bd3dfe998e73c2c6de574a64ba6121446989
data/CHANGELOG.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Beta
2
2
 
3
+ ## 0.1.21.beta
4
+ - Fix issue with iterative flattening function when dealing with empty collections.
5
+
3
6
  ## 0.1.20.beta
4
7
  - Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
5
8
  - Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
data/README.md CHANGED
@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
10
10
  # Quick start
11
11
 
12
12
  1. Build the gem, run `gem build logstash-output-scalyr.gemspec`
13
- 2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.20.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
13
+ 2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.21.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
14
14
  3. Configure the output plugin (e.g. add it to a pipeline .conf)
15
15
  4. Restart Logstash
16
16
 
@@ -321,6 +321,12 @@ If you want to run just the unit tests, you can run the command displayed below.
321
321
  bundle exec rspec spec/logstash/outputs/scalyr_spec.rb spec/scalyr/common/util_spec.rb
322
322
  ```
323
323
 
324
+ Or to run a single test function defined on line XXX
325
+
326
+ ```bash
327
+ bundle exec rspec spec/scalyr/common/util_spec.rb:XXX
328
+ ```
329
+
324
330
  ## Instrumentation and metrics
325
331
 
326
332
  By default, plugin logs a special line with metrics to Scalyr every 5 minutes. This line contains
@@ -42,7 +42,7 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
42
42
  key_list << key_stack.pop
43
43
 
44
44
  # Case when object is a hash
45
- if obj.respond_to?(:has_key?)
45
+ if obj.respond_to?(:has_key?) and obj.keys.count > 0
46
46
  key_list_width << obj.keys.count
47
47
  obj.each do |key, value|
48
48
  key_stack << key
@@ -50,12 +50,12 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
50
50
  end
51
51
 
52
52
  # Case when object is an array we intend to flatten
53
- elsif flatten_arrays and obj.respond_to?(:each)
54
- key_list_width << obj.count
55
- obj.each_with_index do |value, index|
56
- key_stack << index
57
- stack << value
58
- end
53
+ elsif flatten_arrays and obj.respond_to?(:each) and obj.count > 0
54
+ key_list_width << obj.count
55
+ obj.each_with_index do |value, index|
56
+ key_stack << index
57
+ stack << value
58
+ end
59
59
 
60
60
  else
61
61
  result_key = ""
@@ -1,2 +1,2 @@
1
1
  # encoding: utf-8
2
- PLUGIN_VERSION = "v0.1.20.beta"
2
+ PLUGIN_VERSION = "v0.1.21.beta"
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-scalyr'
3
- s.version = '0.1.20.beta'
3
+ s.version = '0.1.21.beta'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Scalyr output plugin for Logstash"
6
6
  s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -1,7 +1,113 @@
1
1
  # encoding: utf-8
2
2
  require "scalyr/common/util"
3
3
 
4
+ LARGE_OBJECT_IN = {
5
+ "level": "info",
6
+ "ts": "2020-08-11T02:26:17.078Z",
7
+ "caller": "api/foo:480",
8
+ "msg": "assign active task foobar",
9
+ "accountId": 12345,
10
+ "cycleId": 6789,
11
+ "uuid": "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
12
+ "task": {
13
+ "Id": 1211111181111111400,
14
+ "TaskTypes": [
15
+ 4,
16
+ 11,
17
+ 10,
18
+ 12,
19
+ 17,
20
+ 14
21
+ ],
22
+ "Ips": [
23
+ "127.0.0.1",
24
+ "127.0.0.2",
25
+ "127.0.0.3",
26
+ "127.0.0.4",
27
+ "127.0.0.5",
28
+ ],
29
+ "FooProps": {
30
+ "10": {
31
+ "TcpPorts": [
32
+ 22,
33
+ 23,
34
+ 25,
35
+ 80,
36
+ 55,
37
+ 8000,
38
+ 8080,
39
+ ],
40
+ "UdpPorts": []
41
+ }
42
+ },
43
+ "Subnet": "127.0.0.0/24"
44
+ },
45
+ "relevance": 0,
46
+ "scannerIp": "10.0.0.2",
47
+ "gatewayIp": "10.0.0.1",
48
+ "gatewayMac": "fa:fa:fa:fa",
49
+ "wired": true,
50
+ "elapsed": 74.86664
51
+ }
4
52
 
53
+ LARGE_OBJECT_OUT = {
54
+ "accountId" => 12345,
55
+ "caller" => "api/foo:480",
56
+ "cycleId" => 6789,
57
+ "elapsed" => 74.86664,
58
+ "gatewayIp" => "10.0.0.1",
59
+ "gatewayMac" => "fa:fa:fa:fa",
60
+ "level" => "info",
61
+ "msg" => "assign active task foobar",
62
+ "relevance" => 0,
63
+ "scannerIp" => "10.0.0.2",
64
+ "task_FooProps_10_TcpPorts_0" => 22,
65
+ "task_FooProps_10_TcpPorts_1" => 23,
66
+ "task_FooProps_10_TcpPorts_2" => 25,
67
+ "task_FooProps_10_TcpPorts_3" => 80,
68
+ "task_FooProps_10_TcpPorts_4" => 55,
69
+ "task_FooProps_10_TcpPorts_5" => 8000,
70
+ "task_FooProps_10_TcpPorts_6" => 8080,
71
+ "task_FooProps_10_UdpPorts" => [],
72
+ "task_Id" => 1211111181111111400,
73
+ "task_Ips_0" => "127.0.0.1",
74
+ "task_Ips_1" => "127.0.0.2",
75
+ "task_Ips_2" => "127.0.0.3",
76
+ "task_Ips_3" => "127.0.0.4",
77
+ "task_Ips_4" => "127.0.0.5",
78
+ "task_Subnet" => "127.0.0.0/24",
79
+ "task_TaskTypes_0" => 4,
80
+ "task_TaskTypes_1" => 11,
81
+ "task_TaskTypes_2" => 10,
82
+ "task_TaskTypes_3" => 12,
83
+ "task_TaskTypes_4" => 17,
84
+ "task_TaskTypes_5" => 14,
85
+ "ts" => "2020-08-11T02:26:17.078Z",
86
+ "uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
87
+ "wired" => true,
88
+ }
89
+
90
+ LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS = {
91
+ "accountId" => 12345,
92
+ "caller" => "api/foo:480",
93
+ "cycleId" => 6789,
94
+ "elapsed" => 74.86664,
95
+ "gatewayIp" => "10.0.0.1",
96
+ "gatewayMac" => "fa:fa:fa:fa",
97
+ "level" => "info",
98
+ "msg" => "assign active task foobar",
99
+ "relevance" => 0,
100
+ "scannerIp" => "10.0.0.2",
101
+ "task_FooProps_10_TcpPorts" => [22, 23, 25, 80, 55, 8000, 8080],
102
+ "task_FooProps_10_UdpPorts" => [],
103
+ "task_Id" => 1211111181111111400,
104
+ "task_Ips" => ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"],
105
+ "task_Subnet" => "127.0.0.0/24",
106
+ "task_TaskTypes" => [4, 11, 10, 12, 17, 14],
107
+ "ts" => "2020-08-11T02:26:17.078Z",
108
+ "uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
109
+ "wired" => true,
110
+ }
5
111
  describe Scalyr::Common::Util do
6
112
  it "does not flatten an already-flat dict" do
7
113
  din = {
@@ -144,6 +250,20 @@ describe Scalyr::Common::Util do
144
250
  expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
145
251
  end
146
252
 
253
+ it "flattens a hash that contains an array with hashes, no array flattening" do
254
+ din = {
255
+ 'a' => 1,
256
+ "b" => {"a": "a"},
257
+ 'c' => { "f" => [100, 200, {"g" => 1}] }
258
+ }
259
+ dout = {
260
+ 'a' => 1,
261
+ "b_a" => "a",
262
+ 'c_f' => [100, 200, {"g" => 1}]
263
+ }
264
+ expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
265
+ end
266
+
147
267
  it "flattens a hash that contains an array, no array flattening" do
148
268
  din = {
149
269
  'a' => 1,
@@ -196,6 +316,120 @@ describe Scalyr::Common::Util do
196
316
  expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
197
317
  end
198
318
 
319
+ it "flattens large hash correctly" do
320
+ expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=true)).to eq(LARGE_OBJECT_OUT)
321
+ end
322
+
323
+ it "flattens large hash correctly not flatten arrays" do
324
+ expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=false)).to eq(LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS)
325
+ end
326
+
327
+ it "flattens hash containing empty list correctly" do
328
+ obj_in = {
329
+ "abc" => 123,
330
+ "array" => [],
331
+ "hash" => {
332
+ "value" => "abc123",
333
+ "another_array" => []
334
+ }
335
+ }
336
+
337
+ obj_out = {
338
+ "abc" => 123,
339
+ "array" => [],
340
+ "hash_value" => "abc123",
341
+ "hash_another_array" => []
342
+ }
343
+ expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
344
+ end
345
+
346
+ it "flattens hash containing empty list correctly not flatten arrays" do
347
+ obj_in = {
348
+ "abc" => 123,
349
+ "array" => [],
350
+ "hash" => {
351
+ "value" => "abc123",
352
+ "another_array" => []
353
+ }
354
+ }
355
+
356
+ obj_out = {
357
+ "abc" => 123,
358
+ "array" => [],
359
+ "hash_value" => "abc123",
360
+ "hash_another_array" => []
361
+ }
362
+ expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=false)).to eq(obj_out)
363
+ end
364
+
365
+ it "flattens hash containing empty hash correctly" do
366
+ obj_in = {
367
+ "abc" => 123,
368
+ "empty_hash" => {},
369
+ "hash" => {
370
+ "value" => "abc123",
371
+ "another_hash" => {}
372
+ }
373
+ }
374
+
375
+ obj_out = {
376
+ "abc" => 123,
377
+ "empty_hash" => {},
378
+ "hash_value" => "abc123",
379
+ "hash_another_hash" => {}
380
+ }
381
+ expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
382
+ end
383
+
384
+ it "flattens hash containing nested bignum correctly" do
385
+ obj_in = {
386
+ "abc" => 123,
387
+ "hash" => {
388
+ "value" => 2000023030042002050202030320240
389
+ }
390
+ }
391
+
392
+ obj_out = {
393
+ "abc" => 123,
394
+ "hash_value" => 2000023030042002050202030320240
395
+ }
396
+ expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
397
+ end
398
+
399
+ it "flattens hash containing nested boolean correctly" do
400
+ obj_in = {
401
+ "abc" => 123,
402
+ "hash" => {
403
+ "value" => true,
404
+ "otherValue" => false
405
+ }
406
+ }
407
+
408
+ obj_out = {
409
+ "abc" => 123,
410
+ "hash_value" => true,
411
+ "hash_otherValue" => false
412
+ }
413
+ expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
414
+ end
415
+
416
+ it "flattens hash containing nested float correctly" do
417
+ obj_in = {
418
+ "abc" => 123,
419
+ "hash" => {
420
+ "value" => 321.12345,
421
+ "otherValue" => 0.0000003
422
+ }
423
+ }
424
+
425
+ obj_out = {
426
+ "abc" => 123,
427
+ "hash_value" => 321.12345,
428
+ "hash_otherValue" => 0.0000003
429
+ }
430
+ expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
431
+ end
432
+
199
433
  it "accepts custom delimiters" do
200
434
  din = {
201
435
  'a' => 1,
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-scalyr
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.20.beta
4
+ version: 0.1.21.beta
5
5
  platform: ruby
6
6
  authors:
7
7
  - Edward Chee
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-08-09 00:00:00.000000000 Z
11
+ date: 2021-08-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement