logstash-output-scalyr 0.1.20.beta → 0.1.21.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/README.md +7 -1
- data/lib/scalyr/common/util.rb +7 -7
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/scalyr/common/util_spec.rb +234 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7dd0297a3ddf2f2acfdad08c410822b8a4ee808800e3cfeffc5df4ec8a991348
|
4
|
+
data.tar.gz: 855b7e3a655656442cf3de76e15ad3d0f90ca460421d4a4f86df46056a4b05a9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8c3024576c03a088008c5b1acc54d02be67be701921392d91f5991ac4482f85f62ea3b07255e38a972735182feccb1520d558f5cb065538dabe35cc86e1ad719
|
7
|
+
data.tar.gz: c807baca5797bbf47cf69c8816a40300a515162617d4a714e719673572f1fe29fe3fa9ab73fd51a711af3953e212bd3dfe998e73c2c6de574a64ba6121446989
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.1.21.beta
|
4
|
+
- Fix issue with iterative flattening function when dealing with empty collections.
|
5
|
+
|
3
6
|
## 0.1.20.beta
|
4
7
|
- Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
|
5
8
|
- Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
|
data/README.md
CHANGED
@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
|
|
10
10
|
# Quick start
|
11
11
|
|
12
12
|
1. Build the gem, run `gem build logstash-output-scalyr.gemspec`
|
13
|
-
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.
|
13
|
+
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.21.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
|
14
14
|
3. Configure the output plugin (e.g. add it to a pipeline .conf)
|
15
15
|
4. Restart Logstash
|
16
16
|
|
@@ -321,6 +321,12 @@ If you want to run just the unit tests, you can run the command displayed below.
|
|
321
321
|
bundle exec rspec spec/logstash/outputs/scalyr_spec.rb spec/scalyr/common/util_spec.rb
|
322
322
|
```
|
323
323
|
|
324
|
+
Or to run a single test function defined on line XXX
|
325
|
+
|
326
|
+
```bash
|
327
|
+
bundle exec rspec spec/scalyr/common/util_spec.rb:XXX
|
328
|
+
```
|
329
|
+
|
324
330
|
## Instrumentation and metrics
|
325
331
|
|
326
332
|
By default, plugin logs a special line with metrics to Scalyr every 5 minutes. This line contains
|
data/lib/scalyr/common/util.rb
CHANGED
@@ -42,7 +42,7 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
|
|
42
42
|
key_list << key_stack.pop
|
43
43
|
|
44
44
|
# Case when object is a hash
|
45
|
-
if obj.respond_to?(:has_key?)
|
45
|
+
if obj.respond_to?(:has_key?) and obj.keys.count > 0
|
46
46
|
key_list_width << obj.keys.count
|
47
47
|
obj.each do |key, value|
|
48
48
|
key_stack << key
|
@@ -50,12 +50,12 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
|
|
50
50
|
end
|
51
51
|
|
52
52
|
# Case when object is an array we intend to flatten
|
53
|
-
elsif flatten_arrays and obj.respond_to?(:each)
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
53
|
+
elsif flatten_arrays and obj.respond_to?(:each) and obj.count > 0
|
54
|
+
key_list_width << obj.count
|
55
|
+
obj.each_with_index do |value, index|
|
56
|
+
key_stack << index
|
57
|
+
stack << value
|
58
|
+
end
|
59
59
|
|
60
60
|
else
|
61
61
|
result_key = ""
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
PLUGIN_VERSION = "v0.1.
|
2
|
+
PLUGIN_VERSION = "v0.1.21.beta"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.21.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -1,7 +1,113 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "scalyr/common/util"
|
3
3
|
|
4
|
+
LARGE_OBJECT_IN = {
|
5
|
+
"level": "info",
|
6
|
+
"ts": "2020-08-11T02:26:17.078Z",
|
7
|
+
"caller": "api/foo:480",
|
8
|
+
"msg": "assign active task foobar",
|
9
|
+
"accountId": 12345,
|
10
|
+
"cycleId": 6789,
|
11
|
+
"uuid": "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
|
12
|
+
"task": {
|
13
|
+
"Id": 1211111181111111400,
|
14
|
+
"TaskTypes": [
|
15
|
+
4,
|
16
|
+
11,
|
17
|
+
10,
|
18
|
+
12,
|
19
|
+
17,
|
20
|
+
14
|
21
|
+
],
|
22
|
+
"Ips": [
|
23
|
+
"127.0.0.1",
|
24
|
+
"127.0.0.2",
|
25
|
+
"127.0.0.3",
|
26
|
+
"127.0.0.4",
|
27
|
+
"127.0.0.5",
|
28
|
+
],
|
29
|
+
"FooProps": {
|
30
|
+
"10": {
|
31
|
+
"TcpPorts": [
|
32
|
+
22,
|
33
|
+
23,
|
34
|
+
25,
|
35
|
+
80,
|
36
|
+
55,
|
37
|
+
8000,
|
38
|
+
8080,
|
39
|
+
],
|
40
|
+
"UdpPorts": []
|
41
|
+
}
|
42
|
+
},
|
43
|
+
"Subnet": "127.0.0.0/24"
|
44
|
+
},
|
45
|
+
"relevance": 0,
|
46
|
+
"scannerIp": "10.0.0.2",
|
47
|
+
"gatewayIp": "10.0.0.1",
|
48
|
+
"gatewayMac": "fa:fa:fa:fa",
|
49
|
+
"wired": true,
|
50
|
+
"elapsed": 74.86664
|
51
|
+
}
|
4
52
|
|
53
|
+
LARGE_OBJECT_OUT = {
|
54
|
+
"accountId" => 12345,
|
55
|
+
"caller" => "api/foo:480",
|
56
|
+
"cycleId" => 6789,
|
57
|
+
"elapsed" => 74.86664,
|
58
|
+
"gatewayIp" => "10.0.0.1",
|
59
|
+
"gatewayMac" => "fa:fa:fa:fa",
|
60
|
+
"level" => "info",
|
61
|
+
"msg" => "assign active task foobar",
|
62
|
+
"relevance" => 0,
|
63
|
+
"scannerIp" => "10.0.0.2",
|
64
|
+
"task_FooProps_10_TcpPorts_0" => 22,
|
65
|
+
"task_FooProps_10_TcpPorts_1" => 23,
|
66
|
+
"task_FooProps_10_TcpPorts_2" => 25,
|
67
|
+
"task_FooProps_10_TcpPorts_3" => 80,
|
68
|
+
"task_FooProps_10_TcpPorts_4" => 55,
|
69
|
+
"task_FooProps_10_TcpPorts_5" => 8000,
|
70
|
+
"task_FooProps_10_TcpPorts_6" => 8080,
|
71
|
+
"task_FooProps_10_UdpPorts" => [],
|
72
|
+
"task_Id" => 1211111181111111400,
|
73
|
+
"task_Ips_0" => "127.0.0.1",
|
74
|
+
"task_Ips_1" => "127.0.0.2",
|
75
|
+
"task_Ips_2" => "127.0.0.3",
|
76
|
+
"task_Ips_3" => "127.0.0.4",
|
77
|
+
"task_Ips_4" => "127.0.0.5",
|
78
|
+
"task_Subnet" => "127.0.0.0/24",
|
79
|
+
"task_TaskTypes_0" => 4,
|
80
|
+
"task_TaskTypes_1" => 11,
|
81
|
+
"task_TaskTypes_2" => 10,
|
82
|
+
"task_TaskTypes_3" => 12,
|
83
|
+
"task_TaskTypes_4" => 17,
|
84
|
+
"task_TaskTypes_5" => 14,
|
85
|
+
"ts" => "2020-08-11T02:26:17.078Z",
|
86
|
+
"uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
|
87
|
+
"wired" => true,
|
88
|
+
}
|
89
|
+
|
90
|
+
LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS = {
|
91
|
+
"accountId" => 12345,
|
92
|
+
"caller" => "api/foo:480",
|
93
|
+
"cycleId" => 6789,
|
94
|
+
"elapsed" => 74.86664,
|
95
|
+
"gatewayIp" => "10.0.0.1",
|
96
|
+
"gatewayMac" => "fa:fa:fa:fa",
|
97
|
+
"level" => "info",
|
98
|
+
"msg" => "assign active task foobar",
|
99
|
+
"relevance" => 0,
|
100
|
+
"scannerIp" => "10.0.0.2",
|
101
|
+
"task_FooProps_10_TcpPorts" => [22, 23, 25, 80, 55, 8000, 8080],
|
102
|
+
"task_FooProps_10_UdpPorts" => [],
|
103
|
+
"task_Id" => 1211111181111111400,
|
104
|
+
"task_Ips" => ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"],
|
105
|
+
"task_Subnet" => "127.0.0.0/24",
|
106
|
+
"task_TaskTypes" => [4, 11, 10, 12, 17, 14],
|
107
|
+
"ts" => "2020-08-11T02:26:17.078Z",
|
108
|
+
"uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
|
109
|
+
"wired" => true,
|
110
|
+
}
|
5
111
|
describe Scalyr::Common::Util do
|
6
112
|
it "does not flatten an already-flat dict" do
|
7
113
|
din = {
|
@@ -144,6 +250,20 @@ describe Scalyr::Common::Util do
|
|
144
250
|
expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
|
145
251
|
end
|
146
252
|
|
253
|
+
it "flattens a hash that contains an array with hashes, no array flattening" do
|
254
|
+
din = {
|
255
|
+
'a' => 1,
|
256
|
+
"b" => {"a": "a"},
|
257
|
+
'c' => { "f" => [100, 200, {"g" => 1}] }
|
258
|
+
}
|
259
|
+
dout = {
|
260
|
+
'a' => 1,
|
261
|
+
"b_a" => "a",
|
262
|
+
'c_f' => [100, 200, {"g" => 1}]
|
263
|
+
}
|
264
|
+
expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
|
265
|
+
end
|
266
|
+
|
147
267
|
it "flattens a hash that contains an array, no array flattening" do
|
148
268
|
din = {
|
149
269
|
'a' => 1,
|
@@ -196,6 +316,120 @@ describe Scalyr::Common::Util do
|
|
196
316
|
expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
|
197
317
|
end
|
198
318
|
|
319
|
+
it "flattens large hash correctly" do
|
320
|
+
expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=true)).to eq(LARGE_OBJECT_OUT)
|
321
|
+
end
|
322
|
+
|
323
|
+
it "flattens large hash correctly not flatten arrays" do
|
324
|
+
expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=false)).to eq(LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS)
|
325
|
+
end
|
326
|
+
|
327
|
+
it "flattens hash containing empty list correctly" do
|
328
|
+
obj_in = {
|
329
|
+
"abc" => 123,
|
330
|
+
"array" => [],
|
331
|
+
"hash" => {
|
332
|
+
"value" => "abc123",
|
333
|
+
"another_array" => []
|
334
|
+
}
|
335
|
+
}
|
336
|
+
|
337
|
+
obj_out = {
|
338
|
+
"abc" => 123,
|
339
|
+
"array" => [],
|
340
|
+
"hash_value" => "abc123",
|
341
|
+
"hash_another_array" => []
|
342
|
+
}
|
343
|
+
expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
|
344
|
+
end
|
345
|
+
|
346
|
+
it "flattens hash containing empty list correctly not flatten arrays" do
|
347
|
+
obj_in = {
|
348
|
+
"abc" => 123,
|
349
|
+
"array" => [],
|
350
|
+
"hash" => {
|
351
|
+
"value" => "abc123",
|
352
|
+
"another_array" => []
|
353
|
+
}
|
354
|
+
}
|
355
|
+
|
356
|
+
obj_out = {
|
357
|
+
"abc" => 123,
|
358
|
+
"array" => [],
|
359
|
+
"hash_value" => "abc123",
|
360
|
+
"hash_another_array" => []
|
361
|
+
}
|
362
|
+
expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=false)).to eq(obj_out)
|
363
|
+
end
|
364
|
+
|
365
|
+
it "flattens hash containing empty hash correctly" do
|
366
|
+
obj_in = {
|
367
|
+
"abc" => 123,
|
368
|
+
"empty_hash" => {},
|
369
|
+
"hash" => {
|
370
|
+
"value" => "abc123",
|
371
|
+
"another_hash" => {}
|
372
|
+
}
|
373
|
+
}
|
374
|
+
|
375
|
+
obj_out = {
|
376
|
+
"abc" => 123,
|
377
|
+
"empty_hash" => {},
|
378
|
+
"hash_value" => "abc123",
|
379
|
+
"hash_another_hash" => {}
|
380
|
+
}
|
381
|
+
expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
|
382
|
+
end
|
383
|
+
|
384
|
+
it "flattens hash containing nested bignum correctly" do
|
385
|
+
obj_in = {
|
386
|
+
"abc" => 123,
|
387
|
+
"hash" => {
|
388
|
+
"value" => 2000023030042002050202030320240
|
389
|
+
}
|
390
|
+
}
|
391
|
+
|
392
|
+
obj_out = {
|
393
|
+
"abc" => 123,
|
394
|
+
"hash_value" => 2000023030042002050202030320240
|
395
|
+
}
|
396
|
+
expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
|
397
|
+
end
|
398
|
+
|
399
|
+
it "flattens hash containing nested boolean correctly" do
|
400
|
+
obj_in = {
|
401
|
+
"abc" => 123,
|
402
|
+
"hash" => {
|
403
|
+
"value" => true,
|
404
|
+
"otherValue" => false
|
405
|
+
}
|
406
|
+
}
|
407
|
+
|
408
|
+
obj_out = {
|
409
|
+
"abc" => 123,
|
410
|
+
"hash_value" => true,
|
411
|
+
"hash_otherValue" => false
|
412
|
+
}
|
413
|
+
expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
|
414
|
+
end
|
415
|
+
|
416
|
+
it "flattens hash containing nested float correctly" do
|
417
|
+
obj_in = {
|
418
|
+
"abc" => 123,
|
419
|
+
"hash" => {
|
420
|
+
"value" => 321.12345,
|
421
|
+
"otherValue" => 0.0000003
|
422
|
+
}
|
423
|
+
}
|
424
|
+
|
425
|
+
obj_out = {
|
426
|
+
"abc" => 123,
|
427
|
+
"hash_value" => 321.12345,
|
428
|
+
"hash_otherValue" => 0.0000003
|
429
|
+
}
|
430
|
+
expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
|
431
|
+
end
|
432
|
+
|
199
433
|
it "accepts custom delimiters" do
|
200
434
|
din = {
|
201
435
|
'a' => 1,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.21.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-08-
|
11
|
+
date: 2021-08-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|