logstash-output-scalyr 0.1.20.beta → 0.1.21.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23ede2e07ed42483146d4a267c8517fed2f8639fb27998e3f37e8732bd50a45d
-  data.tar.gz: 4d3594dd94a908d5852d58fc6e5214982ae731ef8b3de3eb3707455bfa77bd3d
+  metadata.gz: 7dd0297a3ddf2f2acfdad08c410822b8a4ee808800e3cfeffc5df4ec8a991348
+  data.tar.gz: 855b7e3a655656442cf3de76e15ad3d0f90ca460421d4a4f86df46056a4b05a9
 SHA512:
-  metadata.gz: b11f8aaeea8530b316de904d2fd4b654b8b2a33c2c13ca10614ca3cb346a154e494d9ec7c831a1cf0a83449101ed2bb59c97cc5506870bd252c31cbf9dbc9052
-  data.tar.gz: dc6a23b5eae108b7ac8818cfd018c9a9670f69c31fb582b7c8568ebb33720fef19564b54420b2cfc4e7fc04e4a742836bdd2eefc013b15e240379726858fe15f
+  metadata.gz: 8c3024576c03a088008c5b1acc54d02be67be701921392d91f5991ac4482f85f62ea3b07255e38a972735182feccb1520d558f5cb065538dabe35cc86e1ad719
+  data.tar.gz: c807baca5797bbf47cf69c8816a40300a515162617d4a714e719673572f1fe29fe3fa9ab73fd51a711af3953e212bd3dfe998e73c2c6de574a64ba6121446989

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Beta
 
+## 0.1.21.beta
+- Fix issue with iterative flattening function when dealing with empty collections.
+
 ## 0.1.20.beta
 - Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
 - Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.20.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.21.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -321,6 +321,12 @@ If you want to run just the unit tests, you can run the command displayed below.
 bundle exec rspec spec/logstash/outputs/scalyr_spec.rb spec/scalyr/common/util_spec.rb
 ```
 
+Or to run a single test function defined on line XXX
+
+```bash
+bundle exec rspec spec/scalyr/common/util_spec.rb:XXX
+```
+
 ## Instrumentation and metrics
 
 By default, plugin logs a special line with metrics to Scalyr every 5 minutes. This line contains

data/lib/scalyr/common/util.rb

@@ -42,7 +42,7 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
     key_list << key_stack.pop
 
     # Case when object is a hash
-    if obj.respond_to?(:has_key?)
+    if obj.respond_to?(:has_key?) and obj.keys.count > 0
       key_list_width << obj.keys.count
       obj.each do |key, value|
         key_stack << key
@@ -50,12 +50,12 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
       end
 
     # Case when object is an array we intend to flatten
-    elsif flatten_arrays and obj.respond_to?(:each)
-      key_list_width << obj.count
-      obj.each_with_index do |value, index|
-        key_stack << index
-        stack << value
-      end
+    elsif flatten_arrays and obj.respond_to?(:each) and obj.count > 0
+        key_list_width << obj.count
+        obj.each_with_index do |value, index|
+          key_stack << index
+          stack << value
+        end
 
     else
       result_key = ""

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.20.beta"
+PLUGIN_VERSION = "v0.1.21.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.20.beta'
+  s.version         = '0.1.21.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/scalyr/common/util_spec.rb

@@ -1,7 +1,113 @@
 # encoding: utf-8
 require "scalyr/common/util"
 
+LARGE_OBJECT_IN = {
+  "level": "info",
+  "ts": "2020-08-11T02:26:17.078Z",
+  "caller": "api/foo:480",
+  "msg": "assign active task foobar",
+  "accountId": 12345,
+  "cycleId": 6789,
+  "uuid": "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
+  "task": {
+    "Id": 1211111181111111400,
+    "TaskTypes": [
+      4,
+      11,
+      10,
+      12,
+      17,
+      14
+    ],
+    "Ips": [
+      "127.0.0.1",
+      "127.0.0.2",
+      "127.0.0.3",
+      "127.0.0.4",
+      "127.0.0.5",
+    ],
+    "FooProps": {
+      "10": {
+        "TcpPorts": [
+          22,
+          23,
+          25,
+          80,
+          55,
+          8000,
+          8080,
+        ],
+        "UdpPorts": []
+      }
+    },
+    "Subnet": "127.0.0.0/24"
+  },
+  "relevance": 0,
+  "scannerIp": "10.0.0.2",
+  "gatewayIp": "10.0.0.1",
+  "gatewayMac": "fa:fa:fa:fa",
+  "wired": true,
+  "elapsed": 74.86664
+}
 
+LARGE_OBJECT_OUT = {
+  "accountId" => 12345,
+  "caller" => "api/foo:480",
+  "cycleId" => 6789,
+  "elapsed" => 74.86664,
+  "gatewayIp" => "10.0.0.1",
+  "gatewayMac" => "fa:fa:fa:fa",
+  "level" => "info",
+  "msg" => "assign active task foobar",
+  "relevance" => 0,
+  "scannerIp" => "10.0.0.2",
+  "task_FooProps_10_TcpPorts_0" => 22,
+  "task_FooProps_10_TcpPorts_1" => 23,
+  "task_FooProps_10_TcpPorts_2" => 25,
+  "task_FooProps_10_TcpPorts_3" => 80,
+  "task_FooProps_10_TcpPorts_4" => 55,
+  "task_FooProps_10_TcpPorts_5" => 8000,
+  "task_FooProps_10_TcpPorts_6" => 8080,
+  "task_FooProps_10_UdpPorts" => [],
+  "task_Id" => 1211111181111111400,
+  "task_Ips_0" => "127.0.0.1",
+  "task_Ips_1" => "127.0.0.2",
+  "task_Ips_2" => "127.0.0.3",
+  "task_Ips_3" => "127.0.0.4",
+  "task_Ips_4" => "127.0.0.5",
+  "task_Subnet" => "127.0.0.0/24",
+  "task_TaskTypes_0" => 4,
+  "task_TaskTypes_1" => 11,
+  "task_TaskTypes_2" => 10,
+  "task_TaskTypes_3" => 12,
+  "task_TaskTypes_4" => 17,
+  "task_TaskTypes_5" => 14,
+  "ts" => "2020-08-11T02:26:17.078Z",
+  "uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
+  "wired" => true,
+}
+
+LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS = {
+  "accountId" => 12345,
+  "caller" => "api/foo:480",
+  "cycleId" => 6789,
+  "elapsed" => 74.86664,
+  "gatewayIp" => "10.0.0.1",
+  "gatewayMac" => "fa:fa:fa:fa",
+  "level" => "info",
+  "msg" => "assign active task foobar",
+  "relevance" => 0,
+  "scannerIp" => "10.0.0.2",
+  "task_FooProps_10_TcpPorts" => [22, 23, 25, 80, 55, 8000, 8080],
+  "task_FooProps_10_UdpPorts" => [],
+  "task_Id" => 1211111181111111400,
+  "task_Ips" => ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"],
+  "task_Subnet" => "127.0.0.0/24",
+  "task_TaskTypes" => [4, 11, 10, 12, 17, 14],
+  "ts" => "2020-08-11T02:26:17.078Z",
+  "uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
+  "wired" => true,
+}
 describe Scalyr::Common::Util do
   it "does not flatten an already-flat dict" do
     din = {
@@ -144,6 +250,20 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
   end
 
+  it "flattens a hash that contains an array with hashes, no array flattening" do
+    din = {
+        'a' => 1,
+        "b" => {"a": "a"},
+        'c' => { "f" => [100, 200, {"g" => 1}] }
+    }
+    dout = {
+        'a' => 1,
+        "b_a" => "a",
+        'c_f' => [100, 200, {"g" => 1}]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
   it "flattens a hash that contains an array, no array flattening" do
     din = {
         'a' => 1,
@@ -196,6 +316,120 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
   end
 
+  it "flattens large hash correctly" do
+    expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=true)).to eq(LARGE_OBJECT_OUT)
+  end
+
+  it "flattens large hash correctly not flatten arrays" do
+    expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=false)).to eq(LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS)
+  end
+
+  it "flattens hash containing empty list correctly" do
+    obj_in = {
+      "abc" => 123,
+      "array" => [],
+      "hash" => {
+        "value" => "abc123",
+        "another_array" => []
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "array" => [],
+      "hash_value" => "abc123",
+      "hash_another_array" => []
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing empty list correctly not flatten arrays" do
+    obj_in = {
+      "abc" => 123,
+      "array" => [],
+      "hash" => {
+        "value" => "abc123",
+        "another_array" => []
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "array" => [],
+      "hash_value" => "abc123",
+      "hash_another_array" => []
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=false)).to eq(obj_out)
+  end
+
+  it "flattens hash containing empty hash correctly" do
+    obj_in = {
+      "abc" => 123,
+      "empty_hash" => {},
+      "hash" => {
+        "value" => "abc123",
+        "another_hash" => {}
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "empty_hash" => {},
+      "hash_value" => "abc123",
+      "hash_another_hash" => {}
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing nested bignum correctly" do
+    obj_in = {
+      "abc" => 123,
+      "hash" => {
+        "value" => 2000023030042002050202030320240
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "hash_value" => 2000023030042002050202030320240
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing nested boolean correctly" do
+    obj_in = {
+      "abc" => 123,
+      "hash" => {
+        "value" => true,
+        "otherValue" => false
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "hash_value" => true,
+      "hash_otherValue" => false
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing nested float correctly" do
+    obj_in = {
+      "abc" => 123,
+      "hash" => {
+        "value" => 321.12345,
+        "otherValue" => 0.0000003
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "hash_value" => 321.12345,
+      "hash_otherValue" => 0.0000003
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
   it "accepts custom delimiters" do
     din = {
         'a' => 1,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.20.beta
+  version: 0.1.21.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-09 00:00:00.000000000 Z
+date: 2021-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement