fluent-plugin-in-http-splunk-hec 0.1.0.rc1 → 0.1.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e9caf2d676d2aa3333a341df8b86a1bd68acc3362ce6ea456bf216477897445
-  data.tar.gz: a03cba6a749887836a6b3642a056a82fef07c46ac97d1f4060b9df33ef3f10fc
+  metadata.gz: b026a18f2c30a139c8718e3669993e0ccf885f04ecde1447399943d4b2b876ab
+  data.tar.gz: c85f866af026a2629301bd1d3a6af6f69df58b3a00c5eb0c2e472bdea9afc869
 SHA512:
-  metadata.gz: 39c1d2fe30e0b3bbbd3dc7d64d38863b9531cf0802ac11a4af6048f2bfb6264395e5e3bfb5e30b7837526e997de36ccc9cd4eb90608d1118f5470794430a646f
-  data.tar.gz: fd29ef8d685e278de175048bf2f9bd85daa062ff9dc540025c77804b38dc1171202e1d9cef7d058dcfbb39f9b2c9737a35ab1cdb977baa511fa5004e3719fb8d
+  metadata.gz: 5e6802be2857560aa6e06a58ec8d7a53132bfac6fb25eadee5a1d7009d85c3219bcf57ae6dfc8c0e85decf0e37e13baeedba22a62b6078d04e96ac8ba0afb5e3
+  data.tar.gz: c61be912ec432173f1d9a394197366de879462f8bd186ffd0d07f4e3484c2f1c6d87cebdcd32843ec093ae347025fb5fad682cec6deb194cf9a1b2de3cc16d70

data/README.md

@@ -118,6 +118,70 @@ This plugin can handle Splunk HEC HTTP requests with the following configuration
 # And other data pipeline
 ```
 
+### Advanced Usage
+
+`in_http_splunk_hec` can be combined `fluent-plugin-cmetrics` to forward ingested Splunk metric records from mimicking Splunk HTTP HEC endpoint.
+
+When you want to use the below Splunk HTTP HEC aggregator, you have to install `fluent-plugin-cmetrics` before you use that.
+
+This plugin also can aggregate and the latter data pipeline can forward Splunk metric events with the following configuration:
+
+```aconf
+<source>
+  @type http_splunk_hec
+  bind 0.0.0.0
+  port 8089
+  body_size_limit 32MB
+  keepalive_timeout 10
+  # backlog 0
+  add_http_headers false
+  # Use the actual Splunk HTTP HEC endpoint token
+  splunk_token <<YOUR_SPLUNK_HEC_TOKEN>>
+  <parse>
+    @type none
+  </parse>
+</source>
+
+<filter services.collector>
+  @type concatenated_splunk_json
+</filter>
+
+<match services.collector>
+  @type copy
+  <store>
+    @type rdkafka2
+    brokers <<BROKER_URL:BROKER_PORT>>
+    topic services.collector
+    default_topic services.collector
+    use_event_time true
+    required_acks 1
+    <format>
+      # This plugin is included in `fluent-plugin-cmetrics`.
+      @type cmetrics_splunk_metric_payload
+      host_key host
+      cmetrics_name_key $.fields.metric_name
+      cmetrics_value_key $.fields._value
+      # Specifying the below parameter and `$.fields` in `<fields>` plugin custom directive,
+      # out_rdkafka2 can send Splunk metrics style records into Kafka.
+      #
+      # To remove the `$.fields` prefix from Splunk dimensions,
+      # This parameter should be true.
+      only_use_last_field_keys true
+      <fields>
+        $.fields
+      </fields>
+    </format>
+    <buffer>
+      @type memory
+      flush_interval 10s
+    </buffer>
+  </store>
+  <store>
+    @type stdout
+  </store>
+</match>
+```
+
 ## Copyright
 
 * Copyright(c) 2021- Calyptia Inc.

data/fluent-plugin-in-http-splunk-hec.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-in-http-splunk-hec"
-  spec.version = "0.1.0.rc1"
+  spec.version = "0.1.0.rc2"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["cosmo0920.oucc@gmail.com"]
 

data/lib/fluent/plugin/filter_concatenated_splunk_json.rb

@@ -37,8 +37,12 @@ module Fluent
       end
 
       def parse_splunk_timestamp(timestamp)
-        timestamp = Float(timestamp)
-        Fluent::EventTime.from_time(Time.at(timestamp.to_r))
+        if !timestamp.nil?
+          timestamp = Float(timestamp)
+          Fluent::EventTime.from_time(Time.at(timestamp.to_r))
+        else
+          Fluent::EventTime.now
+        end
       end
 
       def filter_stream(tag, es)

data/test/plugin/test_filter_concatenated_splunk_json.rb

@@ -19,6 +19,16 @@ class ConcatenatedSplunkJSONFilterTest < Test::Unit::TestCase
     time_key time
   ]
 
+  test "parsing simple body w/o time_key" do
+    d = create_driver
+    d.run(default_tag: "test") do
+      d.feed(Fluent::EventTime.now, {"message" => '{"event":"Hello, world!", "sourcetype":"manual"}'})
+    end
+    assert do
+      d.filtered.size >= 1
+    end
+  end
+
   test "parsing condensed body" do
     d = create_driver
     d.run(default_tag: "test") do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-in-http-splunk-hec
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc1
+  version: 0.1.0.rc2
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-20 00:00:00.000000000 Z
+date: 2022-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler