logstash-output-splunk_hec 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9acf5738a6f95a638b41287b71bcaec122a713a144ddfdbe4e8eacb58bfd95ca
+  data.tar.gz: 050a2753e156c7cb195701b1460605f767e6351bfedaacd56214f27b955b4a43
+SHA512:
+  metadata.gz: 7c0cc2d47d092a10fedf6ad8831be42eb3fd84ee173f871d6eb5dc5d546d6015539763f56f908b2e0bbe96082f5f9fedda5a6c54d8584a99611f00fe86392a49
+  data.tar.gz: cd7d3d98d661ee1da85893c8444a0df15da2504c9ce52d70ada5357e8da458e3fa3c8bed85c6afaaee7af91a05ffe9bf8a389e4d99a69b4f6f6f49eff084df99

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Initial release, includes basic batching functionality

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Elisha Mawson - elisha.mawson@origin.com.au
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+gemspec
+
+logstash_path = ENV['LOGSTASH_PATH'] || 'C:/Users/mawsone/logstash-8.15.0'
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end
+

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1 @@
+# logstash_splunk_http_event_collector

data/lib/logstash/outputs/splunk_hec.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "net/http"
+require "uri"
+require "json"
+require "concurrent"
+
+class LogStash::Outputs::SplunkHec < LogStash::Outputs::Base
+  config_name "splunk_hec"
+
+  concurrency :shared
+
+  config :token, :validate => :string, :required => true
+  config :host, :validate => :string, :required => true
+  config :port, :validate => :number, :default => 443
+  config :index, :validate => :string, :default => "main"
+  config :sourcetype, :validate => :string, :default => "_json"
+  config :batch_size, :validate => :number, :default => 100
+  config :flush_interval, :validate => :number, :default => 5
+  config :retry_count, :validate => :number, :default => 3
+
+  public
+  def register
+    @http = Net::HTTP.new(@host, @port)
+    @http.use_ssl = true
+    @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    @uri = URI.parse("https://#{@host}:#{@port}/services/collector/event")
+    
+    @event_batch = Concurrent::Array.new
+    @last_flush = Concurrent::AtomicReference.new(Time.now)
+  end
+
+  public
+  def receive(event)
+    format_and_add_to_batch(event)
+    
+    if batch_full? || time_to_flush?
+      flush_batch
+    end
+  end
+
+  public
+  def close
+    flush_batch if @event_batch.any?
+  end
+
+  private
+  def format_and_add_to_batch(event)
+    event_data = event.to_hash
+    event_data.delete("@version")
+
+    hec_event = {
+      "time" => event.get("@timestamp").to_i,
+      "host" => event.get("host")&.fetch("name") { Socket.gethostname },
+      "source" => event.get("source") { "logstash" },
+      "sourcetype" => @sourcetype,
+      "index" => @index,
+      "event" => event_data
+    }
+
+    @event_batch << hec_event
+  end
+
+  private
+  def batch_full?
+    @event_batch.size >= @batch_size
+  end
+
+  private
+  def time_to_flush?
+    Time.now - @last_flush.get >= @flush_interval
+  end
+
+  private
+  def flush_batch
+    return if @event_batch.empty?
+
+    batch_to_send = @event_batch.slice!(0, @batch_size)
+    request = Net::HTTP::Post.new(@uri.request_uri)
+    request["Authorization"] = "Splunk #{@token}"
+    request["Content-Type"] = "application/json"
+    request.body = batch_to_send.map(&:to_json).join("\n")
+
+    @retry_count.times do |attempt|
+      begin
+        response = @http.request(request)
+        if response.code == "200"
+          @logger.debug("Successfully sent batch to Splunk", :batch_size => batch_to_send.size)
+          @last_flush.set(Time.now)
+          return
+        else
+          @logger.warn("Failed to send batch to Splunk, will retry",
+                       :response_code => response.code,
+                       :response_body => response.body,
+                       :attempt => attempt + 1,
+                       :batch_size => batch_to_send.size)
+        end
+      rescue StandardError => e
+        @logger.error("Error sending batch to Splunk, will retry",
+                      :error => e.message,
+                      :attempt => attempt + 1,
+                      :batch_size => batch_to_send.size)
+      end
+      sleep(1)
+    end
+
+    @logger.error("Failed to send batch to Splunk after #{@retry_count} attempts",
+                  :batch_size => batch_to_send.size)
+    @event_batch.concat(batch_to_send)
+  end
+end

data/logstash-output-splunk_hec.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-output-splunk_hec'
+  s.version       = '0.1.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Logstash Output Plugin for SplunkHec'
+  s.authors       = ['Elisha Mawson']
+  s.email         = 'elisha.mawson@origin.com.au'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/splunk_hec_spec.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/splunk_hec"
+require "logstash/codecs/plain"
+
+
+describe LogStash::Outputs::SplunkHec do
+  let(:sample_event) { LogStash::Event.new }
+  let(:output) { LogStash::Outputs::SplunkHec.new }
+
+  before do
+    output.register
+  end
+
+  describe "receive message" do
+    subject { output.receive(sample_event) }
+
+    it "returns a string" do
+      expect(subject).to eq("Event received")
+    end
+  end
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-splunk_hec
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Elisha Mawson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-08-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: elisha.mawson@origin.com.au
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/splunk_hec.rb
+- logstash-output-splunk_hec.gemspec
+- spec/outputs/splunk_hec_spec.rb
+homepage: 
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.5
+signing_key: 
+specification_version: 4
+summary: Logstash Output Plugin for SplunkHec
+test_files:
+- spec/outputs/splunk_hec_spec.rb