logstash-output-sumologic 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 15ebf5a83834cd6cad674708a813abe835270481
+  data.tar.gz: 930252e23ef4d415f321a5f837bc43ced4b56a74
+SHA512:
+  metadata.gz: bb790a13d2f84126b7ea350ae0413398858f3abef2054063de8fbf56913affef97e116f06950c9df413ece298c1753dc674e540a1e4ef5c05708be02a2b1e67c
+  data.tar.gz: f8ce4d93bf8b70a16a26004df5db415f6bc9c2231504cdfa674a4b17df5ab0057e183d19bdd3fc5f0b3cfe0986d00c109d67fdd845c1e9ac0c732b878ebd50cc

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 1.0.0
+ - First public release

data/CONTRIBUTORS

@@ -0,0 +1,13 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped Sumo Logic plug-in along its way.
+
+Contributors:
+* Bin Yi (bin3377)
+* Brain Goleno (bgoleno)
+* Jacek Migdal (jakozaur)
+* Cris Dessonville (cddude229)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Sumo Logic plug-in, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-output-sumologic
+Logstash output plugin for delivering log to Sumo Logic cloud service through HTTP source.

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem 'rspec'

data/LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2016 Sumo Logic <https://www.sumologic.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
+
+This product includes software developed by Elasticsearch (http://www.elastic.co/).

data/README.md

@@ -0,0 +1,49 @@
+# Logstash Sumo Logic Output Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Getting Started
+
+### 1. Create a Sumo Logic HTTP source
+- Create a [Sumo Logic](https://www.sumologic.com/) free account if you currently don't have one.
+- Create a [HTTP source](http://help.sumologic.com/Send_Data/Sources/HTTP_Source) in your account and get the URL for this source. It should be something like 
+```
+https://events.sumologic.net/receiver/v1/http/XXXXXXXXXX
+```
+
+### 2. Install LogStash on your machine
+- Following this [instruction](https://www.elastic.co/guide/en/logstash/current/getting-started-with-logstash.html) to download and install LogStash. This plugin require Logstash 2.3 or higher to run.
+
+### 3. Build plugin gem and install to LogStash
+- Build your plugin gem
+
+In your local Git clone, running:
+```sh
+gem build logstash-output-sumologic.gemspec
+```
+
+You will get a .gem file as `logstash-output-sumologic-1.0.0.gem`
+
+- Install plugin into LogStash
+
+In the Logstash home, running:
+```sh
+bin/logstash-plugin install <path of .gem>
+```
+
+### 4. Start Logstash and send log
+In the Logstash home, running:
+```sh
+bin/logstash -e 'input{stdin{}}output{sumologic{url=>"<url from step 1>"}}'
+```
+
+This will send any input from console to Sumo Logic cloud service.
+
+### 5. Get result from Sumo Logic web app
+- Logon to Sumo Logic [web app](https://prod-www.sumologic.net/ui/) and run [Search](http://help.sumologic.com/Search) or [Live Tail](http://help.sumologic.com/Search/Live_Tail)
+
+### Further things
+- Try it with different input/filter/codec plugins
+- Start LogStash as a service/daemon in your production environment 
+- Report and issue or idea through [Git Hub](https://github.com/SumoLogic/logstash-output-sumologic)

data/lib/logstash/outputs/sumologic.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+require "logstash/json"
+require "logstash/namespace"
+require "logstash/outputs/base"
+require "logstash/plugin_mixins/http_client"
+require 'thread'
+require "uri"
+require "zlib"
+
+# Now you can use logstash to deliver logs to Sumo Logic
+#
+# Create a HTTP Source
+# in your Sumo Logic account and you can now use logstash to parse your log and 
+# send your logs to your account at Sumo Logic.
+#
+class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+  
+  config_name "sumologic"
+  
+  # The hostname to send logs to. This should be given when creating a HTTP Source
+  # on Sumo Logic web app http://help.sumologic.com/Send_Data/Sources/HTTP_Source
+  config :url, :validate => :string, :required => true
+
+  # Include extra HTTP headers on request if needed 
+  config :extra_headers, :validate => :hash, :default => []
+
+  # The formatter of message, by default is message with timestamp as prefix
+  config :format, :validate => :string, :default => "%{@timestamp} %{host} %{message}"
+
+  # Hold messages for at least (x) seconds as a pile; 0 means sending every events immediately  
+  config :interval, :validate => :number, :default => 0
+
+  # Compress the payload 
+  config :compress, :validate => :boolean, :default => false
+
+  public
+  def register
+    # initialize request pool
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times { |t| @request_tokens << true }
+    @timer = Time.now
+    @pile = Array.new
+    @semaphore = Mutex.new
+  end # def register
+
+  public
+  def multi_receive(events)
+    events.each { |event| receive(event) }
+    client.execute!
+  end # def multi_receive
+  
+  public
+  def receive(event)
+    if event == LogStash::SHUTDOWN
+      finished
+      return
+    end
+
+    content = event.sprintf(@format)
+    
+    if @interval <= 0 # means send immediately
+      send_request(content)
+      return
+    end
+
+    @semaphore.synchronize {
+      now = Time.now
+      @pile << content
+
+      if now - @timer > @interval # ready to send
+        send_request(@pile.join($/))
+        @timer = now
+        @pile.clear
+      end
+    }
+  end # def receive
+
+  public
+  def close
+    @semaphore.synchronize {
+      send_request(@pile.join($/))
+      @pile.clear
+    }
+    client.close
+  end # def close
+  
+  private
+  def send_request(content)
+    token = @request_tokens.pop
+    body = if @compress
+      Zlib::Deflate.deflate(content)
+    else
+      content
+    end
+    headers = get_headers()
+
+    request = client.send(:parallel).send(:post, @url, :body => body, :headers => headers)
+    request.on_complete do
+      @request_tokens << token
+    end
+
+    request.on_success do |response|
+      if response.code < 200 || response.code > 299
+        log_failure(
+          "HTTP response #{response.code}",
+          :body => body,
+          :headers => headers
+      )
+      end
+    end
+
+    request.on_failure do |exception|
+      log_failure(
+        "Could not fetch URL",
+        :body => body,
+        :headers => headers,
+        :message => exception.message,
+        :class => exception.class.name,
+        :backtrace => exception.backtrace
+      )
+    end
+
+    request.call
+  end # def send_request
+  
+  private
+  def get_headers()
+    base = { "Content-Type" => "text/plain" }
+    base["Content-Encoding"] = "deflate" if @compress
+    return base.merge(@extra_headers)
+  end # def get_header
+  
+  private
+  def log_failure(message, opts)
+    @logger.error(message, opts)
+  end # def log_failure
+
+end # class LogStash::Outputs::SumoLogic

data/logstash-output-sumologic.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-sumologic'
+  s.version = "1.0.0"
+  s.licenses = ["Apache-2.0"]
+  s.summary = "Deliever the log to Sumo Logic cloud service."
+  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Sumo Logic"]
+  s.email = "byi@sumologic.com"
+  s.homepage = "http://www.sumologic.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/sumologic_spec.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/sumologic"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::SumoLogic do
+  let(:sample_event) { LogStash::Event.new }
+  let(:output) { LogStash::Outputs::SumoLogic.new }
+
+  before do
+    output.register
+  end
+
+  describe "receive message" do
+    subject { output.receive(sample_event) }
+
+    it "returns a string" do
+      expect(subject).to eq("Event received")
+    end
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-sumologic
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Sumo Logic
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: byi@sumologic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/sumologic.rb
+- logstash-output-sumologic.gemspec
+- spec/outputs/sumologic_spec.rb
+homepage: http://www.sumologic.com
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.6
+signing_key: 
+specification_version: 4
+summary: Deliever the log to Sumo Logic cloud service.
+test_files:
+- spec/outputs/sumologic_spec.rb