fluent-plugin-unomaly 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4a41939225a6e96abf03128b090d945ec10d565df3d6f918c0fa7acae6c454b3
+  data.tar.gz: b6fbaa7088611b174780cf37127ba38c53a19cb8f57f6ef0b39f0f9e571f7962
+SHA512:
+  metadata.gz: d910ee8cfcc29d653528376e5b998a49f3c41e05c9693b1d37e48ccecf4fcfe05448f3c8d2c11b918021078ac885afef6c53c80aad9ab6cac5bfd0b1ab191131
+  data.tar.gz: 7162b2702e21e161fb09c2ff45ac3cb5dbc60e4360ae3b76ebfd5e4c3cf9afd270f15b7abf4304b73d5cfec8a9ab3b7f103b38d557e1c08f13d986cbc18a3522

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+*.lock

data/.travis.yml

@@ -0,0 +1,19 @@
+language: ruby
+rvm:
+  - 2.5.0
+gemfile:
+  - Gemfile
+
+before_install: gem update bundler
+script:
+  - bundle install
+  - bundle exec rake test
+sudo: false
+
+deploy:
+  provider: rubygems
+  api_key:
+    secure: LfVGVLEVibxY7IlJAP5T04O02Lz2xT0Z60EVCR3kLy4oaEJowHptuv3F0FF7kZlSJOKmFc1Ibvtebx7vsqIpcNCPt/wOKUQ/fjf1KRBkEuaDwRq/rOpHlBJNJtdF7PrquzuRV2Y7UKGfKjGVgn9rjykzHxFwtuWiDnMxaMADAqMPK3p7CLBixEHQ2Wpyi5/wNJo2k5YIyd/SSIiYiP5ey0n981cBNVTQnjMbhyBoWSjn4GFkmRflPht04ZrF8jyFOYkC7HvQsz3d0oBB+GOXXiZlhb1xZvH2ScRi/2U/rzw61Lqi0mZjEiavtmNgco2bR3+AfgikAI+iH0+lZqPhwZkig2X7SwZYHQIrPUxPp4C1Iwbe6TJgQXgxQ7Fa624p0DmkJan96yqN3iKLm9rz3A64nncyAJLN+2UfngLLtGhGzNeT8w9q/SNmX3N92qQKNH4qqXIe+gCxtxXiJUwbFxn66AY+2s007M9K8xK1iHn6twsQYk5c6Q+j+x3S1JbBvJflM1XQjivoAJ73R2TTui+H5QKo4ZePuESKYkdtxl+qn34OY23HuCJtd5S0YDY9YhlRyot0uqAYtijroEsR53of9KF3nbebKZGnOml6yohSGPx/wTuCDpRKszSfJfjHvoF1ntlM2d+xA/T7LxqFDJbmZkEtaQjp4c9gpCfy05M=
+  gem: fluent-plugin-unomaly
+  on:
+    tags: true

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# v0.1.2
+
+- First public release on rubygems

data/Dockerfile

@@ -0,0 +1,7 @@
+FROM ruby:latest
+RUN mkdir /unomaly
+COPY ./ unomaly
+WORKDIR /unomaly
+RUN bundle install
+RUN bundle exec rake test
+RUN gem build fluent-plugin-unomaly.gemspec

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Unomaly
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# fluent-plugin-unomaly
+
+[![Build Status](https://travis-ci.org/unomaly/fluent-plugin-unomaly.svg?branch=master)](https://travis-ci.org/unomaly/fluent-plugin-unomaly)
+
+This plugin sends Fluents records to the [Unomaly](https://www.unomaly.com) ingestion API (min version Unomaly 2.27).
+
+## Getting started
+
+- Install plugin `gem install fluent-plugin-unomaly`
+- Add to `fluent.conf`
+
+Minimal configuration:
+
+```
+<match tag>
+  @type unomaly
+  host https://my-unomaly.instance
+  flush_interval 1s
+  source_key host
+  message_key message
+</match>
+```
+
+# Important configuration options
+
+| Option                   | Description                                                                       | Default    |
+| ------------------------ | --------------------------------------------------------------------------------- | ---------- |
+| host                     | Unomaly instance address. Must define full path such as "https://my-instance:443" | No default |
+| message_key              | The key in the Logstash event that Unomaly should use for anomaly detection.      | "message"  |
+| source_key               | The event key defining the Unomaly system.                                        | "host"     |
+| accept_self_signed_certs | Accept self signed SSL certificate                                                | "false"    |
+
+## Contributing
+
+Bug reports and pull requests are welcome. This project is intended to
+be a safe, welcoming space for collaboration.

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/docker-compose.yml

@@ -0,0 +1,6 @@
+ruby:
+  build: .
+  volumes:
+    - ./:/unomaly
+  command:
+    - sh

data/fluent-plugin-unomaly.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |spec|
+    spec.name        = 'fluent-plugin-unomaly'
+    spec.version     = '0.1.2'
+  
+    spec.summary     = "Fluentd output plugin for Unomaly"
+    spec.description = "Fluentd output plugin for Unomaly"
+    spec.authors     = ['Unomaly']
+    spec.email       = 'support@unomaly.com'
+    spec.files       = []
+    spec.homepage    = 'https://github.com/unomaly/fluent-plugin-unomaly'
+    spec.license     = 'MIT'
+  
+    spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
+    spec.require_paths = ["lib"]
+  
+    spec.required_ruby_version = '>= 2.0.0'
+  
+    spec.add_runtime_dependency "fluentd", "~> 0.12"
+    spec.add_runtime_dependency "http", "< 3"
+  
+    spec.add_development_dependency "bundler", "~> 1.7"
+    spec.add_development_dependency "rake", "~> 10.0"
+    spec.add_development_dependency "webmock", "~> 2.1"
+    spec.add_development_dependency "test-unit"
+  end

data/lib/fluentd/plugin/out_unomaly.rb

@@ -0,0 +1,133 @@
+require 'fluent/output'
+require 'date'
+require 'time'
+require 'uri'
+require 'net/http'
+require 'net/https'
+
+module Fluent
+  class UnomalyOutput < BufferedOutput
+    Fluent::Plugin.register_output('unomaly', self)
+
+    # config_param defines a parameter. You can refer a parameter via @path instance variable
+
+    # Event batch size to send to Unomaly. Increasing the batch size can increase throughput by reducing HTTP overhead
+    config_param :batch_size,  :integer, :default => 50
+
+    # Unomaly host to send the logs to
+    config_param :host,  :string
+
+    # Key that will be used by Unomaly as the log message
+    config_param :message_key,  :string, :default => "message"
+
+    # Key that will be used by Unomaly as the log message
+    config_param :date_key,  :string, :default => nil
+
+    # Key that will be used by Unomaly as the system key
+    config_param :source_key,  :string, :default => "host"
+
+    # Unomaly api path to push events
+    config_param :api_path,  :string, :default => "/v1/batch"
+
+    # Display debug logs
+    config_param :debug,  :bool, :default => false
+
+    config_param :accept_self_signed_certs, :bool, :default => false
+
+    # This method is called before starting.
+    # 'conf' is a Hash that includes configuration parameters.
+    # If the configuration is invalid, raise Fluent::ConfigError.
+    def configure(conf)
+      super
+      conf["buffer_chunk_limit"] ||= "500k"
+      conf["flush_interval"] ||= "1s"
+    end
+
+    # This method is called when starting.
+    # Open sockets or files here.
+    def start
+      super
+    end
+
+    # This method is called when shutting down.
+    # Shutdown the thread and close sockets or files here.
+    def shutdown
+      super
+    end
+
+    # This method is called when an event reaches to Fluentd.
+    # Convert the event to a raw string.
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # This method is called every flush interval. Write the buffer chunk
+    # to files or databases here.
+    # 'chunk' is a buffer chunk that includes multiple formatted
+    # events. You can use 'data = chunk.read' to get all events and
+    # 'chunk.open {|io| ... }' to get IO objects.
+    #
+    # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
+    def write(chunk)
+      documents = []
+      chunk.msgpack_each do |(tag, time, record)|
+        unomaly_event = {
+            message: record[@message_key],
+            source: record[@source_key],
+            timestamp: Time.at(time).utc.to_datetime.rfc3339
+        }
+        metadata = record.to_hash
+
+        metadata.delete(@source_key)
+        metadata.delete(@message_key)
+        metadata["tag"]=tag
+
+        unomaly_event["metadata"]=flatten(metadata,"")
+
+        documents.push(unomaly_event)
+      end
+      send_batch(documents)
+    end
+
+    def send_batch(events)
+      url = @host + @api_path
+      body = events.to_json
+      uri = URI.parse(url)
+      header = {'Content-Type' => 'application/json'}
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      if @accept_self_signed_certs
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      request = Net::HTTP::Post.new(uri.request_uri, header)
+      request.body = body
+
+      resp = http.request(request)
+      if !resp.kind_of? Net::HTTPSuccess
+        log.error "Error sending batch #{resp.to_s}"
+      end
+    end
+
+
+    def flatten(data, prefix)
+      ret = {}
+      if data.is_a? Hash
+        data.each { |key, value|
+          if prefix.to_s.empty?
+            ret.merge! flatten(value, "#{key.to_s}")
+          else
+            ret.merge! flatten(value, "#{prefix}.#{key.to_s}")
+          end
+        }
+      elsif data.is_a? Array
+        data.each_with_index {|val,index | ret.merge! flatten(val, "#{prefix}.#{index}")}
+      else
+        return {prefix => data.to_s}
+      end
+
+      ret
+    end
+  end
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-unomaly
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Unomaly
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-12-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd output plugin for Unomaly
+email: support@unomaly.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- Dockerfile
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- docker-compose.yml
+- fluent-plugin-unomaly.gemspec
+- lib/fluentd/plugin/out_unomaly.rb
+homepage: https://github.com/unomaly/fluent-plugin-unomaly
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin for Unomaly
+test_files: []