fluent-plugin-insight 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 287cbbf3842b2af256f8ee95a49a7a6e182f2104
+  data.tar.gz: ac0ff0277597bc853337c2e3ce67d4b916442844
+SHA512:
+  metadata.gz: 55ec27d97d5b8a49a72e2679cfbead30b32f66411e9b7c39f8c38f2e64aaddc453d39e16cb387be4a6b729bbc7471ef6fed8405d0e08ae0a54f8243804343741
+  data.tar.gz: dc1da043b1d38347fe060261bfab70f80dc83a9e58aeb7bb3bfe3446383a3ca8d70916d369a54392bcf6c4966dbb31a44280358324881160a1a01f1a4c99a28d

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-insight.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Tweddle Group
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,73 @@
+# Fluent::Plugin::Insight
+Forward logs to Insight, using token based input.
+
+Using Insight REST API gets log tokens by logset id and matches `record['log']` of every record to a log name in logset
+
+## Installation
+
+install with gem or fluent-gem command as:
+
+### native gem
+    $ gem install fluent-plugin-insight
+
+### fluentd gem
+    $ /opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-insight
+
+## Usage
+
+```
+    <match pattern>
+      type insight
+      tags hostname,environment,application
+      prefix %{hostname} %{environment}-%{application}
+      api_key test
+      logset_id id
+      region eu
+    </match>
+```
+
+## Parameters
+
+### type (required)
+The value must be `insight`.
+
+### logset_id (required)
+InsightOPS logset id
+
+### region (required)
+InsightOPS region
+
+### tags
+List of record keys which can be retrieved from record to build a log record prefix
+
+### prefix
+Prefix is added to every log record. It's a format string which uses `tags` values
+
+### protocol
+The default is `tcp`.
+
+### use_ssl
+Enable/disable SSL for data transfers between Fluentd and Insight. The default is `true`.
+
+### port
+Only in case you don't use SSL, the value must be `80`, `514`, or `10000`. The default is `20000` (SSL)
+
+### max_retries
+Number of retries on failure.
+
+## Contributing
+
+1. Fork it ( http://github.com/Tweddle-SE-Team/fluent-plugin-insight/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## MIT
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fluent-plugin-insight.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-insight"
+  spec.version       = "0.0.2"
+  spec.authors       = ["AndrewChubatiuk"]
+  spec.email         = ["ops@tweddle.com"]
+  spec.summary       = "InsightOPS output plugin for Fluent event collector"
+  spec.homepage      = "https://github.com/Tweddle-SE-Team/fluent-plugin-insight"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake", "~> 0"
+end

data/lib/fluent/plugin/out_insight.rb

@@ -0,0 +1,161 @@
+require 'socket'
+require 'yaml'
+require 'openssl'
+require 'net/http'
+require 'json'
+require 'thread'
+
+class Fluent::InsightOutput < Fluent::BufferedOutput
+  class ConnectionFailure < StandardError; end
+
+  Fluent::Plugin.register_output('insight', self)
+
+  INSIGHT_DATA_TEMPLATE = "%{region}.data.logs.insight.rapid7.com"
+  INSIGHT_REST_TEMPLATE = "https://%{region}.rest.logs.insight.rapid7.com"
+  INSIGHT_LOGSETS_TEMPLATE = "/management/logsets/%{logset_id}"
+  THREAD_COUNT = 4
+
+  config_param :use_ssl,     :bool,    :default => true
+  config_param :port,        :integer, :default => 20000
+  config_param :protocol,    :string,  :default => 'tcp'
+  config_param :max_retries, :integer, :default => 3
+  config_param :tags,        :string,  :default => ''
+  config_param :prefix,      :string,  :default => ''
+  config_param :default,     :string,  :default => 'default'
+  config_param :api_key
+  config_param :logset_id
+  config_param :region
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    logsets_url = (INSIGHT_REST_TEMPLATE + INSIGHT_LOGSETS_TEMPLATE) % { :region => @region, :logset_id => @logset_id }
+    @insight_tags = Hash[@tags.split(",").each_with_object(nil).to_a]
+    logset_body = insight_rest_request(logsets_url)
+    threads = []
+    @tokens = Hash.new
+    mutex = Mutex.new
+    if logset_body.key?('logset')
+      logset_info = logset_body['logset']
+      if logset_info.key?('logs_info')
+        logs_info = logset_info['logs_info']
+        log_urls = logs_info.map { |log| log['links'][0]['href'] }
+        THREAD_COUNT.times.map {
+          Thread.new(log_urls, @tokens) do |urls, tokens|
+            while url = mutex.synchronize { urls.pop }
+              log_name, token = insight_log_token(url)
+              mutex.synchronize { tokens[log_name] = token }
+            end
+          end
+        }.each(&:join)
+      else
+        log.warn "No logs info found in logset response"
+      end
+    else
+      log.warn "Logset emtity is empty"
+    end
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def insight_rest_request(url)
+    uri = URI(url)
+    request = Net::HTTP::Get.new(uri)
+    request['content-type'] = 'application/json'
+    request['x-api-key'] = @api_key
+    response = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => uri.scheme == 'https') {|http|
+      http.request(request)
+    }
+    if response.code == "200"
+      return JSON.parse(response.body)
+    end
+      log.error "Request was failed HTTP #{response.code}: \n#{response.body}"
+  end
+
+  def insight_log_token(url)
+    log_body = insight_rest_request(url)
+    if log_body.key?('log')
+      log_info = log_body['log']
+      if log_info.key?('tokens')
+        log.info "Found log #{log_info['name']}"
+        return log_info['name'], log_info['tokens'][0]
+      else
+        log.warn "Log is empty"
+      end
+    else
+      log.warn "Response doesn't contain log info"
+    end
+  end
+
+  def client
+    insight_data_host = INSIGHT_DATA_TEMPLATE % { :region => @region }
+    @_socket ||= if @use_ssl
+      context    = OpenSSL::SSL::SSLContext.new
+      socket     = TCPSocket.new insight_data_host, @port
+      ssl_client = OpenSSL::SSL::SSLSocket.new socket, context
+      ssl_client.connect
+    else
+      if @protocol == 'tcp'
+        TCPSocket.new insight_data_host, @port
+      else
+        udp_client = UDPSocket.new
+        udp_client.connect insight_data_host, @port
+        udp_client
+      end
+    end
+  end
+
+  def format(tag, time, record)
+    return [tag, time, record].to_msgpack
+  end
+
+  def write(chunk)
+    return if @tokens.empty?
+    chunk.msgpack_each do |(tag, time, record)|
+      next unless record.is_a? Hash
+      message = (record.delete('message')&.to_s&.rstrip || '')
+      next if message.empty?
+      @insight_tags.each { |k,v|
+        @insight_tags[k] = record[k]
+      }
+      symbolized_tags = @insight_tags.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+      if @tokens.key?(record['log'])
+        token = @tokens[record['log']]
+        prefix = @prefix % symbolized_tags
+        send_insight(token,  "#{prefix} #{message}")
+      elsif @tokens.key?(@default)
+        token = @tokens[@default]
+        prefix = @prefix % symbolized_tags
+        send_insight(token,  "#{prefix} #{message}")
+      else
+        log.debug "No token found for #{record['log']} and default log doesn't exist"
+      end
+    end
+  end
+
+  def send_insight(token, data)
+    retries = 0
+    begin
+      client.write("#{token} #{data} \n")
+    rescue Errno::EMSGSIZE
+      str_length = data.length
+      send_insight(token, data[0..str_length/2])
+      send_insight(token, data[(str_length/2) + 1..str_length])
+      log.warn "Message Too Long, re-sending it in two part..."
+    rescue => e
+      if retries < @max_retries
+        retries += 1
+        @_socket = nil
+        log.warn "Could not push logs to Insight, resetting connection and trying again. #{e.message}"
+        sleep 5**retries
+        retry
+      end
+      raise ConnectionFailure, "Could not push logs to Insight after #{retries} retries. #{e.message}"
+    end
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-insight
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- AndrewChubatiuk
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-06-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ops@tweddle.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-insight.gemspec
+- lib/fluent/plugin/out_insight.rb
+homepage: https://github.com/Tweddle-SE-Team/fluent-plugin-insight
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.3
+signing_key: 
+specification_version: 4
+summary: InsightOPS output plugin for Fluent event collector
+test_files: []