fluent-plugin-logentries 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54dd09205e71e513063bb8346d512d2bf42477dd
-  data.tar.gz: 9a4280574f047836fcb4bf5f6d3ff81cc458c121
+  metadata.gz: ede9927a84f4346ffdf4298c2c4613a66a96a855
+  data.tar.gz: 7b4572090c1ef0d26a02518700859a0bc508f349
 SHA512:
-  metadata.gz: 3dd6de0d69e28416d7c92fe74e78ee454e798dfdfe3febc23320e92db1187d85f871ec979bae9aceb9c4e96cdbee4d1a3c9c2c8daf7d7b445d6bbb8e8ce70bc1
-  data.tar.gz: 874d63485f551b2625bf0cb411c674b3f234d6dc8ef60708622fc6578eab7067f3a02fc9cee5b85a854a543307de9f17d2bfb84024d9242f4373523161f0fb84
+  metadata.gz: 5ad38d2ba03195a7a557075e113476314739e4625a1b140ed1bb320d3c75c5d1174ed7c4c9ea3c983f21406d3624627e0025d5aa7dbdc1b57d45180f93fb334e
+  data.tar.gz: ae09d8debfaabb0db41fde0bb5cdb479d2db14be08ee69ef6938009bd54155b51fb2776de050458281cdf49551eee8eae0ca11692a2050a5521cab96489bb3ac

data/README.md

@@ -1,29 +1,69 @@
 # Fluent::Plugin::Logentries
+Forward logs to Logentries, using token based input.
 
-TODO: Write a gem description
+Looks at the tag/message to find out where the log should go.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+    /opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-logentries
 
-    gem 'fluent-plugin-logentries'
+## Configruation file (YML)
 
-And then execute:
+```yaml
+    My-Awesome-App:
+       app: MY-LOGENTRIES-TOKEN
+       access: ANOTHER-LOGENTRIES-TOKEN (*)
+       error: ANOTHER-LOGENTRIES-TOKEN-1 (*)
+    Foo:
+       app: 2bfbea1e-10c3-4419-bdad-7e6435882e1f
+       access: 5deab21c-04b1-9122-abdc-09adb2eda22 (*)
+       error: 9acfbeba-c92c-1229-ccac-12c58d82ecc (*)
+```
+(*) `access` and `error` are optional, if you don't use multiple log per host just provide an app token.
 
-    $ bundle
+This file is read everytime the buffer is flushed, it allows on fly modifications.
+## Usage
 
-Or install it yourself as:
+```
+    <match pattern>
+      type logentries
+      config_path /path/to/logentries-tokens.conf
+    </match>
+```
 
-    $ gem install fluent-plugin-logentries
+## Parameters
 
-## Usage
+### type (required)
+The value must be `logentries`.
+
+### config_path (required)
+Path of your configuration file, e.g. `/opt/logentries/tokens.conf`
+
+### use_ssl
+Enable/disable SSL for data transfers between Fluentd and Logentries. The default is `true`.
 
-TODO: Write usage instructions here
+### no_ssl_port
+Only in case you don't use SSL, the value must be `80`, `514`, or `10000`. The default is `80`
+
+### max_retries
+Number of retries on failure.
+
+### tag_access_log, tag_error_log
+This is use in case you tag your access/error log and want them to be push into another log.
 
 ## Contributing
 
-1. Fork it ( http://github.com/<my-github-username>/fluent-plugin-logentries/fork )
+1. Fork it ( http://github.com/woorank/fluent-plugin-logentries/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+## MIT
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/fluent-plugin-logentries.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-logentries"
-  spec.version       = "0.1.1"
+  spec.version       = "0.2.0"
   spec.authors       = ["Woorank"]
   spec.email         = ["dev@woorank.com"]
   spec.summary       = "Logentries output plugin for Fluent event collector"

data/lib/fluent/plugin/out_logentries.rb

@@ -1,4 +1,6 @@
 require 'socket'
+require 'yaml'
+require 'openssl'
 
 class LogentriesOutput < Fluent::BufferedOutput
   class ConnectionFailure < StandardError; end
@@ -6,9 +8,12 @@ class LogentriesOutput < Fluent::BufferedOutput
   # and identifies the plugin in the configuration file.
   Fluent::Plugin.register_output('logentries', self)
 
-  config_param :host, :string
-  config_param :port, :integer, :default => 80
-  config_param :path, :string
+  config_param :use_ssl,        :bool,    :default => true
+  config_param :no_ssl_port,    :integer, :default => 80
+  config_param :config_path,    :string
+  config_param :max_retries,    :integer, :default => 3
+  config_param :tag_access_log, :string,  :default => 'logs-access'
+  config_param :tag_error_log,  :string,  :default => 'logs-error'
 
   def configure(conf)
     super
@@ -23,7 +28,15 @@ class LogentriesOutput < Fluent::BufferedOutput
   end
 
   def client
-    @_socket ||= TCPSocket.new @host, @port
+    @_socket ||= if @use_ssl
+      context    = OpenSSL::SSL::SSLContext.new
+      socket     = TCPSocket.new "api.logentries.com", 20000
+      ssl_client = OpenSSL::SSL::SSLSocket.new socket, context
+
+      ssl_client.connect
+    else
+      TCPSocket.new "data.logentries.com", @no_ssl_port
+    end
   end
 
   # This method is called when an event reaches Fluentd.
@@ -31,27 +44,39 @@ class LogentriesOutput < Fluent::BufferedOutput
     return [tag, record].to_msgpack
   end
 
-  # Scan a given directory for logentries tokens
-  def generate_token(path)
-    tokens = {}
-
-    Dir[path + "*.token"].each do |file|
-      key = File.basename(file, ".token") #remove path/extension from filename
-      #read the first line, remove unwanted char and close it
-      tokens[key] = File.open(file, &:readline).gsub(/\r\n|\r|\n/, '')
+  # Create tokens hash
+  def generate_token
+    begin
+      YAML::load_file(@config_path)
+    rescue Exception => e
+      log.warn "Could not load configuration. #{e.message}"
     end
-
-    tokens
   end
 
-  # returns the correct token to use for a given tag / Records
+  # Returns the correct token to use for a given tag / records
   def get_token(tag, record, tokens)
-    tag    ||= ""
-    record ||= ""
-
+    tag     ||= ""
+    message = record["message"]
+
+    # Config Structure
+    # -----------------------
+    # app-name:
+    #   app: TOKEN
+    #   access: TOKEN (optional)
+    #   error: TOKEN  (optional)
     tokens.each do |key, value|
-      if tag.index(key) != nil || record.index(key) != nil then
-        return value
+      if tag.index(key) != nil || message.index(key) != nil then
+        default = value['app']
+
+        case tag
+          when @tag_access_log
+            return value['access'] || default
+          when @tag_error_log
+            return value['error']  || default
+
+          else
+            return default
+        end
       end
     end
 
@@ -60,17 +85,17 @@ class LogentriesOutput < Fluent::BufferedOutput
 
   # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
   def write(chunk)
-    tokens = generate_token(@path)
+    tokens = generate_token()
+    return unless tokens.is_a? Hash
 
     chunk.msgpack_each do |tag, record|
       next unless record.is_a? Hash
+      next unless record.has_key? "message"
 
       token = get_token(tag, record, tokens)
       next if token.nil?
 
-      if record.has_key?("message")
-        send_logentries(record["message"] + ' ' + token)
-      end
+      send_logentries(token + ' ' + record["message"])
     end
   end
 
@@ -79,11 +104,11 @@ class LogentriesOutput < Fluent::BufferedOutput
     begin
       client.puts data
     rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
-      if retries < 2
+      if retries < @max_retries
         retries += 1
         @_socket = nil
         log.warn "Could not push logs to Logentries, resetting connection and trying again. #{e.message}"
-        sleep 2**retries
+        sleep 5**retries
         retry
       end
       raise ConnectionFailure, "Could not push logs to Logentries after #{retries} retries. #{e.message}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-logentries
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Woorank
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-19 00:00:00.000000000 Z
+date: 2015-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler