fluent-plugin-buffered-metrics 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+/certs/*key*
+/certs/*priv*
+**.gem
+**.gem.sha*

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2017 Red Book Connect LLC. operating as HotSchedules
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.

data/LICENSE-Apache-2.0.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,117 @@
+# Fluentd plugin for deriving metrics from output buffer chunks
+
+## Overview
+The buffered_metrics plugin started off as something originally written to write metrics to the old Stackdriver v1.0 API.  The intended purpose is to be able to derive metrics on scheduled intervals, not in real time as individual events pass through the Fluentd processing pipeline.  It is implemented as a subclass of the BufferedOutput class and is derives metrics from logs on a per-chunk basis, with the specified buffer flushing interval serving as the metrics collection and publishing interval.  It is kind of hacky and relies on Ruby evals in an attempt to make the metrics derivations as generic and flexible as possible.
+
+The metrics backend has been completely reworked as a set of subclasses in n effort to separate the metrics derivation from the metrics publishing.  This was done in an attempt to avoid backend conversion work, going forward, since it should now be possible to swap a new metrics backend into place without having to touch the parsers.
+
+Currently, the graphite backend is known to work and the statsd backend should work, but hasn't really been tested.  However, the statsd backend has bene hard-coded to "count" data types, since buffered collection for statsd doesn't make a lot of sense since timestamps cannot be preserved.
+
+## Installation
+```bash
+gem install fluent-plugin-buffered-metrics
+```
+
+If using the td-agent installation, use the following.
+
+```bash
+/opt/td-agent/embedded/bin/gem install fluent-plugin-buffered-metrics
+```
+
+## Configuration
+
+Using this will probably require using the Fluentd copy output plugin.  Using the Fluentd copy_ex plugin (fluent-plugin-copy_ex) is highly recommended so that output chains do not short-circuit prior to the metrics handling.  Putting this output at the end of the chain is also highly recommended.
+
+### Parameters
+
+`metrics_backend`: name of the metrics backend, currently either 'graphite' or 'statsd' (defaults to 'graphite')
+
+`url`: endpoint for metrics publication (defaults to 'tcp://localhost:2003' for graphite and 'udp://localhost:8125' for statsd)
+
+`prefix`: used prepend all metric names (optional)
+
+`sum_maps`: a JSON serialized hash where the keys are Ruby expressions which evaluate to either a Numeric or non-Numeric type and the values are arrays with the the names (Ruby expresssion which evalate to strings) of the metrics to add the value to, when Numeric. (optional)
+
+`metric_maps`: a JSON serialized hash where the keys evaluate to Booleans ad the values evaluate to metric hashes (optional)
+
+`counter_defaults`: a JSON serialized array contain default values for metrics which should be sent if there are no occurences of the metric in a buffer chunk (optional)
+
+`metric_defaults`: a JSON serialized array contain default values for metrics which should be sent if there are no occurences of the metric in a buffer chunk (optional)
+
+#### Notes
+
+The `url` parameter, in particular, is not something which is typically used in plugins such as this.  In particular, there is *no* hard-coded requirement for the endpoint to be a network socket, regardless of how the metrics backend is typically configured, with individual `host`, `port`, and `proto` parameters required. By using a single `url` parameter, any valid URL can be specified (so long as the handler has been implemented).  For instancee, an URL such as "file:///var/tmp/test.out" could be used in any configuration, even if the metrics backend is typically never configured in this way.  The primary reason for doing this is to allow for a simple way to debug configurations prior to putting them into live service without having to set up an ad hoc port listener or packet sniff the transmission just to see if the outputs are even in the correct format.
+
+### Example configuration
+
+Note that the following is at test configuration, appending to a file on the local filesystem prior to publishing anything to the live backend.
+```
+<match **>
+  @type copy_ex
+  <store ignore_error>
+    [your stand log output configuration(s)]
+  </store>
+  <store ignore_error>
+    @type buffered_metrics
+    metrics_backend graphite
+    prefix fluentd.<hostname>.5m
+    #url tcp://localhost:2003
+    url file:///var/tmp/graphite_test.out
+    sum_defaults [{"-.-.-.count":0},{"-.-.-.bytes":0}]
+    sum_maps {"event['record'].empty? ? false : 1":["-.-.-.count","(['tag','facility','level'].map {|t| event['record'][t] || '-'}+['count']).join('.')"],"event['record'].empty? ? false : event.to_s.length":["-.-.-.bytes","(['tag','facility','level'].map {|t| event['record'][t] || '-'}+['bytes']).join('.')"]}
+    metric_maps {}
+    metric_defaults []
+    flush_interval 5m
+    # Make this a working configuration
+    [standard BufferedOutput parameters]
+  </store>
+</source>
+```
+
+Admittedly, the input specification is rather ugly, but there is really no "pretty" way to specify the inputs in a way which preserves the flexibility.  The "maps" are intended to be a "data-driven programming" inputs, where keys are the matching conditions and the values are the actions on matches.
+
+Note that events are processed using the following data structure.  The following represents the data structure as JSON, but the event is *not* serialized for processing -- the processing is not based on regex parsing a JSON string serialization of the event.  The Fluentd event metadata (`tag` and `timestamp`) are available, as well as the actual event record data in the `record` data structue.
+
+```JSON
+  {
+    "tag": <Fluentd event tag>,
+    "timestamp": <Fluentd event timestamp>.
+    "record": { <keys/values> }
+  }
+```
+
+#### sum_maps:
+```JSON
+  {
+     "event[record].empty? ? false : 1" : [
+        "-.-.-.count",
+        "([tag,facility,level].map {|t| event[record][t] || -}+[count]).join('.')"
+     ],
+     "event[record].empty? ? false : event.to_s.length" : [
+        "-.-.-.bytes",
+        "([tag,facility,level].map {|t| event[record][t] || -}+[bytes]).join('.')"
+     ]
+  }
+```
+
+The `event[record].empty? ? false : 1` key is a Ruby expression which evaluates to `false` (a Boolean type -- not Numeric) when it should do anything with this particular entry in the buffer chunk, and a `1` (an Integer type -- definitely Numeric).  It may be a bit convoluted, but this implements counters (ie. the Numeric value can be anything, including values derived data in the event).
+
+The first element in the array, `-.-.-.count` is not a Ruby expression, so it does not dynamically evaluate to anything.  Every time the key evaulates to "1", the `<prefix>.-.-.-.count` is incremented.
+
+The second element in the array, `([tag,facility,level].map {|t| event[record][t] || -}+[count]).join('.')`, is a Ruby expression, and the name will be dynamically set based on event data.  In this case, all events are expected to have `tag`, `facility`, and `level` keys in the record.  When Ruby evaluates this expression, the metric name becomes `<prefix>.<record tag value>.<record facility>.<record level>`.
+
+So, even though there is only one event matching condition, two distinct metrics (a total, and a subtotal depending on `tag`, `facility`, and `level`) are incremented on every match.  The second key, `event[record].empty? ? false : event.to_s.length`, acts in a similar way, except that the increment value is not the value "1" (it is not a counter), but rather the (approximate) size of the event (in bytes).
+
+#### sum_defaults:
+```JSON
+  [
+     {
+        "-.-.-.count" : 0
+     },
+     {
+        "-.-.-.bytes" : 0
+     }
+  ]
+```
+
+The "defaults" specifications are needed if there are metrics which need to be sent with every metrics publication, even it none of the matches are met and the metric is never created during the run.  This may be more of an artifact, as this was initially done for Stackdriver, which had real issues with gaps in metrics series.

data/certs/oss@hotschedules.com.cert

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIELzCCAxGgAwIBAgIJAMOoRmNuvAZcMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYD
+VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5j
+aXNjbzEVMBMGA1UECgwMSG90c2NoZWR1bGVzMQ4wDAYDVQQLDAVJbmZyYTEfMB0G
+A1UEAwwWaW5mcmEuaG90c2NoZWR1bGVzLmNvbTEjMCEGCSqGSIb3DQEJARYUb3Nz
+QGhvdHNjaGVkdWxlcy5jb20wHhcNMTcwMjE4MDA1NDU4WhcNMjEwMjE4MDA1NDU4
+WjCBpzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM
+DVNhbiBGcmFuY2lzY28xFTATBgNVBAoMDEhvdHNjaGVkdWxlczEOMAwGA1UECwwF
+SW5mcmExHzAdBgNVBAMMFmluZnJhLmhvdHNjaGVkdWxlcy5jb20xIzAhBgkqhkiG
+9w0BCQEWFG9zc0Bob3RzY2hlZHVsZXMuY29tMIIBKDANBgkqhkiG9w0BAQEFAAOC
+ARUAMIIBEAKCAQcAutTytHjHFb5yK/29POqMS3DyxRSRRbKvBus/rEL3reeKDVlf
+eO9a/9U3oUcxHFHeISF9uOBSvL88svxhLqxkKLVA0vMbVgDFoGsOlh93nA5Lmw+H
+SEzG3+Z6JqH8YfAIqVyp1nZcdtC7u8xYpDD65ayjRALLuvOUXORGeqgedlgiwwLp
+2tZ7A9tCzBZnDjcNenD8zXLCOdRjJiBPq/XQS9h4POWvpom0z+jyUkRj3ojqOoXW
+bKKdXzdZ3s5BSwgxlWJo+lK/50+xLpjJe4mEJwySkrluy8GOG/UR/j+Y4I1j1FbY
+RwT9SUSairjh0zgiDC0LTjAJBIenuxxuM9jhRyoiu+YYVwIDAQABo1AwTjAdBgNV
+HQ4EFgQUtlFsvLIJ59o2GyoRTXKEIPEgnn8wHwYDVR0jBBgwFoAUtlFsvLIJ59o2
+GyoRTXKEIPEgnn8wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQcAdELJ
+QdrR55Xak3q5IhLnIdOLJC6ZEdHI2173SSmytInyDX6fe9uZION4I9WCCm/hCp+t
+QILh4feu14iDWmBxzy6F6tiyuEjhaU5Ud9wZ/YDBQhP2AHquoPAufxXZAUFlA5er
+wIYVupToBrWavfbjgRqRHAZ6vwrLwoIiomh2/wv06HLrFmSzj5q6BVp7FFMKDail
+QWzJ4aQl786RMwPa776eVaKQTmQuTUUppvhVq9QdKyxwGPibAYGbYKZwmmtx+BV2
+GJevNV8BSuhh6m2AHCxEzMaD7WKNbewTTgGLUlvppaR9SE2V/yUZ7x1Vp8/mGeO6
+2Zk648Ep9HVPKmwoVuB75+xEQw==
+-----END CERTIFICATE-----

data/fluent-plugin-buffered-metrics.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-buffered-metrics'
+  gem.version       = ENV.key?('RUBYGEM_VERSION') ? ENV['RUBYGEM_VERSION'] : '0.0.1'
+  gem.authors       = ['Alex Yamauchi']
+  gem.email         = ['oss@hotschedules.com']
+  gem.homepage      = 'https://github.com/hotschedules/fluent-plugin-buffered-metrics'
+  gem.description   = %q{Fluentd plugin derive metrics from log buffer chunks and submit to various metrics backends}
+  gem.summary   = %q{Fluentd plugin derive metrics from log buffer chunks and submit to various metrics backends at intervals determined by the buffer flushing frequency.}
+  gem.homepage      = 'https://github.com/hotschedules/fluent-plugin-buffered-metrics'
+  gem.license       = 'Apache-2.0'
+  gem.add_runtime_dependency 'fluentd', '>= 0.10.0'
+  gem.files         = `git ls-files`.split("\n")
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+  gem.signing_key   = File.expand_path( ENV.key?('RUBYGEM_SIGNING_KEY') ? ENV['RUBYGEM_SIGNING_KEY'] : '~/certs/oss@hotschedules.com.key' ) if $0 =~ /\bgem[\.0-9]*\z/
+  gem.cert_chain    = %w[certs/oss@hotschedules.com.cert]
+end

data/lib/fluent/metrics_backends.rb

@@ -0,0 +1,201 @@
+module Fluent
+
+  class SocketLike
+    # Just a little something to try to make any possible output
+    # bakced act like a socket so we don't have to check the type
+    # and change method colls throughout the code.
+
+    def open
+      if @parameters.nil? or @parameters.empty?
+        raise RuntimeError, "SocketLike open called with no parameters set"
+      else
+        if @parameters['proto'] == 'tcp'
+          @socket = TCPSocket.new(
+            @parameters['host'],
+            @parameters['port']
+          )
+        elsif @parameters['proto'] == 'udp'
+          @socket = UDPSocket.new(
+            @parameters['host'],
+            @parameters['port']
+          )
+        elsif @parameters['proto'] == 'unix'
+          @socket = UNIXsocket.new(@parameters['path'])
+        elsif @parameters['proto'] == 'file'
+          @socket = File.new(@parameters['path'],'a')
+        elsif @parameters['proto'] =~ /^http/
+          @socket = Net::HTTP.new(@parameters['host'],@parameters['port'])
+          @socket.use_ssl = @parameters['proto'] == 'https'
+        else
+          raise ArgumentError, 'SocketLike class does not support protocol' + @parameters['proto']
+        end
+      end
+    end
+
+    def write(string)
+      if @parameters['proto'] =~ /^http/
+        req = Net::HTTP::Post.new(@parameters['path'])
+        req.body = string
+        @parameters['headers'].each do |h|
+          req.add_field(h[0],h[1])
+        end
+        #@socket.request(req) or raise IOError "Error writing to backend"
+        @socket.request(req)
+      else
+        #@socket.write(string) or raise IOError "Error writing to backend"
+        @socket.write(string)
+      end
+    end
+
+    def close
+      if @parameters['proto'] =~ /^http/
+        @socket.finish
+      else
+        @socket.close
+      end
+    end
+
+    def initialize(parameters)
+      @parameters = parameters
+      # Is this actually needed here?
+      @socket = nil
+      if @parameters['proto'] =~ /^http/
+        require 'net/http'
+        require 'net/https' if @paramters['proto'] == 'https'
+      end
+    end
+
+  end
+
+  class MetricsBackend
+
+    def get_connection_parameters_defaults
+      # This should be overriden in any any subclass.
+      return {}
+    end
+
+    def set_connection_parameters(url,headers = [])
+      @connection_parameters.merge!(get_connection_parameters_defaults)
+      @connection_parameters.merge!(
+        Hash[['proto','host','port','path'].zip(
+          url.match(/^([^:]*):\/\/([^:\/]*):?(\d*)(\/.*)?/).to_a.map {|e| e.nil? or e.empty? ? nil : e }[1..-1]
+        )]
+      ) unless url.nil? or url.empty?
+      @connection_parameters['headers'] ||= []
+      @connection_parameters['headers'] += headers
+    end
+
+    def get_connection_parameters
+      @connection_parameters
+    end
+
+    def initialize(url = nil,headers = [])
+      @output_buffer = []
+      @connection_parameters = {}
+      set_connection_parameters(url, headers)
+      @connection = SocketLike.new(@connection_parameters)
+    end
+
+    def buffer_dump
+      # Allw this to be overridable to facitatte formats such as
+      # multiliine formats, or things JSON where there are
+      # punctuation differences depending upon position.
+      @output_buffer.join("\n") + "\n"
+    end
+
+    def buffer_flush
+
+      begin
+        @connection.open
+        @connection.write(buffer_dump)
+      rescue
+        raise
+      ensure
+        @connection.close
+      end
+
+      @output_buffer = []
+    end
+
+    def buffer?
+      not @output_buffer.empty?
+    end
+
+    def serialize_entry(entry,time)
+      raise NoMethodError, 'The serialize_entry method has not been specified.'
+    end
+
+    def serialize_array(data)
+      return if data.empty?
+      if data[0].is_a?(Hash)
+        return serialize_array_of_hashes(data)
+      elsif data[0].is_a?(Array)
+        return serialize_array_of_arrays(data)
+      else
+        raise ArgumentError, 'serialize_array method input must be of Hash or Array type'
+      end
+    end
+
+    def serialize(data)
+      if data.is_a?(Hash)
+        return serialize_hash(data)
+      elsif data.is_a(Array)
+        return serialize_array(data)
+      else
+        raise ArgumentError, 'serialize method input must be of Hash or Array type'
+      end
+    end
+
+    def buffer_append_array_of_values(data)
+      @output_buffer += data.map {|e| serialize_entry(e[0],e[1]) }
+    end
+
+    def buffer_append_array_of_hashes(data)
+      @output_buffer += data.map {|e| serialize_entry(e,e['time']) }
+    end
+
+    def buffer_append(data)
+      @output_buffer << serialize(data)
+    end
+
+    def buffer_append_entry(entry,time)
+      @output_buffer << serialize_entry(entry,time)
+    end
+
+  end
+
+  class MetricsBackendGraphite < MetricsBackend
+
+    def get_connection_parameters_defaults
+      return { 'proto' => 'tcp', 'host' => 'localhost', 'port' => 2003 }
+    end
+
+    def serialize_entry(entry,time)
+      return sprintf(
+        '%s%s %s %i',
+        entry.key?('prefix') ? entry['prefix'] + '.' : '',
+        entry['name'],entry['value'].to_s,
+        time.to_i
+      )
+    end
+
+  end
+
+  class MetricsBackendStatsd < MetricsBackend
+
+    def get_connection_parameters_defaults
+      return { 'proto' => 'udp', 'host' => 'localhost', 'port' => 8215 }
+    end
+
+    def serialize_entry(entry,time)
+      return sprintf(
+        '%s%s:%s|c',
+        entry.key?('prefix') ? entry['prefix'] + '.' : '',
+        entry['name'],
+        entry['value'].to_i
+      )
+    end
+
+  end
+
+end

data/lib/fluent/plugin/out_buffered_metrics.rb

@@ -0,0 +1,176 @@
+module Fluent
+  class BufferedMetricsOutput < BufferedOutput
+
+    Plugin.register_output('buffered_metrics', self)
+
+    unless method_defined?(:log)
+      define_method('log') { $log }
+    end
+
+    config_param :metrics_backend, :string, :default => 'graphite'
+    config_param :url, :string, :default => nil
+    config_param :http_headers, :array, :default => []
+    config_param :prefix, :string, :default => nil
+    config_param :instance_id, :string, :default => nil
+    config_param :sum_maps, :hash, :default => {}
+    config_param :sum_defaults, :array, :default => []
+    config_param :metric_maps, :hash, :default => {}
+    config_param :metric_defaults, :array, :default => []
+
+    # The following are overrides for the paramaters inherited from
+    # the superclass to them more sensible defaults. Since this is
+    # likely to be run farily frequently, don't allow for long waits.
+    config_param :retry_limit, :integer, :default => 4
+    config_param :retry_wait, :time, :default => 1.0
+    config_param :max_retry_wait, :time, :default => 5.0
+
+    def initialize
+      super
+      require 'fluent/metrics_backends'
+    end
+
+    def configure(conf)
+      super(conf) {
+        @url = conf.delete('url')
+        @http_headers = conf.delete('http_headers')
+        @metrics_backend = conf.delete('metrics_backend')
+        @prefix = conf.delete('prefix')
+        @sum_maps = conf.delete('sum_maps')
+        @sum_defaults = conf.delete('sum_defaults')
+        @metric_maps = conf.delete('metric_maps')
+        @metric_defaults = conf.delete('metric_defaults')
+      }
+
+      @base_entry = {}
+
+      unless @prefix.nil?
+
+        begin
+          @prefix = eval(@prefix) || eval('"'+@prefix+'"')
+        rescue
+          raise ArgumentError, "Error setting prefix from '#{@prefix}'"
+        end
+
+        @base_entry['prefix'] = @prefix unless @prefix.empty?
+
+      end
+
+      @sum_maps.each do |k,v|
+        @sum_maps[k] = [ v ] unless v.is_a?(Array)
+      end
+
+      @metric_maps.each do |k,v|
+        @metric_maps[k] = [ v ] unless v.is_a?(Array)
+      end
+
+      begin
+        backend_name = @metrics_backend
+        @metrics_backend = Object.const_get(
+          sprintf('Fluent::MetricsBackend%s',backend_name.capitalize)
+        ).new(@url,@http_headers)
+
+      rescue => e
+        log.error "Error initializing metrics backend #{backend_name}"
+        raise e
+      end
+
+    end
+
+    def format(tag, time, record)
+      { 'tag' => tag, 'time' => time, 'record' => record }.to_msgpack
+    end
+
+    def derive_metrics(chunk)
+      # The default timestamp really needs to set from the chunk
+      # metadata.  However, there is no way to do this prior to
+      # v0.14.  Putting the the VERSION conditional settitng, but
+      # I don't really have a way to test it, at the moment.
+
+      if chunk.methods.include?(/metadata/)
+        # This should work for v0.14 and above and is preferable.
+        timestamp = chunk.metadata.timekey.to_f
+      else
+        timestamp = Time.now.to_f
+      end
+
+      sum_data = {}
+      metric_data = {}
+
+      chunk.msgpack_each do |event|
+        @sum_maps.each do |k,v|
+
+          begin
+            incr = eval(k)
+            if incr.is_a?(Numeric)
+              v.each do |e|
+                begin
+                  name = eval(e)
+                rescue Exception
+                  name = eval('"'+e+'"')
+                end
+                sum_data[name] ||= 0
+                sum_data[name] += incr
+              end
+            end
+          rescue Exception => e
+            log.error "Failed to process sum_map (#{k},#{v}) for event: #{event}: #{e.trace}"
+          end
+
+        end
+
+        @metric_maps.each do |k,v|
+
+          begin
+            if eval(k)
+              v.each do |e|
+                val = eval(e)
+                if val.is_a?(Hash) and not val.empty?
+                  @metrics_backend.buffer_append_entry(
+                    @base_entry.merge(val),
+                    event['time']
+                  )
+                end
+              end
+            end
+          rescue Exception => e
+            log.error "Failed to process metric_map (#{k},#{v}) for event: #{event}: #{e.trace}"
+          end
+
+        end
+
+      end
+
+      @sum_defaults.each do |e|
+        if e.key?('name') and not e['name'].nil? and not e['name'].empty?
+          sum_data[e['name']] = e['value'] unless sum_data.key?(e['name'])
+        end
+      end
+
+      @metrics_backend.buffer_append_array_of_hashes(
+        sum_data.map {|name,value|
+          @base_entry.merge({ 'name' => name, 'value' => value, 'time' => timestamp })
+        }
+      )
+
+      @metrics_backend.buffer_append_array_of_hashes(
+         @metric_defaults.map {|e|
+           if e.key?('name') and not e['name'].nil? and not e['name'].empty?
+             @base_entry.merge(e).merge({'time' => timestamp}) unless metric_data.key?(e['name'])
+          end
+         }
+      )
+
+    end
+
+    def write(chunk)
+
+      # The superclass BufferedOutput provides the retry logic.  If the
+      # buffer already has content this must be a retry after a failed
+      # flush, so don't re-scan the chunk for the metrics.
+      derive_metrics(chunk) unless @metrics_backend.buffer?
+      @metrics_backend.buffer_flush if @metrics_backend.buffer?
+
+    end
+
+  end
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification 
+name: fluent-plugin-buffered-metrics
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Alex Yamauchi
+autorequire: 
+bindir: bin
+cert_chain: 
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIELzCCAxGgAwIBAgIJAMOoRmNuvAZcMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYD
+  VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5j
+  aXNjbzEVMBMGA1UECgwMSG90c2NoZWR1bGVzMQ4wDAYDVQQLDAVJbmZyYTEfMB0G
+  A1UEAwwWaW5mcmEuaG90c2NoZWR1bGVzLmNvbTEjMCEGCSqGSIb3DQEJARYUb3Nz
+  QGhvdHNjaGVkdWxlcy5jb20wHhcNMTcwMjE4MDA1NDU4WhcNMjEwMjE4MDA1NDU4
+  WjCBpzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM
+  DVNhbiBGcmFuY2lzY28xFTATBgNVBAoMDEhvdHNjaGVkdWxlczEOMAwGA1UECwwF
+  SW5mcmExHzAdBgNVBAMMFmluZnJhLmhvdHNjaGVkdWxlcy5jb20xIzAhBgkqhkiG
+  9w0BCQEWFG9zc0Bob3RzY2hlZHVsZXMuY29tMIIBKDANBgkqhkiG9w0BAQEFAAOC
+  ARUAMIIBEAKCAQcAutTytHjHFb5yK/29POqMS3DyxRSRRbKvBus/rEL3reeKDVlf
+  eO9a/9U3oUcxHFHeISF9uOBSvL88svxhLqxkKLVA0vMbVgDFoGsOlh93nA5Lmw+H
+  SEzG3+Z6JqH8YfAIqVyp1nZcdtC7u8xYpDD65ayjRALLuvOUXORGeqgedlgiwwLp
+  2tZ7A9tCzBZnDjcNenD8zXLCOdRjJiBPq/XQS9h4POWvpom0z+jyUkRj3ojqOoXW
+  bKKdXzdZ3s5BSwgxlWJo+lK/50+xLpjJe4mEJwySkrluy8GOG/UR/j+Y4I1j1FbY
+  RwT9SUSairjh0zgiDC0LTjAJBIenuxxuM9jhRyoiu+YYVwIDAQABo1AwTjAdBgNV
+  HQ4EFgQUtlFsvLIJ59o2GyoRTXKEIPEgnn8wHwYDVR0jBBgwFoAUtlFsvLIJ59o2
+  GyoRTXKEIPEgnn8wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQcAdELJ
+  QdrR55Xak3q5IhLnIdOLJC6ZEdHI2173SSmytInyDX6fe9uZION4I9WCCm/hCp+t
+  QILh4feu14iDWmBxzy6F6tiyuEjhaU5Ud9wZ/YDBQhP2AHquoPAufxXZAUFlA5er
+  wIYVupToBrWavfbjgRqRHAZ6vwrLwoIiomh2/wv06HLrFmSzj5q6BVp7FFMKDail
+  QWzJ4aQl786RMwPa776eVaKQTmQuTUUppvhVq9QdKyxwGPibAYGbYKZwmmtx+BV2
+  GJevNV8BSuhh6m2AHCxEzMaD7WKNbewTTgGLUlvppaR9SE2V/yUZ7x1Vp8/mGeO6
+  2Zk648Ep9HVPKmwoVuB75+xEQw==
+  -----END CERTIFICATE-----
+
+date: 2018-04-05 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fluentd
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 0
+        - 10
+        - 0
+        version: 0.10.0
+  type: :runtime
+  version_requirements: *id001
+description: Fluentd plugin derive metrics from log buffer chunks and submit to various metrics backends
+email: 
+- oss@hotschedules.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- LICENSE
+- LICENSE-Apache-2.0.txt
+- README.md
+- certs/oss@hotschedules.com.cert
+- fluent-plugin-buffered-metrics.gemspec
+- lib/fluent/metrics_backends.rb
+- lib/fluent/plugin/out_buffered_metrics.rb
+homepage: https://github.com/hotschedules/fluent-plugin-buffered-metrics
+licenses: 
+- Apache-2.0
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Fluentd plugin derive metrics from log buffer chunks and submit to various metrics backends at intervals determined by the buffer flushing frequency.
+test_files: []
+