ffwd-kairosdb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4ca0c918e4d9ca750323bf9c6548a36f4ad439aa
+  data.tar.gz: a92c004e3588e43d1359d8eab17245bd2d51f1a0
+SHA512:
+  metadata.gz: 28eeea4df3f61cdbbb27cabeceb378d6434676395d01ad213bd7163661da80958fb001047fdca2f6aef4c07c2f55415bcd33b82df82596e0e3009c44229a7037
+  data.tar.gz: fec8b9af3382cfbfd135dfb835581b582572c70ce36fe34635da054d99586f813e26c32102dc81090d240f8e7557f135133118e8396a8893c86157a830201b82

data/lib/ffwd/plugin/kairosdb/output.rb

@@ -0,0 +1,130 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/reporter'
+
+require_relative 'utils'
+
+module FFWD::Plugin::KairosDB
+  class Output
+    include FFWD::Reporter
+    setup_reporter :keys => [:dropped_metrics, :sent_metrics, :failed_metrics]
+
+    attr_reader :log, :reporter_meta
+
+    HEADER = {
+      "Content-Type" => "application/json"
+    }
+
+    API_PATH = "/api/v1/datapoints"
+
+    def initialize core, log, url, flush_interval, buffer_limit
+      @log = log
+      @url = url
+      @flush_interval = flush_interval
+      @buffer_limit = buffer_limit
+      @reporter_meta = {:type => "kairosdb_out", :url => @url}
+
+      @buffer = []
+      @pending = nil
+      @c = nil
+
+      @sub = nil
+
+      core.starting do
+        log.info "Will send events to #{@url}"
+
+        @c = EM::HttpRequest.new(@url)
+
+        @sub = core.output.metric_subscribe do |metric|
+          if @buffer.size >= @buffer_limit
+            increment :dropped_metrics, 1
+            next
+          end
+
+          @buffer << metric
+          check_timer!
+        end
+      end
+
+      core.stopping do
+        # Close is buggy, don.
+        #@c.close
+
+        log.info "Closing connection to #{@url}"
+
+        if @sub
+          @sub.unsubscribe
+          @sub = nil
+        end
+
+        if @timer
+          @timer.cancel
+          @timer = nil
+        end
+      end
+    end
+
+    def flush!
+      if @timer
+        @timer.cancel
+        @timer = nil
+      end
+
+      if @pending
+        log.info "Request already in progress, dropping metrics"
+        increment :dropped_metrics, @buffer.size
+        @buffer.clear
+        return
+      end
+
+      unless @c
+        log.error "Dropping metrics, no active connection available"
+        increment :dropped_metrics, @buffer.size
+        @buffer.clear
+        return
+      end
+
+      buffer_size = @buffer.size
+      metrics = Utils.make_metrics(@buffer)
+      metrics = JSON.dump(metrics)
+      @buffer.clear
+
+      log.info "Sending #{buffer_size} metric(s) to #{@url}"
+      @pending = @c.post(:path => API_PATH, :head => HEADER, :body => metrics)
+
+      @pending.callback do
+        increment :sent_metrics, buffer_size
+        @pending = nil
+      end
+
+      @pending.errback do
+        log.error "Failed to submit events: #{@pending.error}"
+        increment :failed_metrics, buffer_size
+        @pending = nil
+      end
+    end
+
+    def check_timer!
+      return if @timer
+
+      log.debug "Setting timer to #{@flush_interval}s"
+
+      @timer = EM::Timer.new(@flush_interval) do
+        flush!
+      end
+    end
+  end
+end

data/lib/ffwd/plugin/kairosdb/utils.rb

@@ -0,0 +1,64 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD::Plugin::KairosDB
+  module Utils
+    # groups similar metadata and escapes them using the suite of safe_*
+    # functions available.
+    #
+    # Should prevent unecessary invocations of safe_entry by only adding new
+    # groups of the source metric differs (||=).
+    def self.make_metrics buffer
+      groups = {}
+
+      buffer.each do |m|
+        entry = {:host => m.host, :name => m.key, :attributes => m.attributes}
+        group = (groups[entry] ||= safe_entry(entry).merge(:datapoints => []))
+        group[:datapoints] << [(m.time.to_f * 1000).to_i, m.value]
+      end
+
+      return groups.values
+    end
+
+    # make safe entry out of available information.
+    def self.safe_entry entry
+      name = entry[:name]
+      host = entry[:host]
+      attributes = entry[:attributes]
+      {:name => safe_string(name), :tags => safe_tags(host, attributes)}
+    end
+
+    # Warning: These are the 'bad' characters I've been able to reverse
+    # engineer so far.
+    def self.safe_string string
+      string = string.to_s
+      string = string.gsub " ", "/"
+      string.gsub ":", "_"
+    end
+
+    # Warning: KairosDB ignores complete metrics if you use tags which have no
+    # values, therefore I have not figured out a way to transport 'tags'.
+    def self.safe_tags host, attributes
+      tags = {"host" => safe_string(host)}
+
+      attributes.each do |key, value|
+        tags[safe_string(key)] = safe_string(value)
+      end
+
+      return tags
+    end
+
+  end
+end

data/lib/ffwd/plugin/kairosdb/version.rb

@@ -0,0 +1,22 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  module Plugin
+    module KairosDB
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/ffwd/plugin/kairosdb.rb

@@ -0,0 +1,41 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'eventmachine'
+require 'em-http'
+
+require 'ffwd/logging'
+require 'ffwd/plugin'
+require 'ffwd/reporter'
+
+require_relative 'kairosdb/output'
+
+module FFWD::Plugin::KairosDB
+  include FFWD::Plugin
+  include FFWD::Logging
+
+  register_plugin "kairosdb"
+
+  DEFAULT_URL = "http://localhost:8080"
+  DEFAULT_FLUSH_INTERVAL = 10
+  DEFAULT_BUFFER_LIMIT = 100000
+
+  def self.setup_output opts, core
+    url = opts[:url] || DEFAULT_URL
+    flush_interval = opts[:flush_interval] || DEFAULT_FLUSH_INTERVAL
+    buffer_limit = opts[:buffer_limit] || DEFAULT_BUFFER_LIMIT
+    Output.new core, log, url, flush_interval, buffer_limit
+  end
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: ffwd-kairosdb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- John-John Tedro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffwd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- udoprog@spotify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/ffwd/plugin/kairosdb.rb
+- lib/ffwd/plugin/kairosdb/output.rb
+- lib/ffwd/plugin/kairosdb/utils.rb
+- lib/ffwd/plugin/kairosdb/version.rb
+homepage: https://github.com/spotify/ffwd
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: KairosDB support for FFWD.
+test_files: []