ffwd-collectd 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 56020d48d93dc000931a007ace619ad887135b62
+  data.tar.gz: 62204ed7e4b07dd2caec921729cce2b9cfdafdf2
+SHA512:
+  metadata.gz: 227f6d4c8a2e86d72f694b2460296108cd171d037f82974a0c5ec721006b2201eeecd84e7bdc443e43e4f06efcf59350b2725813104ce6f5d2af670904a271f8
+  data.tar.gz: 5f881f021cc072c3a200e1e9240d1bef04fea22a874c465d97e149d3eff1fbd62fa0b60e2aab00a0bc1565ba2bb628d575b030ace96418a5bac2f4653332b124

data/lib/ffwd/plugin/collectd/connection.rb

@@ -0,0 +1,83 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/logging'
+require 'ffwd/connection'
+
+require_relative 'parser'
+
+module FFWD::Plugin::Collectd
+  class Connection < FFWD::Connection
+    include FFWD::Logging
+
+    def self.plugin_type
+      "collectd_in"
+    end
+
+    def initialize bind, core, types_db
+      @bind = bind
+      @core = core
+      @types_db = types_db
+    end
+
+    def receive_data(data)
+      Parser.parse(data) do |metric|
+        plugin_key = metric[:plugin]
+        type_key = metric[:type]
+
+        if instance = metric[:plugin_instance] and not instance.empty?
+          plugin_key = "#{plugin_key}-#{instance}"
+        end
+
+        if instance = metric[:type_instance] and not instance.empty?
+          type_key = "#{type_key}-#{instance}"
+        end
+
+        key = "#{plugin_key}/#{type_key}"
+
+        values = metric[:values]
+
+        time = metric[:time]
+        host = metric[:host]
+
+        # Just add a running integer to the end of the key, the 'correct'
+        # solution would have been to read, parse and match from a types.db.
+        #
+        # http://collectd.org/documentation/manpages/types.db.5.shtml
+        if values.size > 1
+          values.each_with_index do |v, i|
+            if @types_db and name = @types_db.get_name(type_key, i)
+              index_key = name
+            else
+              index_key = i.to_s
+            end
+
+            @core.input.metric(
+              :key => "#{key}_#{index_key}", :time => time, :value => v[1],
+              :host => host)
+            @bind.increment :received_metrics
+          end
+        else
+          v = values[0]
+          @core.input.metric(
+            :key => key, :time => time, :value => v[1], :host => host)
+          @bind.increment :received_metrics
+        end
+      end
+    rescue => e
+      log.error "Failed to receive data", e
+    end
+  end
+end

data/lib/ffwd/plugin/collectd/parser.rb

@@ -0,0 +1,159 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/logging'
+
+module FFWD::Plugin::Collectd
+  # An parser implementation of
+  # https://collectd.org/wiki/index.php/Binary_protocol
+  module Parser
+    include FFWD::Logging
+
+    HOST = 0x0000
+    TIME = 0x0001
+    TIME_HR = 0x0008
+    PLUGIN = 0x0002
+    PLUGIN_INSTANCE = 0x0003
+    TYPE = 0x0004
+    TYPE_INSTANCE = 0x0005
+    VALUES = 0x0006
+    INTERVAL = 0x0007
+    INTERVAL_HR = 0x0009
+    MESSAGE = 0x0100
+    SEVERITY = 0x0101
+
+    COUNTER = 0
+    GAUGE = 1
+    DERIVE = 2
+    ABSOLUTE = 3
+
+    SB64 = 0x8000000000000000
+    B64 = 0x10000000000000000
+
+    FACTOR_HR = 2**30
+
+    def self.unsigned_integer data
+      high, low = data.unpack("NN")
+      ((high << 32) | low)
+    end
+
+    def self.signed_integer data
+      us = unsigned_integer(data)
+
+      if (us & SB64) == SB64
+        us - B64
+      else
+        us
+      end
+    end
+
+    def self.values frame, i, size
+      n = frame[i + 4, 2].unpack("n")[0]
+
+      result = []
+
+      types = frame[i + 6, n].unpack("C" * n)
+
+      types.each_with_index do |type, j|
+        o = 6 + n + (j * 8)
+        data = frame[i + o, 8]
+
+        case type
+        when COUNTER
+          result << [:counter, self.unsigned_integer(data)]
+        when GAUGE
+          result << [:gauge, data.unpack("E")[0]]
+        when DERIVE
+          result << [:derive, self.signed_integer(data)]
+        when ABSOLUTE
+          result << [:absolute, self.unsigned_integer(data)]
+        else
+          raise "unkonwn value type: #{type}"
+        end
+
+        j += 1
+      end
+
+      result
+    end
+
+    def self.string frame, i, size
+      frame[4 + i, (size - 5)]
+    end
+
+    def self.numeric frame, i, size
+      unsigned_integer frame[4 + i, 8]
+    end
+
+    def self.time_high_res frame, i, size
+      Time.at(numeric(frame, i, size).to_f / FACTOR_HR)
+    end
+
+    def self.interval_high_res frame, i, size
+      numeric(frame, i, size).to_f / FACTOR_HR
+    end
+
+    # Maintain a current frame, and yield a copy of it to the subscribing
+    # block.
+    #
+    # Reading a 'values' part is the indicator that a block is 'ready'.
+    def self.parse frame
+      raise "invalid frame" if frame.size < 4
+
+      current = {}
+
+      i = 0
+
+      loop do
+        break if i >= frame.size
+
+        type, size = frame[i,4].unpack("nn")
+
+        case type
+        when HOST
+          current[:host] = self.string(frame, i, size)
+        when TIME
+          current[:time] = Time.at(self.numeric(frame, i, size))
+        when TIME_HR
+          current[:time] = self.time_high_res(frame, i, size)
+        when PLUGIN
+          current[:plugin] = self.string(frame, i, size)
+        when PLUGIN_INSTANCE
+          current[:plugin_instance] = self.string(frame, i, size)
+        when TYPE
+          current[:type] = self.string(frame, i, size)
+        when TYPE_INSTANCE
+          current[:type_instance] = self.string(frame, i, size)
+        when VALUES
+          values = self.values(frame, i, size)
+          current[:values] = values
+          yield current
+        when INTERVAL
+          current[:interval] = self.numeric(frame, i, size).to_f
+        when INTERVAL_HR
+          current[:interval] = self.interval_high_res(frame, i, size)
+        when MESSAGE
+          current[:message] = self.string(frame, i, size)
+        when SEVERITY
+          current[:severity] = self.numeric(frame, i, size)
+        else
+          log.warning("cannot understand type: #{type}")
+        end
+
+        i += size
+      end
+    end
+  end
+end

data/lib/ffwd/plugin/collectd/types_db.rb

@@ -0,0 +1,57 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD::Plugin::Collectd
+  # A minimal implementation of a reader for collectd's types.db
+  #
+  # http://collectd.org/documentation/manpages/types.db.5.shtml
+  class TypesDB
+    def initialize database
+      @database = database
+    end
+
+    def get_name key, i
+      unless entry = @database[key]
+        return nil
+      end
+
+      unless type_spec = entry[i]
+        return nil
+      end
+
+      type_spec[0]
+    end
+
+    def self.open path
+      return nil unless File.file? path
+
+      database = {}
+
+      File.open(path) do |f|
+        f.readlines.each do |line|
+          next if line.start_with? "#"
+          parts = line.split(/[\t ]+/, 2)
+          next unless parts.size == 2
+          key, value_specs = parts
+          value_specs = value_specs.split(",").map(&:strip)
+          value_specs = value_specs.map{|s| s.split(':')}
+          database[key] = value_specs
+        end
+      end
+
+      new database
+    end
+  end
+end

data/lib/ffwd/plugin/collectd/version.rb

@@ -0,0 +1,22 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  module Plugin
+    module Collectd
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/ffwd/plugin/collectd.rb

@@ -0,0 +1,50 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/plugin'
+require 'ffwd/protocol'
+require 'ffwd/logging'
+
+require_relative 'collectd/connection'
+require_relative 'collectd/types_db'
+
+module FFWD::Plugin::Collectd
+  include FFWD::Plugin
+  include FFWD::Logging
+
+  register_plugin "collectd"
+
+  DEFAULT_HOST = "localhost"
+  DEFAULT_PORT = 25826
+  DEFAULT_TYPES_DB = "/usr/share/collectd/types.db"
+
+  def self.setup_input opts, core
+    opts[:host] ||= DEFAULT_HOST
+    opts[:port] ||= DEFAULT_PORT
+    opts[:types_db] ||= DEFAULT_TYPES_DB
+    protocol = FFWD.parse_protocol(opts[:protocol] || "udp")
+    types_db = TypesDB.open opts[:types_db]
+    protocol.bind opts, core, log, Connection, types_db
+  end
+
+  def self.setup_tunnel opts, core, tunnel
+    opts[:port] ||= DEFAULT_PORT
+    opts[:types_db] ||= DEFAULT_TYPES_DB
+    protocol = FFWD.parse_protocol(opts[:protocol] || "udp")
+    protocol.tunnel log, opts, Connection
+    types_db = TypesDB.open opts[:types_db]
+    protocol.bind opts, core, tunnel, log, Connection, types_db
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: ffwd-collectd
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- John-John Tedro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffwd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- udoprog@spotify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/ffwd/plugin/collectd/parser.rb
+- lib/ffwd/plugin/collectd/types_db.rb
+- lib/ffwd/plugin/collectd/connection.rb
+- lib/ffwd/plugin/collectd/version.rb
+- lib/ffwd/plugin/collectd.rb
+homepage: https://github.com/spotify/ffwd
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: collectd support for FFWD.
+test_files: []