ffwd 0.1.0

data/lib/ffwd/logging.rb

@@ -0,0 +1,139 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# Logging functionality.
+#
+# Defines FFWD::Logging which when included in either a class or module makes
+# the 'log' field available both as a class and instance field.
+#
+# 'log' in turn is an object with the following fields available.
+#
+# 'debug' - Log a message of level DEBUG..
+# 'info' - Log a message of level INFO.
+# 'warning' - Log a message of level WARNING.
+# 'error' - Log a message of level ERROR.
+#
+# Every function takes the message to log as the only parameter except 'error'
+# which can take an exception as a secondary argument.
+#
+# If an exception is provided, a stacktrace will be printed to log.
+require 'logger'
+
+module FFWD
+  def self.log
+    @log ||= setup_log
+  end
+
+  def self.log_reload
+    @log = setup_log
+  end
+
+  def self.setup_log
+    if log_config[:file]
+      file = log_config[:file]
+      shift_age = log_config[:shift_age]
+
+      return ::Logger.new(file, shift_age=shift_age).tap do |l|
+        l.level = log_config[:level]
+        l.progname = log_config[:progname]
+      end
+    end
+
+    if log_config[:stream]
+      return ::Logger.new(log_config[:stream]).tap do |l|
+        l.level = log_config[:level]
+        l.progname = log_config[:progname]
+      end
+    end
+
+    raise "cannot setup loggin with options: #{log_config}"
+  end
+
+  def self.log_config
+    return @log_config unless @log_config.nil?
+
+    @log_config = {
+      :file => nil,
+      :shift_age => 1,
+      :level => Logger::INFO,
+      :stream => STDOUT,
+      :progname => 'FFWD',
+    }
+  end
+
+  def self.log_disable
+    @log_disable = true
+  end
+
+  def self.log_disabled?
+    @log_disable || false
+  end
+
+  class ClassLogger
+    def initialize klass
+      @progname = klass.name
+    end
+
+    def debug message
+      FFWD.log.debug(@progname){message}
+    end
+
+    def info message
+      FFWD.log.info(@progname){message}
+    end
+
+    def warning message
+      FFWD.log.warn(@progname){message}
+    end
+
+    def error message, e=nil
+      FFWD.log.error(@progname){message}
+
+      return unless e
+
+      FFWD.log.error(@progname){"Caused by #{e.class}: #{e}"}
+      e.backtrace.each do |b|
+        FFWD.log.error(@progname){"  #{b}"}
+      end
+    end
+  end
+
+  class FakeLogger
+    def debug message; end
+    def info message; end
+    def warning message; end
+    def error message, e=nil; end
+  end
+
+  module Logging
+    module ClassMethods
+      attr_accessor :log
+    end
+
+    def log
+      self.class.log
+    end
+
+    def self.included klass
+      klass.extend ClassMethods
+
+      if FFWD.log_disabled?
+        klass.log = FakeLogger.new
+      else
+        klass.log = ClassLogger.new klass
+      end
+    end
+  end
+end

data/lib/ffwd/metric.rb

@@ -0,0 +1,55 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  # Struct used to define all fields related to a metric.
+  MetricStruct = Struct.new(
+    # The time at which the metric was collected.
+    :time,
+    # The unique key of the metric.
+    :key,
+    # A numeric value associated with the metric.
+    :value,
+    # The host from which the metric originated.
+    :host,
+    # The source metric this metric was derived from (if any).
+    :source,
+    # Tags associated to the metric.
+    :tags,
+    # Attributes (extra fields) associated to the metric.
+    :attributes
+  )
+
+  # A convenience class for each individual metric.
+  class Metric < MetricStruct
+    def self.make opts={}
+      new(opts[:time], opts[:key], opts[:value], opts[:host], opts[:source],
+          opts[:tags], opts[:attributes])
+    end
+
+    # Convert metric to a sparse hash.
+    def to_h
+      d = {}
+      d[:time] = time.to_i if time
+      d[:key] = key if key
+      d[:value] = value if value
+      d[:host] = host if host
+      d[:source] = source if source
+      d[:tags] = tags.to_a if tags
+      d[:attributes] = attributes if attributes
+      d
+    end
+  end
+end

data/lib/ffwd/metric_emitter.rb

@@ -0,0 +1,50 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'utils'
+require_relative 'metric'
+
+module FFWD
+  # Used to emit metrics to an 'output' channel
+  #
+  # Can take two parts of a configuration 'base' and 'opts' to decide which
+  # metadata emitted metrics should be decorated with.
+  class MetricEmitter
+    def self.build output, base, opts
+      host = opts[:host] || base[:host] || FFWD.current_host
+      tags = FFWD.merge_sets base[:tags], opts[:tags]
+      attributes = FFWD.merge_hashes base[:attributes], opts[:attributes]
+      new output, host, tags, attributes
+    end
+
+    def initialize output, host, tags, attributes
+      @output = output
+      @host = host
+      @tags = tags
+      @attributes = attributes
+    end
+
+    def emit m
+      m[:time] ||= Time.now
+      m[:host] ||= @host if @host
+      m[:tags] = FFWD.merge_sets @tags, m[:tags]
+      m[:attributes] = FFWD.merge_hashes @attributes, m[:attributes]
+
+      @output.metric Metric.make(m)
+    rescue => e
+      log.error "Failed to emit metric", e
+    end
+  end
+end

data/lib/ffwd/plugin.rb

@@ -0,0 +1,149 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'logging'
+
+module FFWD
+  module Plugin
+    class Loaded
+      attr_reader :source, :name
+
+      def initialize source, name, options
+        @source = source
+        @name = name
+        @mod = options[:mod]
+        @setup_input_method = load_method @mod, options[:setup_input_method_name]
+        @setup_output_method = load_method @mod, options[:setup_output_method_name]
+        @setup_tunnel_method = load_method @mod, options[:setup_tunnel_method_name]
+      end
+
+      def capabilities
+        capabilities = []
+
+        if not @setup_input_method.nil?
+          capabilities << "input"
+        end
+
+        if not @setup_output_method.nil?
+          capabilities << "output"
+        end
+
+        if not @setup_tunnel_method.nil?
+          capabilities << "tunnel"
+        end
+
+        return capabilities
+      end
+
+      def can?(kind)
+        not get(kind).nil?
+      end
+
+      def get(kind)
+        return @setup_input_method if kind == :input
+        return @setup_output_method if kind == :output
+        return @setup_tunnel_method if kind == :tunnel
+        return nil
+      end
+
+      private
+
+      def load_method mod, method_name
+        return nil unless mod.respond_to? method_name
+        return mod.method method_name
+      end
+    end
+
+    class Setup
+      attr_reader :name, :config
+
+      def initialize name, setup, config
+        @name = name
+        @setup = setup
+        @config = config
+      end
+
+      def setup *args
+        @setup.call @config, *args
+      end
+    end
+
+    def self.discovered
+      @@discovered ||= {}
+    end
+
+    def self.loaded
+      @@loaded ||= {}
+    end
+
+    module ClassMethods
+      def register_plugin(name, opts={})
+        options = {:mod => self}
+
+        options[:setup_input_method_name] = (opts[:setup_input_method] || :setup_input)
+        options[:setup_output_method_name] = (opts[:setup_output_method] || :setup_output)
+        options[:setup_tunnel_method_name] = (opts[:setup_tunnel_method] || :setup_tunnel)
+
+        FFWD::Plugin.discovered[name] = options
+      end
+    end
+
+    def self.included mod
+      mod.extend ClassMethods
+    end
+
+    def self.category
+      'plugin'
+    end
+
+    def self.load_discovered source
+      FFWD::Plugin.discovered.each do |name, options|
+        FFWD::Plugin.loaded[name] = Loaded.new source, name, options
+      end
+
+      FFWD::Plugin.discovered.clear
+    end
+
+    def self.load_plugins log, kind_name, config, kind
+      result = []
+
+      if config.nil?
+        return result
+      end
+
+      config.each_with_index do |plugin_config, index|
+        d = "#{kind_name} plugin ##{index}"
+
+        if (name = plugin_config[:type]).nil?
+          log.error "#{d}: Missing :type attribute for '#{kind_name}'"
+        end
+
+        if (plugin = FFWD::Plugin.loaded[name]).nil?
+          log.error "#{d}: Not an available plugin '#{name}'"
+          next
+        end
+
+        unless plugin.can?(kind)
+          log.error "#{d}: Not an #{kind_name} plugin '#{name}'"
+          next
+        end
+
+        result << Setup.new(name, plugin.get(kind), plugin_config)
+      end
+
+      return result
+    end
+  end
+end

data/lib/ffwd/plugin/json_line.rb

@@ -0,0 +1,47 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'eventmachine'
+
+require 'ffwd/protocol'
+require 'ffwd/plugin'
+require 'ffwd/logging'
+
+require_relative 'json_line/connection'
+
+module FFWD::Plugin::JsonLine
+  include FFWD::Plugin
+  include FFWD::Logging
+
+  register_plugin "json_line"
+
+  DEFAULT_HOST = "localhost"
+  DEFAULT_PORT = 19000
+
+  def self.setup_input opts, core
+    opts[:host] ||= DEFAULT_HOST
+    opts[:port] ||= DEFAULT_PORT
+    buffer_limit = opts["buffer_limit"] || 1000
+    protocol = FFWD.parse_protocol(opts[:protocol] || "tcp")
+    protocol.bind opts, core, log, Connection, buffer_limit
+  end
+
+  def self.setup_tunnel opts, core, tunnel
+    opts[:port] ||= DEFAULT_PORT
+    buffer_limit = opts["buffer_limit"] || 1000
+    protocol = FFWD.parse_protocol(opts[:protocol] || "tcp")
+    protocol.tunnel opts, core, tunnel, log, Connection, buffer_limit
+  end
+end

data/lib/ffwd/plugin/json_line/connection.rb

@@ -0,0 +1,118 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'eventmachine'
+
+require 'ffwd/logging'
+require 'ffwd/connection'
+
+module FFWD::Plugin::JsonLine
+  class Connection < FFWD::Connection
+    include FFWD::Logging
+    include EM::Protocols::LineText2
+
+    EVENT_FIELDS = [
+      ["key", :key],
+      ["value", :value],
+      ["host", :host],
+      ["state", :state],
+      ["description", :description],
+      ["ttl", :ttl],
+      ["tags", :tags],
+      ["attributes", :attributes],
+    ]
+
+    METRIC_FIELDS = [
+      ["proc", :proc],
+      ["key", :key],
+      ["value", :value],
+      ["tags", :tags],
+      ["attributes", :attributes]
+    ]
+
+    def self.plugin_type
+      "json_line_in"
+    end
+
+    def initialize bind, core, buffer_limit
+      @bind = bind
+      @core = core
+      @buffer_limit = buffer_limit
+    end
+
+    def receive_line data
+      data = JSON.load(data)
+
+      unless type = data["type"]
+        log.error "Field 'type' missing from received line"
+        return
+      end
+
+      if type == "metric"
+        @core.input.metric read_metric(data)
+        @bind.increment :received_metric
+        return
+      end
+
+      if type == "event"
+        @core.input.event read_event(data)
+        @bind.increment :received_event
+        return
+      end
+
+      log.error "No such type: #{type}"
+    rescue => e
+      log.error "Failed to receive line", e
+    end
+
+    def read_tags d, source
+      return if (tags = d["tags"]).nil?
+      d[:tags] = tags.to_set
+    end
+
+    def read_time d, source
+      return if (time = d["time"]).nil?
+      d[:time] = Time.at time
+    end
+
+    def read_metric data
+      d = {}
+
+      read_tags d, data["tags"]
+      read_time d, data["time"]
+
+      METRIC_FIELDS.each do |from, to|
+        next if (v = data[from]).nil?
+        d[to] = v
+      end
+
+      d
+    end
+
+    def read_event data
+      d = {}
+
+      read_tags d, data["tags"]
+      read_time d, data["time"]
+
+      EVENT_FIELDS.each do |from, to|
+        next if (v = data[from]).nil?
+        d[to] = v
+      end
+
+      d
+    end
+  end
+end