ffwd 0.1.0

data/lib/ffwd/connection.rb

@@ -0,0 +1,40 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'eventmachine'
+
+module FFWD
+  # Connections are used by input plugins in the protocol stack.
+  #
+  # The sole purpose this exists is to incorporate a datasink functionality
+  # in the EM::Connection.
+  #
+  # The datasink is used by tunnels to 'hook into' outgoing data.
+  class Connection < EM::Connection
+    def datasink= sink
+      @datasink = sink
+    end
+
+    # send_data indirection.
+    def send_data data
+      if @datasink
+        @datasink.send_data data
+        return
+      end
+
+      super data
+    end
+  end
+end

data/lib/ffwd/core.rb

@@ -0,0 +1,173 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'json'
+require 'eventmachine'
+
+require_relative 'channel'
+require_relative 'debug'
+require_relative 'lifecycle'
+require_relative 'logging'
+require_relative 'plugin_channel'
+require_relative 'processor'
+require_relative 'protocol'
+require_relative 'statistics'
+require_relative 'utils'
+
+require_relative 'core/emitter'
+require_relative 'core/interface'
+require_relative 'core/processor'
+require_relative 'core/reporter'
+
+module FFWD
+  class Core
+    include FFWD::Lifecycle
+    include FFWD::Logging
+
+    def initialize plugins, opts={}
+      @tunnel_plugins = plugins[:tunnel] || []
+      @input_plugins = plugins[:input] || []
+      @output_plugins = plugins[:output] || []
+
+      @statistics_opts = opts[:statistics]
+      @debug_opts = opts[:debug]
+      @core_opts = opts[:core] || {}
+      @processors = FFWD::Processor.load_processors(opts[:processor] || {})
+
+      @output_channel = FFWD::PluginChannel.build 'output'
+      @input_channel = FFWD::PluginChannel.build 'input'
+
+      @system_channel = Channel.new log, "system_channel"
+
+      memory_config = (@core_opts[:memory] || {})
+      @memory_limit = (memory_config[:limit] || 1000).to_f.round(3)
+      @memory_limit95 = @memory_limit * 0.95
+
+      if @memory_limit < 0
+        raise "memory limit must be non-negative number"
+      end
+
+      @emitter = Core::Emitter.build @output_channel, @core_opts
+      @processor = Core::Processor.build @input_channel, @emitter, @processors
+
+      @debug = nil
+
+      if @debug_opts
+        @debug = FFWD::Debug.setup @debug_opts
+        @debug.monitor "core.input", @input_channel, FFWD::Debug::Input
+        @debug.monitor "core.output", @output_channel, FFWD::Debug::Output
+        @debug.depend_on self
+      end
+
+      # Configuration for statistics module.
+      @statistics = nil
+
+      if config = @statistics_opts
+        @statistics = FFWD::Statistics.setup @emitter, @system_channel, config
+        @statistics.depend_on self
+      end
+
+      @interface = Core::Interface.new(
+        @input_channel, @output_channel,
+        @tunnel_plugins, @statistics, @debug, @processors, @core_opts
+      )
+
+      @interface.depend_on self
+
+      @input_instances = @input_plugins.map do |plugin|
+        plugin.setup @interface
+      end
+
+      @output_instances = @output_plugins.map do |plugin|
+        plugin.setup @interface
+      end
+
+      unless @statistics.nil?
+        reporters = [@input_channel, @output_channel, @processor]
+        reporters += @input_instances.select{|i| FFWD.is_reporter?(i)}
+        reporters += @output_instances.select{|i| FFWD.is_reporter?(i)}
+        @statistics.register "core", Core::Reporter.new(reporters)
+      end
+
+      # Make the core-related channels depend on core.
+      # They will then be orchestrated with core when it's being
+      # started/stopped.
+      @input_channel.depend_on self
+      @output_channel.depend_on self
+    end
+
+    # Main entry point.
+    #
+    # Since all components are governed by the lifecycle of core, it should
+    # mostly be a matter of calling 'start'.
+    def run
+      # What to do when we receive a shutdown signal?
+      shutdown_handler = proc do
+        # Hack to get out of trap context and into EM land.
+        EM.add_timer(0) do
+          log.info "Shutting down"
+          stop
+          EM.stop
+        end
+      end
+
+      EM.run do
+        Signal.trap("INT", &shutdown_handler)
+        Signal.trap("TERM", &shutdown_handler)
+
+        start
+        setup_memory_monitor
+      end
+
+      stopping do
+      end
+    end
+
+    private
+
+    # Sets up a memory monitor based of :core -> :memory -> :limit.
+    # Will warn at least once before shutting down.
+    def setup_memory_monitor
+      if @memory_limit == 0
+        log.warning "WARNING!!! YOU ARE RUNNING FFWD WITHOUT A MEMORY LIMIT, THIS COULD DAMAGE YOUR SYSTEM"
+        log.warning "To configure it, set the (:core -> :memory -> :limit) option to a non-zero number!"
+        return
+      end
+
+      log.info "Memory limited to #{@memory_limit} MB (:core -> :memory -> :limit)"
+
+      memory_one_warning = false
+
+      @system_channel.subscribe do |system|
+        memory = system[:memory]
+
+        mb = (memory[:resident].to_f / 1000000).round(3)
+
+        if memory_one_warning and mb > @memory_limit
+          log.error "Memory limit exceeded (#{mb}/#{@memory_limit} MB): SHUTTING DOWN"
+          EM.stop
+          next
+        end
+
+        if mb > @memory_limit95
+          log.warning "Memory limit almost reached (#{mb}/#{@memory_limit} MB)"
+          memory_one_warning = true
+        else
+          memory_one_warning = false
+        end
+      end
+    end
+  end
+end

data/lib/ffwd/core/emitter.rb

@@ -0,0 +1,38 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../logging'
+require_relative '../utils'
+require_relative '../metric_emitter'
+require_relative '../event_emitter'
+
+module FFWD
+  class Core; end
+
+  class Core::Emitter
+    attr_reader :event, :metric
+
+    def self.build output, opts={}
+      event = EventEmitter.build output, opts, opts[:event] || {}
+      metric = MetricEmitter.build output, opts, opts[:metric] || {}
+      new(event, metric)
+    end
+
+    def initialize event, metric
+      @event = event
+      @metric = metric
+    end
+  end
+end

data/lib/ffwd/core/interface.rb

@@ -0,0 +1,47 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../lifecycle'
+
+module FFWD
+  class Core; end
+
+  class Core::Interface
+    include FFWD::Lifecycle
+
+    attr_reader :input, :output
+    attr_reader :tunnel_plugins, :statistics, :debug, :processors
+    attr_reader :tags, :attributes
+
+    def initialize(input, output, tunnel_plugins, statistics, debug,
+                   processors, opts)
+      @input = input
+      @output = output
+      @tunnel_plugins = tunnel_plugins
+      @statistics = statistics
+      @debug = debug
+      @processors = processors
+      @opts = opts
+      @tags = opts[:tags] || []
+      @attributes = opts[:attributes] || {}
+    end
+
+    def reconnect input
+      self.class.new(
+        input, @output, @tunnel_plugins, @statistics, @debug, @processors,
+        @opts)
+    end
+  end
+end

data/lib/ffwd/core/processor.rb

@@ -0,0 +1,92 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../event_emitter'
+require_relative '../lifecycle'
+
+module FFWD
+  class Core; end
+
+  # Component responsible for receiving and internally route metrics and
+  # events.
+  #
+  # The term 'processor' is used because depending on the set of provided
+  # processors it might be determined that the received metric should be
+  # provided to one of them instead.
+  #
+  # If no processor matches, it is just passed straight through.
+  class Core::Processor
+    def self.build input, emitter, processors
+      processors = Hash[processors.map{|p| [p.name, p.setup(emitter)]}]
+      reporters = processors.select{|k, p| FFWD.is_reporter?(p)}.map{|k, p| p}
+      new(input, emitter, processors, reporters)
+    end
+
+    def initialize input, emitter, processors, reporters
+      @emitter = emitter
+      @processors = processors
+      @reporters = reporters
+
+      subs = []
+
+      @processors.each do |name, p|
+        p.depend_on input
+      end
+
+      input.starting do
+        subs << input.metric_subscribe do |m|
+          process_metric m
+        end
+
+        subs << input.event_subscribe do |e|
+          process_event e
+        end
+      end
+
+      input.stopping do
+        subs.each(&:unsubscribe).clear
+      end
+    end
+
+    def report!
+      @reporters.each do |reporter|
+        reporter.report! do |d|
+          yield d
+        end
+      end
+    end
+
+    private
+
+    def process_metric m
+      m[:time] ||= Time.now
+
+      unless p = m[:proc]
+        return @emitter.metric.emit m
+      end
+
+      unless p = @processors[p]
+        return @emitter.metric.emit m
+      end
+
+      p.process m
+    end
+
+    def process_event e
+      e[:time] ||= Time.now
+      @emitter.event.emit e
+    end
+  end
+end

data/lib/ffwd/core/reporter.rb

@@ -0,0 +1,32 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  class Core; end
+
+  class Core::Reporter
+    def initialize reporters
+      @reporters = reporters
+    end
+
+    def report!
+      @reporters.each do |reporter|
+        reporter.report! do |d|
+          yield d
+        end
+      end
+    end
+  end
+end

data/lib/ffwd/debug.rb

@@ -0,0 +1,76 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'json'
+require 'eventmachine'
+
+require_relative 'logging'
+require_relative 'event'
+require_relative 'metric'
+require_relative 'retrier'
+require_relative 'lifecycle'
+
+require_relative 'debug/tcp'
+
+module FFWD::Debug
+  module Input
+    def self.serialize_event event
+      event = Hash[event]
+
+      if tags = event[:tags]
+        event[:tags] = tags.to_a
+      end
+
+      event
+    end
+
+    def self.serialize_metric metric
+      metric = Hash[metric]
+
+      if tags = metric[:tags]
+        metric[:tags] = tags.to_a
+      end
+
+      metric
+    end
+  end
+
+  module Output
+    def self.serialize_event event
+      event.to_h
+    end
+
+    def self.serialize_metric metric
+      metric.to_h
+    end
+  end
+
+  DEFAULT_REBIND_TIMEOUT = 10
+  DEFAULT_HOST = "localhost"
+  DEFAULT_PORT = 19001
+
+  def self.setup opts={}
+    host = opts[:host] || DEFAULT_HOST
+    port = opts[:port] || DEFAULT_PORT
+    rebind_timeout = opts[:rebind_timeout] || DEFAULT_REBIND_TIMEOUT
+    proto = FFWD.parse_protocol(opts[:protocol] || "tcp")
+
+    if proto == FFWD::TCP
+      return TCP.new host, port, rebind_timeout
+    end
+
+    throw Exception.new("Unsupported protocol '#{proto}'")
+  end
+end