ffwd 0.1.0

data/lib/ffwd/plugin/log.rb

@@ -0,0 +1,35 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/event'
+require 'ffwd/logging'
+require 'ffwd/metric'
+require 'ffwd/plugin'
+
+require_relative 'log/writer'
+
+module FFWD::Plugin
+  module Log
+    include FFWD::Plugin
+    include FFWD::Logging
+
+    register_plugin "log"
+
+    def self.setup_output opts, core
+      prefix = opts[:prefix]
+      Writer.new core, prefix
+    end
+  end
+end

data/lib/ffwd/plugin/log/writer.rb

@@ -0,0 +1,42 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../../logging'
+
+module FFWD::Plugin::Log
+  class Writer
+    include FFWD::Logging
+
+    def initialize core, prefix
+      @p = prefix ? "#{prefix} " : ""
+
+      subs = []
+
+      core.output.starting do
+        subs << core.output.event_subscribe do |e|
+          log.info "Event: #{@p}#{e.to_h}"
+        end
+
+        subs << core.output.metric_subscribe do |m|
+          log.info "Metric: #{@p}#{m.to_h}"
+        end
+      end
+
+      core.output.stopping do
+        subs.each(&:unsubscribe).clear
+      end
+    end
+  end
+end

data/lib/ffwd/plugin_channel.rb

@@ -0,0 +1,64 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'lifecycle'
+require_relative 'logging'
+require_relative 'reporter'
+
+module FFWD
+  # A set of channels, one for metrics and one for events.
+  # This is simply a convenience class to group the channel that are available
+  # to a plugin in one direction (usually either input or output).
+  class PluginChannel
+    include FFWD::Lifecycle
+    include FFWD::Reporter
+    include FFWD::Logging
+
+    setup_reporter :keys => [:metrics, :events]
+
+    attr_reader :reporter_meta, :events, :metrics, :name
+
+    def self.build name
+      events = FFWD::Channel.new log, "#{name}.events"
+      metrics = FFWD::Channel.new log, "#{name}.metrics"
+      new name, metrics, events
+    end
+
+    def initialize name, events, metrics
+      @name = name
+      @events = events
+      @metrics = metrics
+      @reporter_meta = {:plugin_channel => @name, :type => "plugin_channel"}
+    end
+
+    def event_subscribe &block
+      @events.subscribe(&block)
+    end
+
+    def event event
+      @events << event
+      increment :events
+    end
+
+    def metric_subscribe &block
+      @metrics.subscribe(&block)
+    end
+
+    def metric metric
+      @metrics << metric
+      increment :metrics
+    end
+  end
+end

data/lib/ffwd/plugin_loader.rb

@@ -0,0 +1,121 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'logging'
+require_relative 'plugin'
+
+# Some crazy code to load modules from a specific directory structure.
+module FFWD::PluginLoader
+  include FFWD::Logging
+
+  MODULE_NAME = 'ffwd'
+
+  def self.plugin_directories= directories
+    @plugin_directories = directories
+  end
+
+  def self.plugin_directories
+    @plugin_directories || []
+  end
+
+  # Discover plugins in the specified directory that are prefixed with 'ffwd-'.
+  def self.discover_plugins dir
+    return [] unless File.directory? dir
+
+    Dir.foreach(dir).map do |entity|
+      next if entity.start_with? "."
+      next unless entity.start_with? "#{MODULE_NAME}-"
+      full_path = File.join dir, entity, 'lib'
+      next unless File.directory? full_path
+      yield full_path
+    end
+  end
+
+  def self.plugin_paths
+    return @plugin_paths if @plugin_paths
+
+    @plugin_paths = []
+
+    plugin_directories.map do |dir|
+      discover_plugins(dir) do |path|
+        @plugin_paths << path
+      end
+    end
+
+    return @plugin_paths
+  end
+
+  def self.load_paths
+    return @load_paths if @load_paths
+
+    @load_paths = []
+
+    Gem::Specification.latest_specs(true).collect do |spec|
+      @load_paths << ["from gem: #{spec.full_name}",
+                      File.join(spec.full_gem_path, 'lib')]
+    end
+
+    plugin_paths.each do |path|
+      @load_paths << ["from plugin directory: #{path}", path]
+    end
+
+    $LOAD_PATH.each do |path|
+      @load_paths << ["from $LOAD_PATH: #{path}", path]
+    end
+
+    unless plugin_paths.empty?
+      $LOAD_PATH.concat plugin_paths
+    end
+
+    return @load_paths
+  end
+
+  def self.list_modules module_category, blacklist, &block
+    load_paths.each do |source, path|
+      dir = File.join(path, MODULE_NAME, module_category)
+
+      next unless File.directory? dir
+
+      Dir.foreach dir do |entity|
+        next if entity.start_with? "."
+        next unless entity.end_with? ".rb"
+        next unless File.file? File.join(dir, entity)
+
+        base = entity.slice(0, entity.size - 3)
+
+        if blacklist.include? base
+          log.warning "Ignoring blacklisted module: #{base}"
+          next
+        end
+
+        path = [MODULE_NAME, module_category, base].join('/')
+        yield source, path
+      end
+    end
+  end
+
+  def self.load mod, blacklist
+    self.list_modules(mod.category, blacklist) do |source, m|
+      begin
+        require m
+      rescue LoadError => e
+        log.error "Failed to require '#{m}'", e
+      end
+
+      # Initialize all newly discovered plugins.
+      mod.load_discovered source
+    end
+  end
+end

data/lib/ffwd/processor.rb

@@ -0,0 +1,96 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'logging'
+require_relative 'lifecycle'
+
+module FFWD::Processor
+  include FFWD::Lifecycle
+  include FFWD::Logging
+
+  class Setup
+    attr_reader :name
+
+    def initialize name, klass, options
+      @name = name
+      @klass = klass
+      @options = options
+    end
+
+    def setup emitter
+      @klass.new emitter, @options
+    end
+  end
+
+  # Module to include for processors.
+  #
+  # Usage:
+  #
+  # class MyProcessor
+  #   include FFWD::Processor
+  #
+  #   register_processor "my_processor"
+  #
+  #   def initialize opts
+  #     .. read options ..
+  #   end
+  #
+  #   def start emitter
+  #     ... setup EventMachine tasks ...
+  #   end
+  #
+  #   def process emitter, m
+  #     ... process a single metric ...
+  #     emitter.metrics.emit ...
+  #   end
+  # end
+  def process m
+    raise Exception.new("process: Not Implemented")
+  end
+
+  module ClassMethods
+    def register_processor(name)
+      unless FFWD::Processor.registry[name].nil?
+        raise "Already registered '#{name}'"
+      end
+
+      FFWD::Processor.registry[name] = self
+    end
+  end
+
+  def name
+    self.class.name
+  end
+
+  def self.registry
+    @@registry ||= {}
+  end
+
+  def self.category
+    'processor'
+  end
+
+  def self.included(mod)
+    mod.extend ClassMethods
+  end
+
+  def self.load_discovered source
+  end
+
+  # setup hash of processor setup classes.
+  def self.load_processors config
+    registry.map{|name, klass| Setup.new name, klass, config[name] || {}}
+  end
+end

data/lib/ffwd/processor/count.rb

@@ -0,0 +1,109 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../processor'
+require_relative '../logging'
+require_relative '../event'
+require_relative '../utils'
+require_relative '../reporter'
+
+module FFWD::Processor
+  #
+  # Implements counting statistics (similar to statsd).
+  #
+  class CountProcessor
+    include FFWD::Logging
+    include FFWD::Processor
+    include FFWD::Reporter
+
+    register_processor "count"
+    setup_reporter(
+      :reporter_meta => {:processor => "count"},
+      :keys => [:dropped, :received]
+    )
+
+    def initialize emitter, opts={}
+      @emitter = emitter
+      @cache_limit = opts[:cache_limit] || 1000
+      @timeout = opts[:timeout] || 300
+      @period = opts[:period] || 10
+      @cache = {}
+      @timer = nil
+
+      starting do
+        log.info "Starting count processor on a window of #{@period}s"
+      end
+
+      stopping do
+        log.info "Stopping count processor"
+        @timer.cancel if @timer
+        @timer = nil
+      end
+    end
+
+    def flush_caches! now
+      @cache.each do |key, entry|
+        count = entry[:count]
+        last = entry[:last]
+
+        if now - last > @timeout
+          @cache.delete key
+          next
+        end
+
+        entry[:count] = 0
+
+        @emitter.metric.emit(
+          :key => "#{key}.sum", :value => count, :source => key)
+      end
+    end
+
+    def check_timer
+      return if @timer
+
+      log.debug "Starting timer"
+
+      @timer = EM::Timer.new(@period) do
+        @timer = nil
+        digest!
+      end
+    end
+
+    def digest! now
+      ms = FFWD.timing do
+        flush_caches! now
+      end
+
+      log.debug "Digest took #{ms}ms"
+    end
+
+    def process m
+      key = m[:key]
+      value = m[:value]
+
+      now = Time.now
+
+      unless (entry = @cache[key])
+        return increment :dropped if @cache.size >= @cache_limit
+        entry = @cache[key] = {:count => 0, :last => now}
+      end
+
+      increment :received
+      entry[:count] += value
+      entry[:last] = now
+      check_timer
+    end
+  end
+end