ffwd 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1d0b671545d89979b849e559f5a5e5c6c0a86d8a
+  data.tar.gz: 3470a5c756d427bac159f4ea1c2c5a45099f87ef
+SHA512:
+  metadata.gz: 84bfcbee1c667878c8760b3c1722c8d3b3a822d8d1f23d4c0e25e0bde744d815c4d74f7d3f6baf4a52be058b2d8c18a5506117f51e29e8e7e6c0236dd3b2ab11
+  data.tar.gz: 0e97592bd405068ae86f0d6057cc9d8d0989b310858b7e070a53dfc0c583c71de538edf12086c60658f768191423f2be5b02d38ff4ec98cf5ca078fcdfd20ef1

data/bin/ffwd

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+if __FILE__ == $0
+    lib = File.expand_path File.join('..', '..', 'lib'), $0
+    $:.insert(0, lib) if File.file? File.join(lib, 'ffwd.rb')
+end
+
+require 'ffwd'
+exit FFWD::main(ARGV || [])

data/bin/fwc

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+if __FILE__ == $0
+    lib = File.expand_path File.join('..', '..', 'lib'), $0
+    $:.insert(0, lib) if File.file? File.join(lib, 'ffwd.rb')
+end
+
+begin
+    require 'fwc'
+rescue LoadError
+    require 'rubygems'
+    require 'fwc'
+end
+
+exit FWC::main(ARGV || [])

data/lib/em/all.rb

@@ -0,0 +1,68 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# EventMachine extension to create a deferrable that correctly manages multiple
+# deferred objects in parallel.
+
+module EM
+  class All
+    include EM::Deferrable
+
+    def initialize
+      @defs = []
+      @errors = []
+      @results = []
+      @all_ok = true
+      @any_ok = false
+      @done = 0
+    end
+
+    def <<(d)
+      @defs << d
+
+      index = @results.size
+
+      @results << nil
+      @errors << nil
+
+      d.callback do |*args|
+        @results[index] = args
+        @any_ok = true
+        check_results
+      end
+
+      d.errback do |*args|
+        @errors[index] = args
+        @all_ok = false
+        check_results
+      end
+    end
+
+    private
+
+    def check_results
+      @done += 1
+
+      return unless @defs.size == @done
+
+      unless @all_ok
+        fail(@errors)
+        return
+      end
+
+      succeed(@results)
+    end
+  end
+end

data/lib/ffwd.rb

@@ -0,0 +1,250 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'yaml'
+require 'optparse'
+
+require_relative 'ffwd/core'
+require_relative 'ffwd/logging'
+require_relative 'ffwd/plugin'
+require_relative 'ffwd/plugin_loader'
+require_relative 'ffwd/processor'
+require_relative 'ffwd/schema'
+
+module FFWD
+  DEFAULT_PLUGIN_DIRECTORIES = [
+    './plugins'
+  ]
+
+  def self.load_yaml path
+    return YAML.load_file path
+  rescue => e
+    log.error "Failed to load config: #{path} (#{e})"
+    return nil
+  end
+
+  def self.merge_configurations target, source
+    if target.is_a? Hash
+      raise "source not a Hash: #{source}" unless source.is_a? Hash
+
+      source.each do |key, value|
+        target[key] = merge_configurations target[key], source[key]
+      end
+
+      return target
+    end
+
+    if target.is_a? Array
+      raise "source not an Array: #{source}" unless source.is_a? Array
+      return target + source
+    end
+
+    # override source
+    return source
+  end
+
+  def self.load_config_dir dir, config
+    Dir.entries(dir).sort.each do |entry|
+      entry_path = File.join dir, entry
+
+      next unless File.file? entry_path
+
+      if entry.start_with? "."
+        log.debug "Ignoring: #{entry_path} (hidden file)"
+        next
+      end
+
+      c = load_yaml entry_path
+
+      if c.nil?
+        log.warn "Ignoring: #{entry_path} (invalid yaml)"
+        next
+      end
+
+      yield c
+    end
+  end
+
+  def self.opts
+    @@opts ||= {:debug => false, :config => nil, :config_dir => nil,
+                :list_plugins => false, :list_schemas => false,
+                :dump_config => false,
+                :plugin_directories => DEFAULT_PLUGIN_DIRECTORIES}
+  end
+
+  def self.parser
+    @@parser ||= OptionParser.new do |o|
+      o.banner = "Usage: ffwd [options]"
+
+      o.on "-d", "--[no-]debug" do |d|
+        opts[:debug] = d
+      end
+
+      o.on "-c", "--config <path>" do |path|
+        opts[:config_path] = path
+      end
+
+      o.on "-d", "--config-dir <path>" do |path|
+        opts[:config_dir] = path
+      end
+
+      o.on "--list-plugins", "Print available plugins." do
+        opts[:list_plugins] = true
+      end
+
+      o.on "--list-schemas", "Print available schemas." do
+        opts[:list_schemas] = true
+      end
+
+      o.on "--dump-config", "Dump the configuration that has been loaded." do
+        opts[:dump_config] = true
+      end
+
+      o.on "--plugin-directory <dir>", "Load plugins from the specified directory." do |dir|
+        opts[:plugin_directories] << dir
+      end
+    end
+  end
+
+  def self.parse_options args
+    parser.parse args
+  end
+
+  def self.setup_plugins config
+    plugins = {}
+
+    plugins[:tunnel] = FFWD::Plugin.load_plugins(
+      log, "Tunnel", config[:tunnel], :tunnel)
+
+    plugins[:input] = FFWD::Plugin.load_plugins(
+      log, "Input", config[:input], :input)
+
+    plugins[:output] = FFWD::Plugin.load_plugins(
+      log, "Output", config[:output], :output)
+
+    plugins
+  end
+
+  def self.main args
+    parse_options args
+
+    FFWD.log_config[:level] = opts[:debug] ? Logger::DEBUG : Logger::INFO
+
+    config = {:debug => nil}
+
+    if config_path = opts[:config_path]
+      unless File.file? config_path
+        puts "Configuration path does not exist: #{config_path}"
+        puts ""
+        puts parser.help
+        return 1
+      end
+
+      unless source = load_yaml(config_path)
+        return 0
+      end
+
+      merge_configurations config, source
+    end
+
+    if config_dir = opts[:config_dir]
+      unless File.directory? config_dir
+        puts "Configuration directory does not exist: #{path}"
+        puts ""
+        puts parser.help
+        return 1
+      end
+
+      load_config_dir(config_dir, config) do |c|
+        merge_configurations config, c
+      end
+    end
+
+    if config[:logging]
+      config[:logging].each do |key, value|
+        FFWD.log_config[key] = value
+      end
+    end
+
+    FFWD.log_reload
+
+    if FFWD.log_config[:file]
+      puts "Logging to file: #{FFWD.log_config[:file]}"
+    end
+
+    blacklist = config[:blacklist] || {}
+
+    directories = ((config[:plugin_directories] || []) +
+                   (opts[:plugin_directories] || []))
+
+    PluginLoader.plugin_directories = directories
+
+    PluginLoader.load FFWD::Plugin, blacklist[:plugins] || []
+    PluginLoader.load FFWD::Processor, blacklist[:processors] || []
+    PluginLoader.load FFWD::Schema, blacklist[:schemas] || []
+
+    stop_early = false
+    stop_early ||= opts[:list_plugins]
+    stop_early ||= opts[:list_schemas]
+    stop_early ||= opts[:dump_config]
+
+    plugins = setup_plugins config
+
+    if opts[:list_plugins]
+      puts "Available Plugins:"
+
+      FFWD::Plugin.loaded.each do |name, plugin|
+        puts "Plugin '#{name}' (#{plugin.source})"
+        puts "  supports: #{plugin.capabilities.join(' ')}"
+      end
+
+      puts "Activated Plugins:"
+
+      plugins.each do |kind, kind_plugins|
+        puts "#{kind}:"
+
+        if kind_plugins.empty?
+          puts "  (no active)"
+        end
+
+        kind_plugins.each do |p|
+          puts "  #{p.name}: #{p.config}"
+        end
+      end
+    end
+
+    if opts[:list_schemas]
+      puts "Available Schemas:"
+
+      FFWD::Schema.loaded.each do |key, schema|
+        name, content_type = key
+        puts "Schema '#{name}' #{content_type} (#{schema.source})"
+      end
+    end
+
+    if opts[:dump_config]
+      puts "Dumping Configuration:"
+      puts config
+    end
+
+    if stop_early
+      return 0
+    end
+
+    core = FFWD::Core.new plugins, config
+    core.run
+    return 0
+  end
+end

data/lib/ffwd/channel.rb

@@ -0,0 +1,62 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'set'
+
+# Defines containers which are limited in size.
+module FFWD
+  class Channel
+    class Sub
+      attr_reader :channel, :block
+
+      def initialize channel, block
+        @channel = channel
+        @block = block
+      end
+
+      def unsubscribe
+        @channel.unsubscribe self
+      end
+    end
+
+    attr_reader :subs
+
+    def initialize log, name
+      @log = log
+      @name = name
+      @subs = Set.new
+    end
+
+    def <<(item)
+      @subs.each do |sub|
+        begin
+          sub.block.call item
+        rescue => e
+          @log.error "#{@name}: Subscription failed", e
+        end
+      end
+    end
+
+    def subscribe(&block)
+      s = Sub.new(self, block)
+      @subs << s
+      return s
+    end
+
+    def unsubscribe sub
+      @subs.delete sub
+    end
+  end
+end

data/lib/ffwd/circular_buffer.rb

@@ -0,0 +1,78 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+Node = Struct.new(:value, :next)
+
+module FFWD
+  class CircularBuffer
+    attr_reader :buffer, :capacity, :size
+
+    def initialize capacity
+      @capacity = capacity
+      @first = nil
+      @last = nil
+      @size = 0
+    end
+
+    def clear
+      @first = nil
+      @last = nil
+      @size = 0
+    end
+
+    def empty?
+      size == 0
+    end
+
+    def peek
+      return nil if @first.nil?
+      @first.value
+    end
+
+    def shift
+      return nil if @first.nil?
+      value = @first.value
+      @first = @first.next
+      @last = nil if @first.nil?
+      @size -= 1
+      value
+    end
+
+    def << item
+      node = Node.new(item, nil)
+
+      if @last.nil?
+        @first = @last = node
+      else
+        @last = @last.next = node
+      end
+
+      if @size >= @capacity
+        @first = @first.next
+      else
+        @size += 1
+      end
+    end
+
+    def each
+      current = @first
+
+      while not current.nil?
+        yield current.value
+        current = current.next
+      end
+    end
+  end
+end