ffwd 0.1.0

data/lib/ffwd/tunnel.rb

@@ -0,0 +1,27 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'tunnel/tcp'
+require_relative 'tunnel/udp'
+
+module FFWD
+  FAMILIES = {:tcp => Tunnel::TCP, :udp => Tunnel::UDP}
+
+  def self.tunnel family, port, core, plugin, log, connection, args
+    impl = FAMILIES[family]
+    raise "Unsupported family: #{family}" if impl.nil?
+    return impl.new port, core, plugin, log, connection, args
+  end
+end

data/lib/ffwd/tunnel/plugin.rb

@@ -0,0 +1,47 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# describes the protocol that has to be implemented by a tunnel.
+
+module FFWD::Tunnel
+  class Plugin
+    # Object type that should be returned by 'tcp'.
+    class Handle
+      def close &block
+        raise "Not implemented: close"
+      end
+
+      def data &block
+        raise "Not implemented: data"
+      end
+
+      def send_data data
+        raise "Not implemented: send_data"
+      end
+    end
+
+    def tcp port, &block
+      raise "Not implemented: tcp"
+    end
+
+    def udp port, &block
+      raise "Not implemented: udp"
+    end
+
+    def send_data addr, data
+      raise "Not implemented: send_data"
+    end
+  end
+end

data/lib/ffwd/tunnel/tcp.rb

@@ -0,0 +1,60 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../lifecycle'
+require_relative '../reporter'
+
+module FFWD::Tunnel
+  class TCP
+    include FFWD::Lifecycle
+    include FFWD::Reporter
+
+    setup_reporter :keys => [
+      :received_events, :received_metrics, :failed_events, :failed_metrics]
+
+    attr_reader :log
+
+    def initialize port, core, plugin, log, connection, args
+      @port = port
+      @core = core
+      @plugin = plugin
+      @log = log
+      @connection = connection
+      @args = args
+
+      starting do
+        @plugin.tcp @port do |handle|
+          log.debug "Open tcp/#{@port}"
+
+          instance = @connection.new(nil, self, @core, *@args)
+          instance.datasink = handle
+
+          handle.data do |data|
+            instance.receive_data data
+          end
+
+          handle.close do
+            log.debug "Close tcp/#{@port}"
+            instance.unbind
+          end
+        end
+      end
+
+      stopping do
+        log.info "Stopped tunneling tcp/#{@port}"
+      end
+    end
+  end
+end

data/lib/ffwd/tunnel/udp.rb

@@ -0,0 +1,61 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../lifecycle'
+require_relative '../reporter'
+
+module FFWD::Tunnel
+  class UDP
+    include FFWD::Lifecycle
+    include FFWD::Reporter
+
+    setup_reporter :keys => [
+      :received_events, :received_metrics, :failed_events, :failed_metrics]
+
+    attr_reader :log
+
+    def initialize port, core, plugin, log, connection, args
+      @port = port
+      @core = core
+      @plugin = plugin
+      @log = log
+      @connection = connection
+      @args = args
+
+      @instance = nil
+
+      starting do
+        @instance = @connection.new(nil, self, @core, *@args)
+
+        @plugin.udp @port do |handle, data|
+          @instance.datasink = handle
+          @instance.receive_data data
+          @instance.datasink = nil
+        end
+
+        @log.info "Tunneling udp/#{@port}"
+      end
+
+      stopping do
+        if @instance
+          @instance.unbind
+          @instance = nil
+        end
+
+        @log.info "Stopped tunnelling udp/#{@port}"
+      end
+    end
+  end
+end

data/lib/ffwd/utils.rb

@@ -0,0 +1,46 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'socket'
+require 'set'
+
+module FFWD
+  # Merge two sets (arrays actually)
+  def self.merge_sets(a, b)
+    return Set.new(a + b).to_a if a and b
+    a || b || []
+  end
+
+  # Merge two hashes.
+  def self.merge_hashes(a, b)
+    return a.merge(b) if a and b
+    b || a || {}
+  end
+
+  def self.is_reporter? var
+    var.respond_to? :report!
+  end
+
+  def self.current_host
+    Socket.gethostname
+  end
+
+  def self.timing &block
+    start = Time.now
+    block.call
+    stop = Time.now
+    ((stop - start) * 1000).round(3)
+  end
+end

data/lib/ffwd/version.rb

@@ -0,0 +1,18 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  VERSION = "0.1.0"
+end

data/lib/fwc.rb

@@ -0,0 +1,206 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'optparse'
+require 'eventmachine'
+require 'json'
+require 'set'
+require 'logger'
+
+require 'ffwd/debug'
+
+module FWC
+  def self.log
+    @logger ||= ::Logger.new(STDOUT).tap do |l|
+      l.level = Logger::INFO
+      l.formatter = proc do |severity, datetime, progname, msg|
+        t = datetime.strftime("%H:%M:%S")
+        "#{t} #{severity}: #{msg}\n"
+      end
+    end
+  end
+
+  class TraceConnection < EM::Connection
+    include EM::Protocols::LineText2
+
+    def initialize instances
+      @instances = instances
+    end
+
+    def unbind
+      FWC.log.error "Connection lost"
+      EM.stop
+    end
+
+    def receive_line line
+      data = JSON.load line
+
+      @instances.each do |instance|
+        instance.receive data
+      end
+    end
+  end
+
+  def self.opts
+    @@opts ||= {
+      :debug => false,
+      :host => "localhost",
+      :port => FFWD::Debug::DEFAULT_PORT,
+      :summary => false,
+      :raw => false,
+      :raw_threshold => 100,
+      :report_interval => 10
+    }
+  end
+
+  def self.parser
+    @@parser ||= OptionParser.new do |o|
+      o.banner = "Usage: fwc [options]"
+
+      o.on "-d", "--[no-]debug" do |d|
+        opts[:debug] = d
+      end
+
+      o.on "-s", "--summary", "Show a periodic summary of everything seen" do
+        opts[:summary] = true
+      end
+
+      o.on "-r", "--raw", "Display raw metrics and events, as soon as they are seen" do
+        opts[:raw] = true
+      end
+
+      o.on "--raw-threshold <rate>", "Limit of how many messages/s is allowed before disabling output" do |d|
+        opts[:raw_threshold] = d.to_i
+      end
+
+      o.on "-i", "--report-interval", "Interval in seconds to generate report" do |d|
+        opts[:report_interval] = d.to_i
+      end
+    end
+  end
+
+  def self.parse_options(args)
+    parser.parse args
+  end
+
+  class Summary
+    def initialize
+      @groups = {}
+    end
+
+    def report!
+      return if @groups.empty?
+
+      FWC.log.info "Summary Report:"
+
+      @groups.sort.each do |id, group|
+        items = group[:items].to_a
+
+        FWC.log.info "  #{group[:id]} (#{group[:type]})"
+        items.sort.each do |key, count|
+          key = "<nil>" if key.nil?
+          FWC.log.info "    #{key} #{count}"
+        end
+      end
+
+      @groups = {}
+    end
+
+    def receive data
+      id = [data["id"], data["type"]]
+      group = (@groups[id] ||= {:id => data["id"], :type => data["type"],
+                                :items => {}})
+
+      key = data["data"]["key"]
+      items = group[:items]
+
+      if v = items[key]
+        items[key] = v + 1
+      else
+        items[key] = 1
+      end
+    end
+  end
+
+  class Raw
+    def initialize threshold
+      @threshold = threshold
+      @count = 0
+      @rate = 0
+      @disabled = false
+      @first = Time.now
+    end
+
+    def report!
+      return if @count == 0
+
+      FWC.log.info "Raw Report:"
+      FWC.log.info "  count: #{@count}"
+
+      @first = Time.now
+      @count = 0
+      @disabled = false
+    end
+
+    def receive data
+      if not @disabled and @count > 100
+        diff = (Time.now - @first)
+        rate = (@count / diff)
+
+        if rate > @threshold
+          r = rate.to_i
+          desc = "#{r}/s > #{@threshold}/s"
+          FWC.log.info "Raw: disabled - rate too high (#{desc})"
+          @disabled = true
+        end
+      end
+
+      unless @disabled
+        FWC.log.info "#{@count}: #{data}"
+      end
+
+      @count += 1
+    end
+  end
+
+  def self.main(args)
+    parse_options args
+
+    handlers = []
+
+    if opts[:summary]
+      handlers << Summary.new
+    end
+
+    if opts[:raw]
+      handlers << Raw.new(opts[:raw_threshold])
+    end
+
+    if handlers.empty?
+      puts "No methods specified"
+      return 1
+    end
+
+    EM.run do
+      EM.connect(opts[:host], opts[:port], FWC::TraceConnection, handlers)
+
+      EM::PeriodicTimer.new(opts[:report_interval]) do
+        handlers.each(&:report!)
+      end
+    end
+
+    return 0
+  end
+end