ffwd 0.1.0

data/lib/ffwd/protocol/tcp/plain_connect.rb

@@ -0,0 +1,74 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../../reporter'
+
+module FFWD::TCP
+  class PlainConnect
+    include FFWD::Reporter
+
+    setup_reporter :keys => [
+      :dropped_events, :dropped_metrics,
+      :sent_events, :sent_metrics,
+      :failed_events, :failed_metrics
+    ]
+
+    attr_reader :log
+
+    def reporter_meta
+      @c.reporter_meta
+    end
+
+    INITIAL_TIMEOUT = 2
+
+    def initialize core, log, connection
+      @log = log
+      @c = connection
+
+      subs = []
+
+      core.starting do
+        @c.connect
+        subs << core.output.event_subscribe{|e| handle_event e}
+        subs << core.output.metric_subscribe{|e| handle_metric e}
+      end
+
+      core.stopping do
+        @c.disconnect
+        subs.each(&:unsubscribe).clear
+      end
+    end
+
+    def handle_event event
+      return increment :dropped_events, 1 unless @c.writable?
+      @c.send_event event
+      increment :sent_events, 1
+    rescue => e
+      log.error "Failed to handle event", e
+      log.error "The following event could not be flushed: #{event.to_h}"
+      increment :failed_events, 1
+    end
+
+    def handle_metric metric
+      return increment :dropped_metrics, 1 unless @c.writable?
+      @c.send_metric metric
+      increment :sent_metrics, 1
+    rescue => e
+      log.error "Failed to handle metric", e
+      log.error "The following metric could not be flushed: #{metric.to_h}"
+      increment :failed_metrics, 1
+    end
+  end
+end

data/lib/ffwd/protocol/udp.rb

@@ -0,0 +1,48 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'eventmachine'
+
+require_relative 'udp/connect'
+require_relative 'udp/bind'
+
+require_relative '../tunnel'
+
+module FFWD::UDP
+  def self.family
+    :udp
+  end
+
+  DEFAULT_REBIND_TIMEOUT = 10
+
+  def self.connect opts, core, log, handler
+    raise "Missing required key :host" if (host = opts[:host]).nil?
+    raise "Missing required key :port" if (port = opts[:port]).nil?
+    Connect.new core, log, host, port, handler
+  end
+
+  def self.bind opts, core, log, connection, *args
+    raise "Missing required key :host" if (host = opts[:host]).nil?
+    raise "Missing required key :port" if (port = opts[:port]).nil?
+    rebind_timeout = opts[:rebind_timeout] || DEFAULT_REBIND_TIMEOUT
+    Bind.new core, log, host, port, connection, args, rebind_timeout
+  end
+
+  def self.tunnel opts, core, plugin, log, connection, *args
+    raise "Missing required key :port" if (port = opts[:port]).nil?
+    FFWD.tunnel self.family, port, core, plugin, log, connection, args
+  end
+end
+

data/lib/ffwd/protocol/udp/bind.rb

@@ -0,0 +1,64 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'eventmachine'
+
+require_relative '../../reporter'
+require_relative '../../retrier'
+
+module FFWD::UDP
+  class Bind
+    include FFWD::Reporter
+
+    setup_reporter :keys => [
+      :received_events, :received_metrics,
+      :failed_events, :failed_metrics
+    ]
+
+    attr_reader :reporter_meta
+
+    def initialize core, log, host, port, connection, args, rebind_timeout
+      @peer = "#{host}:#{port}"
+      @reporter_meta = {
+        :type => connection.plugin_type,
+        :listen => @peer, :family => 'udp'
+      }
+
+      @sig = nil
+
+      info = "udp://#{@peer}"
+
+      r = FFWD.retry :timeout => rebind_timeout do |a|
+        @sig = EM.open_datagram_socket host, port, connection, self, core, *args
+        log.info "Bind on #{info} (attempt #{a})"
+      end
+
+      r.error do |a, t, e|
+        log.error "Failed to bind #{info} (attempt #{a}), retry in #{t}s", e
+      end
+
+      r.depend_on core
+
+      core.stopping do
+        log.info "Unbinding #{info}"
+
+        if @sig
+          @sig.unbind
+          @sig = nil
+        end
+      end
+    end
+  end
+end

data/lib/ffwd/protocol/udp/connect.rb

@@ -0,0 +1,110 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative '../../reporter'
+require_relative '../../retrier'
+
+module FFWD::UDP
+  class Connect
+    include FFWD::Reporter
+
+    attr_reader :reporter_meta, :log
+
+    setup_reporter :keys => [
+      :dropped_events, :dropped_metrics,
+      :sent_events, :sent_metrics
+    ]
+
+    def initialize core, log, host, port, handler
+      @log = log
+      @host = host
+      @port = port
+      @handler = handler
+
+      @bind_host = "0.0.0.0"
+      @host_ip = nil
+      @c = nil
+      @peer = "#{host}:#{port}"
+      @reporter_meta = {
+        :type => @handler.name, :peer => peer
+      }
+
+      info = "udp://#{@peer}"
+
+      subs = []
+
+      r = FFWD.retry :timeout => resolve_timeout do |a|
+        unless @host_ip
+          @host_ip = resolve_ip @host
+          raise "Could not resolve: #{@host}" if @host_ip.nil?
+        end
+
+        @c = EM.open_datagram_socket(@bind_host, nil)
+        log.info "Setup of output to #{info} successful"
+
+        subs << core.output.event_subscribe{|e| handle_event e}
+        subs << core.output.metric_subscribe{|m| handle_metric m}
+      end
+
+      r.error do |a, t, e|
+        log.error "Setup of output to #{info} failed (attempt #{a}), retry in #{t}s", e
+      end
+
+      r.depend_on core
+
+      core.stopping do
+        if @c
+          @c.close
+          @c = nil
+        end
+
+        subs.each(&:unsubscribe).clear
+      end
+    end
+
+    private
+
+    def handle_event event
+      unless @c
+        increment :dropped_events, 1
+        return
+      end
+
+      data = @handler.serialize_event event
+      @c.send_datagram data, @host_ip, @port
+      increment :sent_events, 1
+    end
+
+    def handle_metric metric
+      unless @c
+        increment :dropped_metrics, 1
+        return
+      end
+
+      data = @handler.serialize_metric metric
+      @c.send_datagram data, @host_ip, @port
+      increment :sent_metrics, 1
+    end
+
+    def resolve_ip host
+      Socket.getaddrinfo(@host, nil, nil, :DGRAM).each do |item|
+        next if item[0] != "AF_INET"
+        return item[3]
+      end
+
+      return nil
+    end
+  end
+end

data/lib/ffwd/reporter.rb

@@ -0,0 +1,65 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD::Reporter
+  def self.map_meta meta
+    Hash[meta.map{|k, v| [k.to_s, v]}]
+  end
+
+  module ClassMethods
+    def reporter_keys
+      @_reporter_keys ||= [:total]
+    end
+
+    def reporter_meta
+      @_reporter_meta ||= nil
+    end
+
+    def reporter_meta_method
+      @_reporter_meta_method ||= :reporter_meta
+    end
+
+    def setup_reporter opts={}
+      @_reporter_keys = [:total] + (opts[:keys] || [])
+      @_reporter_meta = opts[:reporter_meta]
+      @_reporter_meta_method = opts[:id_method] || :reporter_meta
+    end
+  end
+
+  def self.included mod
+    mod.extend ClassMethods
+  end
+
+  def reporter_data
+    @_reporter_keys ||= self.class.reporter_keys
+    @_reporter_data ||= Hash[@_reporter_keys.map{|k| [k, 0]}]
+  end
+
+  def increment n, c=1
+    reporter_data[n] += c
+    reporter_data[:total] += c
+  end
+
+  def report!
+    @_reporter_meta ||= FFWD::Reporter.map_meta(
+      self.class.reporter_meta || send(self.class.reporter_meta_method))
+
+    reporter_data.each do |k, v|
+      yield(:key => k, :value => v,
+            :meta => @_reporter_meta)
+      reporter_data[k] = 0
+    end
+  end
+end

data/lib/ffwd/retrier.rb

@@ -0,0 +1,72 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'logging'
+require_relative 'lifecycle'
+
+module FFWD
+  # Try to execute a block on an exponential timer until it no longer throws
+  # an exception.
+  class Retrier
+    include FFWD::Lifecycle
+
+    def initialize timeout, &block
+      @block = block
+      @timer = nil
+      @timeout = timeout
+      @current_timeout = @timeout
+      @attempt = 0
+      @error_callbacks = []
+
+      starting do
+        try_block
+      end
+
+      stopping do
+        if @timer
+          @timer.cancel
+          @timer = nil
+        end
+      end
+    end
+
+    def error &block
+      @error_callbacks << block
+    end
+
+    def try_block
+      @attempt += 1
+      @block.call @attempt
+      @current_timeout = @timeout
+    rescue => e
+      @error_callbacks.each do |block|
+        block.call @attempt, @current_timeout, e
+      end
+
+      @timer = EM::Timer.new(@current_timeout) do
+        @current_timeout *= 2
+        @timer = nil
+        try_block
+      end
+    end
+  end
+
+  DEFAULT_TIMEOUT = 10
+
+  def self.retry opts={}, &block
+    timeout = opts[:timeout] || DEFAULT_TIMEOUT
+    Retrier.new(timeout, &block)
+  end
+end

data/lib/ffwd/schema.rb

@@ -0,0 +1,92 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  SCHEMA_DEFAULT_SUPPORT = [
+    :dump_metric,
+    :dump_event
+  ]
+
+  DEFUALT_SCHEMA = 'default'
+  DEFAULT_CONTENT_TYPE = 'application/json'
+
+  def self.parse_schema opts, support=SCHEMA_DEFAULT_SUPPORT
+    name = opts[:schema] || DEFUALT_SCHEMA
+    content_type = opts[:content_type] || DEFAULT_CONTENT_TYPE
+    key = [name, content_type]
+
+    schema = FFWD::Schema.loaded[key]
+
+    if schema.nil?
+      raise "No schema '#{name}' for content type '#{content_type}'"
+    end
+
+    unless schema.support? support
+      raise "Schema #{schema} does not support all of: #{support}"
+    end
+
+    return schema.mod
+  end
+
+  module Schema
+    class Loaded
+      attr_reader :source, :mod
+
+      def initialize source, mod
+        @source = source
+        @mod = mod
+      end
+
+      def support? support
+        support.each do |m|
+          return false unless @mod.respond_to? m
+        end
+
+        return true
+      end
+    end
+
+    def self.loaded
+      @@loaded ||= {}
+    end
+
+    def self.discovered
+      @@discovered ||= {}
+    end
+
+    module ClassMethods
+      def register_schema name, content_type, impl
+        key = [name, content_type]
+        FFWD::Schema.discovered[key] = impl
+      end
+    end
+
+    def self.included mod
+      mod.extend ClassMethods
+    end
+
+    def self.category
+      'schema'
+    end
+
+    def self.load_discovered source
+      FFWD::Schema.discovered.each do |key, mod|
+        FFWD::Schema.loaded[key] = Loaded.new source, mod
+      end
+
+      FFWD::Schema.discovered.clear
+    end
+  end
+end