ffwd-kafka 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f573e5f2791717f35db451565fbe6c6fa81eed31
+  data.tar.gz: 801ab9687daa06525d5700f664d23e7ee0e8303c
+SHA512:
+  metadata.gz: 73165428dc728720ee1eea06a94c9a9ab72a2e40128b9391649b432aa1b9b0ce2e36db343e143e5365762742433a24bb58e9dfa2493be3c27d6012c9e81c19c3
+  data.tar.gz: 348dd79932ef81d62ba544c49887594c0270f71da0ea252c3fa1dbb39893ed4389487cb5b1b6fe4e3929c311429c04da622ad9524567fba540df39f61bf99554

data/lib/ffwd/plugin/kafka/output.rb

@@ -0,0 +1,102 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/logging'
+require 'ffwd/producing_client'
+
+require_relative 'producer'
+
+module FFWD::Plugin::Kafka
+  class Output < FFWD::ProducingClient::Producer
+    include FFWD::Logging
+    include FFWD::Reporter
+
+    setup_reporter :keys => [:kafka_routing_error, :kafka_routing_success]
+
+    attr_reader :reporter_meta
+
+    MAPPING = [:host, :ttl, :key, :time, :value, :tags, :attributes]
+
+    def initialize producer, brokers, schema, router, partitioner
+      @producer = producer
+      @brokers = brokers
+      @schema = schema
+      @router = router
+      @partitioner = partitioner
+      @reporter_meta = {:producer_type => "kafka", :producer => @producer}
+      @instance = nil
+    end
+
+    def setup
+      if not @brokers or @brokers.empty?
+        log.error "No usable initial list of brokers"
+        return
+      end
+
+      @instance = Producer.new @brokers, @producer
+    end
+
+    def teardown
+      if @instance
+        @instance.stop
+        @instance = nil
+      end
+    end
+
+    def produce events, metrics
+      unless @instance
+        return nil
+      end
+
+      expected_messages = events.size + metrics.size
+      messages = []
+
+      events.each do |e|
+        message = make_event_message e
+        next if message.nil?
+        messages << message
+      end
+
+      metrics.each do |e|
+        message = make_metric_message e
+        next if message.nil?
+        messages << message
+      end
+
+      if messages.size < expected_messages
+        increment :kafka_routing_error, expected_messages - messages.size
+      end
+
+      increment :kafka_routing_success, messages.size
+      @instance.send_messages messages
+    end
+
+    def make_event_message e
+      topic = @router.route_event e
+      return nil if topic.nil?
+      data = @schema.dump_event e
+      key = @partitioner.partition e
+      MessageToSend.new topic, data, key
+    end
+
+    def make_metric_message m
+      topic = @router.route_metric m
+      return nil if topic.nil?
+      data = @schema.dump_metric m
+      key = @partitioner.partition m
+      MessageToSend.new topic, data, key
+    end
+  end
+end

data/lib/ffwd/plugin/kafka/partitioners.rb

@@ -0,0 +1,67 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD::Plugin::Kafka
+  # Use the key for partitioning.
+  module KeyPartitioner
+    def self.partition d
+      d.key
+    end
+  end
+
+  # Use the host for partitioning.
+  module HostPartitioner
+    def self.partition d
+      d.host
+    end
+  end
+
+  # Use a custom attribute for partitioning.
+  class AttributePartitioner
+    DEFAULT_ATTRIBUTE = :site
+
+    def self.build opts
+      attr = opts[:attribute] || DEFAULT_ATTRIBUTE
+      new attr
+    end
+
+    def initialize attr
+      @attr = attr.to_sym
+      @attr_s = attr.to_s
+    end
+
+    # currently there is an issue where you can store both symbols and string
+    # as attribute keys, we need to take that into account.
+    def partition d
+      if v = d.attributes[@attr]
+        return v
+      end
+
+      d.attributes[@attr_s]
+    end
+  end
+
+  def self.build_partitioner type, opts
+    if type == :host
+      return HostPartitioner
+    end
+
+    if type == :key
+      return KeyPartitioner
+    end
+
+    return AttributePartitioner.build opts
+  end
+end

data/lib/ffwd/plugin/kafka/producer.rb

@@ -0,0 +1,99 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'poseidon'
+
+module FFWD::Plugin::Kafka
+  MessageToSend = Poseidon::MessageToSend
+
+  # A Kafka producer proxy for Poseidon (a kafka library) that delegates all
+  # blocking work to the EventMachine thread pool.
+  class Producer
+    class Request
+      include EM::Deferrable
+    end
+
+    def initialize *args
+      @args = args
+      @mutex = Mutex.new
+      @request = nil
+      @stopped = false
+    end
+
+    def stop
+      @stopped = true
+      shutdown
+    end
+
+    def shutdown
+      return if @request
+
+      @mutex.synchronize do
+        @producer.shutdown
+      end
+    end
+
+    def send_messages messages
+      execute do |p|
+        p.send_messages messages
+      end
+    end
+
+    def make_producer
+      if EM.reactor_thread?
+        raise "Should not be called in the reactor thread"
+      end
+
+      @mutex.synchronize do
+        @producer ||= Poseidon::Producer.new(*@args)
+      end
+    end
+
+    # Execute the provided block on a dedicated thread.
+    # The sole provided argument is an instance of Poseidon::Producer.
+    def execute &block
+      raise "Expected block" unless block_given?
+      raise "Request already pending" if @request
+
+      if @stopped
+        r = Request.new
+        r.fail "producer stopped"
+        return r
+      end
+
+      @request = Request.new
+
+      EM.defer do
+        begin
+          result = block.call make_producer
+
+          EM.next_tick do
+            @request.succeed result
+            @request = nil
+            shutdown if @stopped
+          end
+        rescue => e
+          EM.next_tick do
+            @request.fail e
+            @request = nil
+            shutdown if @stopped
+          end
+        end
+      end
+
+      @request
+    end
+  end
+end

data/lib/ffwd/plugin/kafka/routers.rb

@@ -0,0 +1,63 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD::Plugin::Kafka
+  # Use a custom attribute for partitioning.
+  class AttributeRouter
+    DEFAULT_METRIC_PATTERN = "metrics-%s"
+    DEFAULT_EVENT_PATTERN = "events-%s"
+    DEFAULT_ATTRIBUTE = :site
+
+    def self.build opts
+      metric_pattern = opts[:metric_pattern] || DEFAULT_METRIC_PATTERN
+      event_pattern = opts[:event_pattern] || DEFAULT_EVENT_PATTERN
+      attr = opts[:attribute] || DEFAULT_ATTRIBUTE
+      new(metric_pattern, event_pattern, attr)
+    end
+
+    def initialize metric_pattern, event_pattern, attr
+      @metric_pattern = metric_pattern
+      @event_pattern = event_pattern
+      @attr = attr.to_sym
+      @attr_s = attr.to_s
+    end
+
+    def value d
+      if v = d.attributes[@attr]
+        return v
+      end
+
+      d.attributes[@attr_s]
+    end
+    p
+    def route_event e
+      return nil unless v = value(e)
+      @event_pattern % [v]
+    end
+
+    def route_metric m
+      return nil unless v = value(m)
+      @metric_pattern % [v]
+    end
+  end
+
+  def self.build_router type, opts
+    if type == :attribute
+      return AttributeRouter.build opts
+    end
+
+    raise "Unsupported router type: #{type}"
+  end
+end

data/lib/ffwd/plugin/kafka/version.rb

@@ -0,0 +1,22 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  module Plugin
+    module Kafka
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/ffwd/plugin/kafka.rb

@@ -0,0 +1,53 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/logging'
+require 'ffwd/plugin'
+require 'ffwd/producing_client'
+require 'ffwd/reporter'
+require 'ffwd/schema'
+
+require_relative 'kafka/output'
+require_relative 'kafka/partitioners'
+require_relative 'kafka/routers'
+
+module FFWD::Plugin
+  module Kafka
+    include FFWD::Plugin
+    include FFWD::Logging
+
+    register_plugin "kafka"
+
+    DEFAULT_PRODUCER = "ffwd"
+    DEFAULT_ROUTING_METHOD = :attribute
+    DEFAULT_ROUTING_KEY = :site
+    DEFAULT_BROKERS = ["localhost:9092"]
+    DEFAULT_PARTITIONER = :host
+    DEFAULT_ROUTER = :attribute
+
+    def self.setup_output opts, core
+      producer = opts[:producer] || DEFAULT_PRODUCER
+      brokers = opts[:brokers] || DEFAULT_BROKERS
+      partitioner = FFWD::Plugin::Kafka.build_partitioner(
+        opts[:partitioner] || DEFAULT_PARTITIONER, opts)
+      router = FFWD::Plugin::Kafka.build_router(
+        opts[:router] || DEFAULT_ROUTER, opts)
+      schema = FFWD.parse_schema opts
+
+      producer = Output.new producer, brokers, schema, router, partitioner
+      FFWD.producing_client core.output, producer, opts
+    end
+  end
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: ffwd-kafka
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- John-John Tedro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffwd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- udoprog@spotify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/ffwd/plugin/kafka/output.rb
+- lib/ffwd/plugin/kafka/producer.rb
+- lib/ffwd/plugin/kafka/routers.rb
+- lib/ffwd/plugin/kafka/version.rb
+- lib/ffwd/plugin/kafka/partitioners.rb
+- lib/ffwd/plugin/kafka.rb
+homepage: https://github.com/spotify/ffwd
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Kafka support for FFWD.
+test_files: []