yakc 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: aba05e366fbce78b59bfabd65a5dc506ff79db18
+  data.tar.gz: 52d04d092dc31ba17edbdeb9179e223357852ac4
+SHA512:
+  metadata.gz: 293a6fed65d496861799724c8eb3053bd6967d68de902ff23fd218a75656783956c3f6598d3f017e3a29b0d76d69955e3c6f7c9fdc888c2b6e11887ef638598f
+  data.tar.gz: f950ee60dc9132490bf70d41dc1f6e84c3e5bd9b7141318aaa010dea800bdc2c83cda5d98babcb73caf69dfd48a9886769f2868cddcc9ea23b1e59ddfdade889

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in yakc.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Greg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,183 @@
+# YAKC: Yet another Kafka (0.8) consumer
+
+YAKC is a generic Kavka 0.8 consumer based on the now-dead Poseidon (i know, i know). It will listen to as many topics as you specify and hand them off via a handler to consumer classes.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'yakc'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install yakc
+
+## Usage
+
+There are 2 main componets:
+
+### Message Handler
+
+This is the bit of code that handles what to do with the messages once they are received. There are 2 stages to this process:
+
+1. The message is parsed using *your* message parser(inherited from the `YAKC::Message` class) that does the parsing and validity checking. 
+2. The parsed message payload is broadcast to the system. You can specify your own publisher, but by default the handler will use [Yeller](http://www.github.com/gaorlov/yeller). It will broadcast the message with the key: "topic::event"
+
+To set it up:
+
+```ruby
+  handler = YAKC::MessageBroadcaster.new publisher: MyBroadcaster, message_class: MyMessageClass
+  # or, if you are okay with Yeller
+  handler = YAKC::MessageBroadcaster.new message_class: MyMessageClass
+```
+
+And now you're ready to init the [reader](#reader)
+
+#### Publisher Interface
+
+If you don't like Yeller, or want something that can talk cross-process, you can implement your own.
+
+The publisher interface is pretty simple: it has to implement
+* `broadcast( message, topic )` : This is the function that handles where the messages go.
+ 
+#### Message Interface
+
+The message parser needs to implement:
+
+1. `parse( raw_message )` : This converts the raw Kafka data to the format of your choice
+2. `broadcastable?` : This determines whether the message is valid and shoud be broadcast.
+3: `event` : The name of the picked up event. This is the name that gets broadcast 
+
+For example if your messages are encoded in Avro and look loosely like:
+```json
+{ "event": {"name":"myEventName",
+            "timestamp":"00:00:00:12/12/12"}},
+  "my_field":"value",
+  // etc
+}
+```
+
+Your message parser class would look something like
+
+```ruby
+class AvroMessage < YAKC::Message
+  
+  def broadcastable?
+    # an event is probably okay to transmit if we can extract its name
+    event["name"]
+  end
+
+  def event
+    @payload["event"] || {}
+  end
+
+  protected
+
+  def parse( message )
+    data = StringIO.new(message.value)
+    msg = Avro::DataFile::Reader.new(data, Avro::IO::DatumReader.new)
+    
+  rescue Avro::DataFile::DataFileError => e
+    Rails.logger.error e
+    {}
+  end
+end
+
+```
+
+
+### Reader
+
+The reader does(surprise) the reading and pushes the read rad messages to the handler, which you have to specify.
+
+It implements:
+
+* `read` : an infinite loop that consumes messages on all the specified topics (see [setup](#setup) below) and sends them to the handler
+
+Here's how you would use it:
+
+```ruby
+  handler = YAKC::MessageBroadcaster.new message_class: AvroMessage
+  reader = YAKC::Reader.new message_handler: handler
+
+  reader.read
+```
+
+## Setup
+
+And now for the full setup. You will need to specify the `zookeepers`; the Kafka `brokers`; the `app` and `suffix`, which are used to generate the consumer group name; the topic list; and a logger. 
+
+There are 2 ways of doing this. You can either set those up as ENV vars ("ZOOKEEPERS"(comma separated list), "BROKERS"(comma separated list), "APP", "SUFFIX", "TOPICS") and set up the logger by hand, or, you can do it in ruby, like:
+
+```ruby
+  YAKC.configure do |config|
+    config.logger     = Rails.logger
+    config.zookeepers = ["localhost:9092"]
+    config.brokers    = ["localhost:2181"]
+    config.app        = "MyApp"
+    config.suffix     = Rails.env
+    config.topics     = ["clickstream", "logs", "exceptions"] # whatever you're listening for
+  end
+```
+
+## Example
+
+Here's what a full experience would look like:
+
+The reader would look like
+```ruby
+  # in your initializer
+  YAKC.configure do |config|
+    # we'll assume the rest are set up in the env
+    config.logger     = Rails.logger
+  end
+```
+
+In your reader job
+
+```ruby
+  handler = YAKC::MessageBroadcaster.new message_class: AvroMessage
+  reader = YAKC::Reader.new message_handler: handler
+
+  reader.read
+```
+
+And the consumers would listen to the events
+
+Let's say you have an app that listens to exceptions that we pass around in kafka. It then stores them in a DB and passesthem off to Honeybadger. Your `Exception` model could do something like
+
+```ruby
+class Exception < ActiveRecord::Base
+  include Yeller::Subscribable
+
+  # we don't care about the event type, so we subscribe to "exception::.*"
+  subscribe with: :from_kafka_event, to: "exception::.*"
+
+  def self.from_kafka_event( message )
+    create message
+    Honeybadger.notify message
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/gaorlov/yakc.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "yakc"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/yakc/configuration.rb

@@ -0,0 +1,14 @@
+module YAKC
+  class Configuration
+    attr_accessor :zookeepers, :brokers, :app, :suffix, :topics, :logger
+
+    def initialize
+      logger      = Logger.new(STDOUT)
+      brokers     = ENV.fetch("BROKERS", "localhost:9092").split(",")
+      zookeepers  = ENV.fetch("ZOOKEEPERS", "localhost:2181").split(",") 
+      app         = ENV["APP"]
+      suffix      = ENV["SUFFIX"]
+      topics      = ENV["TOPICS"]
+    end
+  end
+end

data/lib/yakc/fallthrough_instrumenter.rb

@@ -0,0 +1,10 @@
+module YAKC
+  module Instrumenter
+    class FallthroughInstrumenter
+
+      def instrument( message )
+        yield
+      end
+    end
+  end
+end

data/lib/yakc/message.rb

@@ -0,0 +1,23 @@
+module YAKC
+  class Message
+    attr_reader :payload
+
+    def initialize( message )
+      @payload = parse( message )
+    end
+
+    def broadcastable?
+      # implement me
+    end
+
+    def event
+      # implement me
+    end
+
+    protected
+
+    def parse( message )
+      message
+    end
+  end
+end

data/lib/yakc/message_broadcaster.rb

@@ -0,0 +1,29 @@
+module YAKC
+  class MessageBroadcaster
+    attr_accessor :publisher, :message_class, :instrumenter
+
+    def initialize( publisher: Yeller, instrumenter: FallthroughInstrumenter, message_parser: )
+      @publisher        = publisher
+      @message_class    = message_parser
+      @instrumenter     = instrumenter.new
+      raise "MessageBroadcaster must have a valid message class" unless @message_class
+    end
+
+    def handle( topic, message )
+      msg = @message_class.new( message )
+
+      @instrumenter.instrument( msg ) do 
+        if msg.broadcastable?
+          # broadcast the specific topic event 
+          @publisher.broadcast msg.payload, broadcast_key( topic, msg.event )        
+        end
+      end
+    end
+
+    private
+
+    def broadcast_key( topic, event )
+      "#{topic}::#{event}"
+    end
+  end
+end

data/lib/yakc/reader.rb

@@ -0,0 +1,45 @@
+module YAKC
+  class Reader
+    attr_reader :message_handler, :terminated
+
+    def initialize( message_handler: )
+      @message_handler  = message_handler
+      @config           = YAKC.configuration      
+
+      raise KeyError, "YAKC::Reader initialized without a message handler. Please specify one so that your receives messages don't end up on the floor. For more info, go to: https://github.com/gaorlov/yakc#message-handler" unless message_handler
+      
+      Signal.trap("INT") do
+        @terminated = true
+      end
+    end
+
+    def read
+      loop do
+        consumers.map do |consumer|
+          consumer.fetch do |partition, bulk|
+            bulk.each do |message|
+              message_handler.handle topic, message
+            end
+          end
+          return if terminated
+        end
+      end
+    rescue => e
+      YACK.logger.error e
+      retry
+    end
+
+    private
+
+    def consumers
+      @consumers ||= topics.map do |topic|
+        Poseidon::ConsumerGroup.new(
+        "#{app}-#{topic}-consumer-#{suffix}", 
+        @config.brokers,
+        @config.zookeepers, 
+        topic,
+        {})
+      end
+    end
+  end
+end

data/lib/yakc/version.rb

@@ -0,0 +1,3 @@
+module Yakc
+  VERSION = "0.1.0"
+end

data/lib/yakc.rb

@@ -0,0 +1,26 @@
+require "yakc/version"
+require 'active_support'
+require 'active_support/core_ext/module/delegation'
+require 'yeller'
+
+module YAKC
+  autoload :Configuration,            'yakc/configuration'
+  autoload :FallthroughInstrumenter,  'yakc/fallthorugh_instrumenter'
+  autoload :MessageBroadcaster,       'yakc/message_broadcaster'
+  autoload :Reader,                   'yakc/reader'
+  autoload :Message,                  'yakc/message'
+
+  class << self
+    attr_writer :configuration
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+  delegate :logger, to: :configuration
+end

data/yakc.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'yakc/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "yakc"
+  spec.version       = Yakc::VERSION
+  spec.authors       = ["Greg"]
+  spec.email         = ["greg@avvo.com"]
+
+  spec.summary       = "Multitopic Poseidon-base Kafaka consumer"
+  spec.homepage      = "http://www.github.com/gaorlov/yakc"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "yeller"
+  spec.add_dependency "poseidon_cluster", "0.3.2.avvo3"
+  spec.add_dependency "activesupport"
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "simplecov"
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: yakc
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Greg
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-12-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yeller
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poseidon_cluster
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.2.avvo3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.2.avvo3
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- greg@avvo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/yakc.rb
+- lib/yakc/configuration.rb
+- lib/yakc/fallthrough_instrumenter.rb
+- lib/yakc/message.rb
+- lib/yakc/message_broadcaster.rb
+- lib/yakc/reader.rb
+- lib/yakc/version.rb
+- yakc.gemspec
+homepage: http://www.github.com/gaorlov/yakc
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Multitopic Poseidon-base Kafaka consumer
+test_files: []
+has_rdoc: