glass_octopus 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7b1a88b4eb29013db080c1e6a08c9b6f5af6fb0b8e844805074e1101554bbe7a
+  data.tar.gz: 24d003c5d814ccec622277a87385830fbc1825d327063d2e9a5f7714a58b84a4
+SHA512:
+  metadata.gz: d317a58e73e7a313ddc4cca6b2e998a1554e214c5616990f1e4e582318dfad461d102053ffce34a47cf2d90653e23de1b2c2cbdc382fff4835dfbc4078339a80
+  data.tar.gz: de289957eff59996e7055f789c34867146ecf07617dadc80b13b4dc25a7f7a4afaedd7e4bbfe84b4ef61728d4c668401a2c3d114cf3b92a0f0f6f1823a1ca949

data/.env

@@ -0,0 +1,3 @@
+KAFKA_0_8_PORT=9092
+KAFKA_0_10_PORT=9093
+ZOOKEEPER_EXTERNAL_PORT=2181

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.yardopts

@@ -0,0 +1 @@
+--hide-api private

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem "poseidon", github: "bpot/poseidon"
+gem "poseidon_cluster", github: "bsm/poseidon_cluster"
+gem "ruby-kafka"

data/Guardfile

@@ -0,0 +1,7 @@
+notification :terminal_notifier
+
+guard :minitest, all_on_start: false do
+  watch(%r{^test/(.*)\/?(.*)_test\.rb$})
+  watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/#{m[1]}#{m[2]}_test.rb" }
+  watch(%r{^test/test_helper\.rb$})      { 'test' }
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Secret Sauce Partners, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# Glass Octopus
+
+A Kafka consumer framework. Like Rack but for Kafka.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'glass_octopus'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install glass_octopus
+
+This gem requires Ruby 2.1 or higher.
+
+## Getting started
+
+Pick your adapter:
+
+* For Kafka 0.8.x use poseidon and poseidon-cluster
+
+        # in your Gemfile
+        gem "glass_octopus"
+        gem "poseidon", github: "bpot/poseidon"
+        gem "poseidon_cluster", github: "bsm/poseidon_cluster"
+
+* For Kafka 0.9+ use ruby-kafka
+
+        # in your Gemfile
+        gem "glass_octopus"
+        gem "ruby-kafka"
+
+
+```ruby
+# in app.rb
+require "bundler/setup"
+require "glass_octopus"
+
+app = GlassOctopus.build do
+  use GlassOctopus::Middleware::CommonLogger
+
+  run Proc.new { |ctx|
+    puts "Got message: #{ctx.message.key} => #{ctx.message.value}"
+  }
+end
+
+GlassOctopus.run(app) do |config|
+  config.adapter :ruby_kafka do |kafka|
+    kafka.broker_list = %[localhost:9092]
+    kafka.topic       = "mytopic"
+    kafka.group       = "mygroup"
+    kafka.client      = { logger: config.logger }
+  end
+end
+```
+
+Run it with `bundle exec ruby app.rb`
+
+For more examples look into the [examples](examples) directory.
+
+For the API documentation please see the [documentation site][rubydoc]
+
+## Development
+
+Install docker and docker-compose to run Kafka and zookeeper for tests.
+
+1. Set the `ADVERTISED_HOST` environment variable
+2. Run `rake docker:up`
+3. Now you can run the tests.
+
+Run all tests including integration tests:
+
+    $ rake test:all
+
+Running tests without integration tests:
+
+    $ rake # or rake test
+
+When you are done run `rake docker:down` to clean up docker containers.
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](http://opensource.org/licenses/MIT).
+
+[rubydoc]: http://www.rubydoc.info/github/sspinc/glass-octopus

data/Rakefile

@@ -0,0 +1,77 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+  t.verbose = ENV.key?("VERBOSE")
+  t.warning = false
+end
+
+namespace :test do
+  desc "Run all tests including integration tests"
+  task :all do
+    ENV["TEST_KAFKA_INTEGRATION"] = "yes"
+    Rake::Task[:test].invoke
+  end
+end
+
+namespace :docker do
+  require "socket"
+
+  desc "Start docker containers"
+  task :up do
+    start
+    wait(9093)
+    docker_compose("run --rm kafka_0_10 kafka-topics.sh --zookeeper zookeeper --create --topic test_topic --replication-factor 1 --partitions 1")
+    wait(9092)
+    docker_compose("run --rm kafka_0_8 bash -c '$KAFKA_HOME/bin/kafka-topics.sh --zookeeper kafka_0_8 --create --topic test_topic --replication-factor 1 --partitions 1'")
+  end
+
+  desc "Stop and remove docker containers"
+  task :down do
+    docker_compose("down")
+  end
+
+  desc "Reset docker containers"
+  task :reset => [:down, :up]
+
+  def start
+    docker_compose("up -d")
+  end
+
+  def stop
+    docker_compose("down")
+  end
+
+  def docker_compose(args)
+    env = {
+      "ADVERTISED_HOST"          => docker_machine_ip,
+      "KAFKA_0_8_EXTERNAL_PORT"  => "9092",
+      "KAFKA_0_10_EXTERNAL_PORT" => "9093",
+      "ZOOKEEPER_EXTERNAL_PORT"  => "2181",
+    }
+    system(env, "docker-compose #{args}")
+  end
+
+  def docker_machine_ip
+    return @docker_ip if defined? @docker_ip
+
+    if ENV.key?("ADVERTISED_HOST")
+      @docker_ip = ENV["ADVERTISED_HOST"]
+    else
+      active = %x{docker-machine active}.chomp
+      @docker_ip = %x{docker-machine ip #{active}}.chomp
+    end
+  end
+
+  def wait(port)
+    Socket.tcp(docker_machine_ip, port, connect_timeout: 5).close
+  rescue Errno::ECONNREFUSED
+    puts "waiting for #{docker_machine_ip}:#{port}"
+    sleep 1
+    retry
+  end
+end

data/bin/guard

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'guard' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("guard", "guard")

data/bin/rake

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3'
+services:
+  kafka_0_8:
+    image: sspinc/kafka
+    environment:
+      - ADVERTISED_HOST=${ADVERTISED_HOST}
+      - ADVERTISED_PORT=${KAFKA_0_8_PORT}
+    ports:
+      - ${KAFKA_0_8_PORT}:${KAFKA_0_8_PORT}
+      - ${ZOOKEEPER_EXTERNAL_PORT}:2181
+  kafka_0_10:
+    image: ches/kafka:0.10.2.1
+    environment:
+      - KAFKA_ADVERTISED_HOST_NAME=${ADVERTISED_HOST}
+      - KAFKA_ADVERTISED_PORT=${KAFKA_0_10_PORT}
+      - KAFKA_PORT=${KAFKA_0_10_PORT}
+      - ZOOKEEPER_IP=zookeeper
+    ports:
+      - ${KAFKA_0_10_PORT}:${KAFKA_0_10_PORT}
+    links:
+      - zookeeper
+  zookeeper:
+    image: zookeeper:3.4
+    expose:
+      - "2181"

data/example/advanced.rb

@@ -0,0 +1,35 @@
+require "bundler/setup"
+require "glass_octopus"
+
+app = GlassOctopus.build do
+  use GlassOctopus::Middleware::CommonLogger
+
+  run Proc.new { |ctx|
+    puts "Got message: #{ctx.message.key} => #{ctx.message.value}"
+  }
+end
+
+def array_from_env(key, default:)
+  return default unless ENV.key?(key)
+  ENV.fetch(key).split(",").map(&:strip)
+end
+
+GlassOctopus.run(app) do |config|
+  config.logger = Logger.new("glass_octopus.log")
+
+  config.adapter :poseidon do |kafka_config|
+    kafka_config.broker_list    = array_from_env("KAFKA_BROKER_LIST", default: %w[localhost:9092])
+    kafka_config.zookeeper_list = array_from_env("ZOOKEEPER_LIST", default: %w[localhost:2181])
+    kafka_config.topic          = ENV.fetch("KAFKA_TOPIC", "mytopic")
+    kafka_config.group          = ENV.fetch("KAFKA_GROUP", "mygroup")
+    kafka_config.logger         = config.logger
+  end
+
+  config.executor = Concurrent::ThreadPoolExecutor.new(
+    max_threads: 25,
+    min_threads: 7
+  )
+
+  config.shutdown_timeout = 30
+end
+

data/example/basic.rb

@@ -0,0 +1,24 @@
+require "bundler/setup"
+require "glass_octopus"
+
+app = GlassOctopus.build do
+  use GlassOctopus::Middleware::CommonLogger
+
+  run Proc.new { |ctx|
+    puts "Got message: #{ctx.message.key} => #{ctx.message.value}"
+  }
+end
+
+def array_from_env(key, default:)
+  return default unless ENV.key?(key)
+  ENV.fetch(key).split(",").map(&:strip)
+end
+
+GlassOctopus.run(app) do |config|
+  config.adapter :poseidon do |kafka_config|
+    kafka_config.broker_list    = array_from_env("KAFKA_BROKER_LIST", default: %w[localhost:9092])
+    kafka_config.zookeeper_list = array_from_env("ZOOKEEPER_LIST", default: %w[localhost:2181])
+    kafka_config.topic          = ENV.fetch("KAFKA_TOPIC", "mytopic")
+    kafka_config.group          = ENV.fetch("KAFKA_GROUP", "mygroup")
+  end
+end

data/example/ruby_kafka.rb

@@ -0,0 +1,24 @@
+require "bundler/setup"
+require "glass_octopus"
+
+app = GlassOctopus.build do
+  use GlassOctopus::Middleware::CommonLogger
+
+  run Proc.new { |ctx|
+    puts "Got message: #{ctx.message.key} => #{ctx.message.value}"
+  }
+end
+
+def array_from_env(key, default:)
+  return default unless ENV.key?(key)
+  ENV.fetch(key).split(",").map(&:strip)
+end
+
+GlassOctopus.run(app) do |config|
+  config.adapter :ruby_kafka do |kafka|
+    kafka.broker_list = array_from_env("KAFKA_BROKER_LIST", default: %w[localhost:9092])
+    kafka.topic       = ENV.fetch("KAFKA_TOPIC", "mytopic")
+    kafka.group       = ENV.fetch("KAFKA_GROUP", "mygroup")
+    kafka.client      = { logger: config.logger }
+  end
+end

data/glass_octopus.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'glass_octopus/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "glass_octopus"
+  spec.version       = GlassOctopus::VERSION
+  spec.authors       = ["Tamás Michelberger"]
+  spec.email         = ["tomi@secretsaucepartners.com"]
+
+  spec.summary       = %q{A Kafka consumer framework. Like Rack but for Kafka.}
+  spec.homepage      = "https://github.com/sspinc/glass-octopus"
+  spec.license       = "MIT"
+
+  spec.description   = <<-EOF
+GlassOctopus provides a minimal, modular and adaptable interface for developing
+Kafka consumers in Ruby. In its philosophy it is very close to Rack.
+EOF
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.1"
+
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "minitest-color", "~> 0"
+  spec.add_development_dependency "guard", "~> 2.14"
+  spec.add_development_dependency "guard-minitest", "~> 2.4"
+  spec.add_development_dependency "terminal-notifier-guard", "~> 1.7"
+end

data/lib/glass-octopus.rb

@@ -0,0 +1 @@
+require "glass_octopus"

data/lib/glass_octopus.rb

@@ -0,0 +1,50 @@
+require "glass_octopus/version"
+require "glass_octopus/middleware"
+require "glass_octopus/runner"
+require "glass_octopus/application"
+require "glass_octopus/builder"
+
+module GlassOctopus
+  # Run an application. The application can be anything that responds to
+  # +#call+. It is invoked with with a context that has the message and other
+  # goodies.
+  #
+  # @param app [#call] application to process messages
+  # @param runner [#run] a runner that takes care of running and shutting down
+  #   the application. See {Runner} for more details.
+  # @yield [config] configure your application in this block, this is called
+  #   before connecting to Kafka
+  # @yieldparam config [Configuration] the configuration object
+  # @raise [ArgumentError] when no block for configuration is passed
+  def self.run(app, runner: Runner, &block)
+    raise ArgumentError, "A block must be given to set up the #{name}." unless block_given?
+    go_app = Application.new(app, &block)
+    runner.run(go_app)
+  end
+
+  # Build a middleware stack.
+  # Basically a shortcut to +Builder.new { }.to_app+
+  #
+  # @example
+  #
+  #   require "glass_octopus"
+  #
+  #   app = GlassOctopus.build do
+  #     use GlassOctopus::Middleware::CommonLogger
+  #
+  #     run Proc.new { |context|
+  #       puts "Hello, #{context.message.value}"
+  #     }
+  #   end
+  #
+  #   GlassOctopus.run(app) do |config|
+  #     # set config here
+  #   end
+  #
+  # @see Builder
+  # @yield use the builder DSL to build your middleware stack
+  # @return [#call] an application that can be fed into the {.run}
+  def self.build(&block)
+    Builder.new(&block).to_app
+  end
+end