glass_octopus 1.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b1a88b4eb29013db080c1e6a08c9b6f5af6fb0b8e844805074e1101554bbe7a
-  data.tar.gz: 24d003c5d814ccec622277a87385830fbc1825d327063d2e9a5f7714a58b84a4
+  metadata.gz: a76c86f04773d07d518c89bb377c122803caea7cf57fc864f2a1a23dfe755024
+  data.tar.gz: fc0f807a5fc32dc6e41e368da42417b7cc36f055bdb78f4c68b861bdf0d9633b
 SHA512:
-  metadata.gz: d317a58e73e7a313ddc4cca6b2e998a1554e214c5616990f1e4e582318dfad461d102053ffce34a47cf2d90653e23de1b2c2cbdc382fff4835dfbc4078339a80
-  data.tar.gz: de289957eff59996e7055f789c34867146ecf07617dadc80b13b4dc25a7f7a4afaedd7e4bbfe84b4ef61728d4c668401a2c3d114cf3b92a0f0f6f1823a1ca949
+  metadata.gz: 6745d2aca4945d4cfa3b41084bdb8a00af9cfcc4f0c78e662b49b7d6a6be4cda48dbe060d8059b447e6d0633638529e649554f5895283c9930ec68c8917e6929
+  data.tar.gz: c1bf27a0be7757fa3a14bb51ac3c4f877763114e8a158246cef4698243633b2afd476793deafb4c1158a025d32e2efe6ed6f1f8732b303697f92aa4aff19d5fd

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.swp

data/Gemfile

@@ -1,7 +1,3 @@
 source "https://rubygems.org"
 
 gemspec
-
-gem "poseidon", github: "bpot/poseidon"
-gem "poseidon_cluster", github: "bsm/poseidon_cluster"
-gem "ruby-kafka"

data/README.md

@@ -24,19 +24,13 @@ This gem requires Ruby 2.1 or higher.
 
 Pick your adapter:
 
-* For Kafka 0.8.x use poseidon and poseidon-cluster
-
-        # in your Gemfile
-        gem "glass_octopus"
-        gem "poseidon", github: "bpot/poseidon"
-        gem "poseidon_cluster", github: "bsm/poseidon_cluster"
-
-* For Kafka 0.9+ use ruby-kafka
+* ruby-kafka
 
         # in your Gemfile
         gem "glass_octopus"
         gem "ruby-kafka"
 
+Currently only `ruby-kafka` is supported out of the box. If you need to use another adapter you can pass a class to `config.adapter`. See documentation for `GlassOctopus::Configuration#adapter`.
 
 ```ruby
 # in app.rb
@@ -55,25 +49,125 @@ GlassOctopus.run(app) do |config|
   config.adapter :ruby_kafka do |kafka|
     kafka.broker_list = %[localhost:9092]
     kafka.topic       = "mytopic"
-    kafka.group       = "mygroup"
-    kafka.client      = { logger: config.logger }
+    kafka.group_id    = "mygroup"
+    kafka.client_id   = "myapp"
   end
 end
 ```
 
 Run it with `bundle exec ruby app.rb`
 
-For more examples look into the [examples](examples) directory.
+For more examples look into the [example](example) directory.
 
 For the API documentation please see the [documentation site][rubydoc]
 
+### Handling Avro messages with Schema Registry
+
+Glass Octopus can be used with Avro messages validated against a schema. For this, you need a running [Schema Registry](https://docs.confluent.io/current/schema-registry/docs/index.html) service.
+You also need to have the `avro_turf` gem installed.
+
+```ruby
+# in your Gemfile
+gem "avro_turf"
+```
+
+Add the `AvroParser` middleware with the Schema Registry URL to your app.
+
+```ruby
+# in app.rb
+app = GlassOctopus.build do
+  use GlassOctopus::Middleware::AvroParser, "http://schema_registry_url:8081"
+  # ...
+end
+```
+
+### Supported middleware
+
+* ActiveRecord
+
+    Return any active connection to the pool after the message has been processed.
+
+    ```ruby
+    app = GlassOctopus.build do
+      use GlassOctopus::Middleware::ActiveRecord
+      # ...
+    end
+    ```
+
+* New Relic
+
+    Record message processing as background transactions. Also captures uncaught exceptions.
+
+    ```ruby
+    app = GlassOctopus.build do
+      use GlassOctopus::Middleware::NewRelic, MyConsumer
+      # ...
+    end
+    ```
+
+* Sentry
+
+    Report uncaught exceptions to Sentry.
+
+    ```ruby
+    app = GlassOctopus.build do
+      use GlassOctopus::Middleware::Sentry
+      # ...
+    end
+    ```
+
+* Common logger
+
+    Log processed messages and runtime of the processing.
+
+    ```ruby
+    app = GlassOctopus.build do
+      use GlassOctopus::Middleware::CommonLogger
+      # ...
+    end
+    ```
+
+* Parse messages as JSON
+
+    Parse message value as JSON. The resulting hash is placed in `context.params`.
+
+    ```ruby
+    app = GlassOctopus.build do
+      use GlassOctopus::Middleware::JsonParser
+      # ...
+      run MyConsumer
+    end
+
+    class MyConsumer
+      def call(ctx)
+        puts ctx.params # message value parsed as JSON
+        puts ctx.message # Raw unaltered message
+      end
+    end
+    ```
+
+    Optionally you can specify a class to be instantiated with the message hash.
+
+    ```ruby
+    app = GlassOctopus.build do
+      use GlassOctopus::Middleware::JsonParser, class: MyMessage
+      run MyConsumer
+    end
+
+    class MyMessage
+      def initialize(attributes)
+        attributes.each { |k,v| public_send("#{k}=", v) }
+      end
+    end
+    ```
+
 ## Development
 
-Install docker and docker-compose to run Kafka and zookeeper for tests.
+Install docker and docker-compose to run Kafka and Zookeeper for tests.
+
+Start Kafka and Zookeeper
 
-1. Set the `ADVERTISED_HOST` environment variable
-2. Run `rake docker:up`
-3. Now you can run the tests.
+    $ docker-compose up
 
 Run all tests including integration tests:
 
@@ -83,7 +177,7 @@ Running tests without integration tests:
 
     $ rake # or rake test
 
-When you are done run `rake docker:down` to clean up docker containers.
+When you are done run `docker-compose down` to clean up docker containers.
 
 ## License
 

data/Rakefile

@@ -17,61 +17,3 @@ namespace :test do
     Rake::Task[:test].invoke
   end
 end
-
-namespace :docker do
-  require "socket"
-
-  desc "Start docker containers"
-  task :up do
-    start
-    wait(9093)
-    docker_compose("run --rm kafka_0_10 kafka-topics.sh --zookeeper zookeeper --create --topic test_topic --replication-factor 1 --partitions 1")
-    wait(9092)
-    docker_compose("run --rm kafka_0_8 bash -c '$KAFKA_HOME/bin/kafka-topics.sh --zookeeper kafka_0_8 --create --topic test_topic --replication-factor 1 --partitions 1'")
-  end
-
-  desc "Stop and remove docker containers"
-  task :down do
-    docker_compose("down")
-  end
-
-  desc "Reset docker containers"
-  task :reset => [:down, :up]
-
-  def start
-    docker_compose("up -d")
-  end
-
-  def stop
-    docker_compose("down")
-  end
-
-  def docker_compose(args)
-    env = {
-      "ADVERTISED_HOST"          => docker_machine_ip,
-      "KAFKA_0_8_EXTERNAL_PORT"  => "9092",
-      "KAFKA_0_10_EXTERNAL_PORT" => "9093",
-      "ZOOKEEPER_EXTERNAL_PORT"  => "2181",
-    }
-    system(env, "docker-compose #{args}")
-  end
-
-  def docker_machine_ip
-    return @docker_ip if defined? @docker_ip
-
-    if ENV.key?("ADVERTISED_HOST")
-      @docker_ip = ENV["ADVERTISED_HOST"]
-    else
-      active = %x{docker-machine active}.chomp
-      @docker_ip = %x{docker-machine ip #{active}}.chomp
-    end
-  end
-
-  def wait(port)
-    Socket.tcp(docker_machine_ip, port, connect_timeout: 5).close
-  rescue Errno::ECONNREFUSED
-    puts "waiting for #{docker_machine_ip}:#{port}"
-    sleep 1
-    retry
-  end
-end

data/docker-compose.yml

@@ -1,25 +1,22 @@
 version: '3'
 services:
-  kafka_0_8:
-    image: sspinc/kafka
+  kafka:
+    image: confluentinc/cp-kafka:5.0.1
     environment:
-      - ADVERTISED_HOST=${ADVERTISED_HOST}
-      - ADVERTISED_PORT=${KAFKA_0_8_PORT}
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:32181
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:29092
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
     ports:
-      - ${KAFKA_0_8_PORT}:${KAFKA_0_8_PORT}
-      - ${ZOOKEEPER_EXTERNAL_PORT}:2181
-  kafka_0_10:
-    image: ches/kafka:0.10.2.1
-    environment:
-      - KAFKA_ADVERTISED_HOST_NAME=${ADVERTISED_HOST}
-      - KAFKA_ADVERTISED_PORT=${KAFKA_0_10_PORT}
-      - KAFKA_PORT=${KAFKA_0_10_PORT}
-      - ZOOKEEPER_IP=zookeeper
-    ports:
-      - ${KAFKA_0_10_PORT}:${KAFKA_0_10_PORT}
-    links:
+      - 29092:29092
+    depends_on:
       - zookeeper
   zookeeper:
-    image: zookeeper:3.4
+    image: confluentinc/cp-zookeeper:5.0.1
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=32181
+      - ZOOKEEPER_TICK_TIME=2000
+      - ZOOKEEPER_SYNC_LIMIT=2
     expose:
-      - "2181"
+      - 32181/tcp
+    ports:
+      - 32181:32181

data/example/{ruby_kafka.rb → avro.rb}

@@ -3,6 +3,7 @@ require "glass_octopus"
 
 app = GlassOctopus.build do
   use GlassOctopus::Middleware::CommonLogger
+  use GlassOctopus::Middleware::AvroParser, "http://schema_registry_url:8081"
 
   run Proc.new { |ctx|
     puts "Got message: #{ctx.message.key} => #{ctx.message.value}"
@@ -14,11 +15,14 @@ def array_from_env(key, default:)
   ENV.fetch(key).split(",").map(&:strip)
 end
 
+
 GlassOctopus.run(app) do |config|
+  config.logger = Logger.new(STDOUT)
+
   config.adapter :ruby_kafka do |kafka|
     kafka.broker_list = array_from_env("KAFKA_BROKER_LIST", default: %w[localhost:9092])
     kafka.topic       = ENV.fetch("KAFKA_TOPIC", "mytopic")
-    kafka.group       = ENV.fetch("KAFKA_GROUP", "mygroup")
-    kafka.client      = { logger: config.logger }
+    kafka.group_id    = ENV.fetch("KAFKA_GROUP", "mygroup")
+    kafka.client_id   = "myapp"
   end
 end

data/example/basic.rb

@@ -15,10 +15,12 @@ def array_from_env(key, default:)
 end
 
 GlassOctopus.run(app) do |config|
-  config.adapter :poseidon do |kafka_config|
-    kafka_config.broker_list    = array_from_env("KAFKA_BROKER_LIST", default: %w[localhost:9092])
-    kafka_config.zookeeper_list = array_from_env("ZOOKEEPER_LIST", default: %w[localhost:2181])
-    kafka_config.topic          = ENV.fetch("KAFKA_TOPIC", "mytopic")
-    kafka_config.group          = ENV.fetch("KAFKA_GROUP", "mygroup")
+  config.logger = Logger.new(STDOUT)
+
+  config.adapter :ruby_kafka do |kafka|
+    kafka.broker_list = array_from_env("KAFKA_BROKER_LIST", default: %w[localhost:9092])
+    kafka.topic       = ENV.fetch("KAFKA_TOPIC", "mytopic")
+    kafka.group_id    = ENV.fetch("KAFKA_GROUP", "mygroup")
+    kafka.client_id   = "myapp"
   end
 end

data/glass_octopus.gemspec

@@ -23,12 +23,14 @@ EOF
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.1"
-
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "minitest-color", "~> 0"
   spec.add_development_dependency "guard", "~> 2.14"
   spec.add_development_dependency "guard-minitest", "~> 2.4"
   spec.add_development_dependency "terminal-notifier-guard", "~> 1.7"
+  spec.add_development_dependency "ruby-kafka", "~> 0.7.0"
+  spec.add_development_dependency "avro_turf", "~> 0.8.0"
+  spec.add_development_dependency "sinatra", ">= 2.2.0"
+  spec.add_development_dependency "webmock", "~> 3.3.0"
 end

data/lib/glass_octopus/application.rb

@@ -15,14 +15,12 @@ module GlassOctopus
     end
 
     def run
-      @consumer = Consumer.new(connection, processor, config.executor, logger)
+      @consumer = Consumer.new(connection, processor, config.logger)
       @consumer.run
     end
 
-    def shutdown(timeout=nil)
-      timeout ||= config.shutdown_timeout
-      @consumer.shutdown(timeout) if @consumer
-
+    def shutdown
+      @consumer.shutdown if @consumer
       nil
     end
 

data/lib/glass_octopus/configuration.rb

@@ -1,59 +1,81 @@
 require "logger"
-require "concurrent"
-
-require "glass_octopus/bounded_executor"
 
 module GlassOctopus
   # Configuration for the application.
   #
-  # @!attribute [rw] connection_adapter
-  #   Connection adapter that connects to the Kafka.
-  # @!attribute [rw] executor
-  #   A thread pool executor to process messages concurrently. Defaults to
-  #   a {BoundedExecutor} with 25 threads.
+  # @!attribute [r] connection_adapter
+  #   The configured connection adapter.
+  #   @see #adapter
   # @!attribute [rw] logger
   #   A standard library compatible logger for the application. By default it
   #   logs to the STDOUT.
-  # @!attribute [rw] shutdown_timeout
-  #   Number of seconds to wait for the processing to finish before shutting down.
   class Configuration
-    attr_accessor :connection_adapter,
-                  :executor,
-                  :logger,
-                  :shutdown_timeout
+    attr_accessor :logger
+    attr_reader :connection_adapter
 
     def initialize
       self.logger = Logger.new(STDOUT).tap { |l| l.level = Logger::INFO }
-      self.executor = default_executor
-      self.shutdown_timeout = 10
     end
 
-    # Creates a new adapter
+    # Configures a new adapter.
+    #
+    # When a class is passed as +type+ the class will be instantiated.
+    #
+    # @example Using a custom adapter class
+    #   config.adapter(MyAdapter) do |c|
+    #     c.bootstrap_servers = %w[localhost:9092]
+    #     c.group_id = "mygroup"
+    #     c.topic = "mytopic"
+    #   end
+    #
+    #   class MyAdapter
+    #     def initialize
+    #       @options = OpenStruct.new
+    #       yield @options
+    #     end
+    #
+    #     def fetch_message
+    #       @consumer.each do |fetched_message|
+    #         message = Message.new(
+    #           fetched_message.topic,
+    #          fetched_message.partition,
+    #          fetched_message.offset,
+    #          fetched_message.key,
+    #          fetched_message.value
+    #        )
+    #
+    #         yield message
+    #       end
+    #     end
     #
-    # @param type [:poseidon, :ruby_kafka] type of the adapter to use
+    #     def connect
+    #       # Connect to Kafka...
+    #       @consumer = ...
+    #       self
+    #     end
+    #
+    #     def close
+    #       @consumer.close
+    #     end
+    #   end
+    #
+    # @param type [:ruby_kafka, Class] type of the adapter to use
     # @yield a block to conigure the adapter
     # @yieldparam config configuration object
     #
-    # @see PoseidonAdapter
     # @see RubyKafkaAdapter
     def adapter(type, &block)
-      self.connection_adapter = build_adapter(type, &block)
-    end
-
-    # @api private
-    def default_executor
-      BoundedExecutor.new(Concurrent::FixedThreadPool.new(25), limit: 25)
+      @connection_adapter = build_adapter(type, &block)
     end
 
     # @api private
     def build_adapter(type, &block)
       case type
-      when :poseidon
-        require "glass_octopus/connection/poseidon_adapter"
-        PoseidonAdapter.new(&block)
       when :ruby_kafka
         require "glass_octopus/connection/ruby_kafka_adapter"
-        RubyKafkaAdapter.new(&block)
+        RubyKafkaAdapter.new(logger, &block)
+      when Class
+        type.new(&block)
       else
         raise ArgumentError, "Unknown adapter: #{type}"
       end

data/lib/glass_octopus/connection/ruby_kafka_adapter.rb

@@ -11,8 +11,7 @@ module GlassOctopus
   #   adapter = GlassOctopus::RubyKafkaAdapter.new do |kafka_config|
   #     kafka_config.broker_list = %w[localhost:9092]
   #     kafka_config.topic       = "mytopic"
-  #     kafka_config.group       = "mygroup"
-  #     kafka_config.kafka       = { logger: Logger.new(STDOUT) }
+  #     kafka_config.group_id    = "mygroup"
   #   end
   #
   #   adapter.connect.fetch_message do |message|
@@ -30,7 +29,8 @@ module GlassOctopus
     #
     #   * +broker_list+: list of Kafka broker addresses
     #   * +topic+: name of the topic to subscribe to
-    #   * +group+: name of the consumer group
+    #   * +group_id+: name of the consumer group
+    #   * +client_id+: the identifier for this application
     #
     #   Optional configuration:
     #
@@ -41,10 +41,13 @@ module GlassOctopus
     #   Check the ruby-kafka documentation for driver specific configurations.
     #
     # @raise [OptionsInvalid]
-    def initialize
+    def initialize(logger=nil)
       config = OpenStruct.new
       yield config
+
       @options = config.to_h
+      @options[:group_id] ||= @options[:group]
+      @options[:logger] ||= logger
       validate_options
 
       @kafka = nil
@@ -89,16 +92,16 @@ module GlassOctopus
 
     # @api private
     def connect_to_kafka
-      Kafka.new(
-        seed_brokers: options.fetch(:broker_list),
-        **options.fetch(:client, {})
-      )
+      client_options = options.fetch(:client, {}).merge(logger: @options[:logger])
+      client_options.merge!(client_id: @options[:client_id]) if @options.key?(:client_id)
+
+      Kafka.new(seed_brokers: options.fetch(:broker_list), **client_options)
     end
 
     # @api private
     def create_consumer(kafka)
       kafka.consumer(
-        group_id: options.fetch(:group),
+        group_id: options.fetch(:group_id),
         **options.fetch(:consumer, {})
       )
     end
@@ -106,8 +109,8 @@ module GlassOctopus
     # @api private
     def validate_options
       errors = []
-      [:broker_list, :group, :topic].each do |key|
-        errors << "Missing key: #{key}" unless options.key?(key)
+      [:broker_list, :group_id, :topic].each do |key|
+        errors << "Missing key: #{key}" unless options[key]
       end
 
       raise OptionsInvalid.new(errors) if errors.any?

data/lib/glass_octopus/consumer.rb

@@ -1,39 +1,33 @@
-require "glass_octopus/unit_of_work"
+require "glass_octopus/context"
 
 module GlassOctopus
   # @api private
   class Consumer
-    attr_reader :connection, :processor, :executor, :logger
+    attr_reader :connection, :processor, :logger
 
-    def initialize(connection, processor, executor, logger)
+    def initialize(connection, processor,  logger)
       @connection = connection
       @processor  = processor
-      @executor   = executor
       @logger     = logger
     end
 
     def run
       connection.fetch_message do |message|
-        work = UnitOfWork.new(message, processor, logger)
-        submit(work)
+        process_message(message)
       end
     end
 
-    def shutdown(timeout=10)
+    def shutdown
       connection.close
-      executor.shutdown
-      logger.info("Waiting for workers to terminate...")
-      executor.wait_for_termination(timeout)
     end
 
-    def submit(work)
-      if executor.post(work) { |work| work.perform }
-        logger.debug { "Accepted message: #{work.message.to_h}" }
-      else
-        logger.warn { "Rejected message: #{work.message.to_h}" }
-      end
-    rescue Concurrent::RejectedExecutionError
-      logger.warn { "Rejected message: #{work.message.to_h}" }
+    # Unit of work. Builds a context for a message and runs it through the
+    # middleware stack. It catches and logs all application level exceptions.
+    def process_message(message)
+      processor.call(Context.new(message, logger))
+    rescue => ex
+      logger.error("#{ex.class} - #{ex.message}:")
+      logger.error(ex.backtrace.join("\n")) if ex.backtrace
     end
   end
 end

data/lib/glass_octopus/middleware/avro_parser.rb

@@ -0,0 +1,35 @@
+require "delegate"
+
+begin
+  require "avro_turf/messaging"
+rescue LoadError
+  raise "Can't find 'avro_turf' gem. Please add it to your Gemfile or install it."
+end
+
+module GlassOctopus
+  module Middleware
+    class AvroParser
+      def initialize(app, schema_registry_url, options={})
+        @app = app
+        @decoder = AvroTurf::Messaging.new(registry_url: schema_registry_url, logger: options[:logger])
+      end
+
+      def call(ctx)
+        message = @decoder.decode(ctx.message.value)
+        ctx = ContextWithAvroParsedMessage.new(ctx, message)
+        @app.call(ctx)
+      end
+
+      private
+
+      class ContextWithAvroParsedMessage < SimpleDelegator
+        attr_reader :params
+
+        def initialize(wrapped_ctx, params)
+          super(wrapped_ctx)
+          @params = params
+        end
+      end
+    end
+  end
+end

data/lib/glass_octopus/middleware/common_logger.rb

@@ -23,8 +23,12 @@ module GlassOctopus
         runtime = Benchmark.realtime { yield }
         runtime *= 1000 # Convert to milliseconds
 
-        logger.send(@log_level, format(FORMAT,
-          ctx.message.topic, ctx.message.partition, ctx.message.key, runtime))
+        logger.send(@log_level) { format_message(ctx, runtime) }
+      end
+
+      def format_message(ctx, runtime)
+        format(FORMAT,
+          ctx.message.topic, ctx.message.partition, ctx.message.key, runtime)
       end
     end
   end

data/lib/glass_octopus/middleware/new_relic.rb

@@ -20,7 +20,12 @@ module GlassOctopus
       end
 
       def call(ctx)
-        perform_action_with_newrelic_trace(@options) do
+        options = @options.merge(params: {
+          topic: ctx.message.topic,
+          partition: ctx.message.partition,
+          offset: ctx.message.offset,
+        })
+        perform_action_with_newrelic_trace(options) do
           @app.call(ctx)
         end
       rescue Exception => ex

data/lib/glass_octopus/middleware.rb

@@ -3,6 +3,7 @@ module GlassOctopus
     autoload :ActiveRecord, "glass_octopus/middleware/active_record"
     autoload :CommonLogger, "glass_octopus/middleware/common_logger"
     autoload :JsonParser, "glass_octopus/middleware/json_parser"
+    autoload :AvroParser, "glass_octopus/middleware/avro_parser"
     autoload :Mongoid, "glass_octopus/middleware/mongoid"
     autoload :NewRelic, "glass_octopus/middleware/new_relic"
     autoload :Sentry, "glass_octopus/middleware/sentry"

data/lib/glass_octopus/version.rb

@@ -1,3 +1,3 @@
 module GlassOctopus
-  VERSION = "1.0.0"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,35 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: glass_octopus
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Tamás Michelberger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-13 00:00:00.000000000 Z
+date: 2022-06-01 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: concurrent-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +94,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: ruby-kafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: avro_turf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
 description: |
   GlassOctopus provides a minimal, modular and adaptable interface for developing
   Kafka consumers in Ruby. In its philosophy it is very close to Rack.
@@ -123,7 +159,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".env"
 - ".gitignore"
 - ".yardopts"
 - Gemfile
@@ -134,31 +169,28 @@ files:
 - bin/guard
 - bin/rake
 - docker-compose.yml
-- example/advanced.rb
+- example/avro.rb
 - example/basic.rb
-- example/ruby_kafka.rb
 - glass_octopus.gemspec
 - lib/glass-octopus.rb
 - lib/glass_octopus.rb
 - lib/glass_octopus/application.rb
-- lib/glass_octopus/bounded_executor.rb
 - lib/glass_octopus/builder.rb
 - lib/glass_octopus/configuration.rb
 - lib/glass_octopus/connection/options_invalid.rb
-- lib/glass_octopus/connection/poseidon_adapter.rb
 - lib/glass_octopus/connection/ruby_kafka_adapter.rb
 - lib/glass_octopus/consumer.rb
 - lib/glass_octopus/context.rb
 - lib/glass_octopus/message.rb
 - lib/glass_octopus/middleware.rb
 - lib/glass_octopus/middleware/active_record.rb
+- lib/glass_octopus/middleware/avro_parser.rb
 - lib/glass_octopus/middleware/common_logger.rb
 - lib/glass_octopus/middleware/json_parser.rb
 - lib/glass_octopus/middleware/mongoid.rb
 - lib/glass_octopus/middleware/new_relic.rb
 - lib/glass_octopus/middleware/sentry.rb
 - lib/glass_octopus/runner.rb
-- lib/glass_octopus/unit_of_work.rb
 - lib/glass_octopus/version.rb
 homepage: https://github.com/sspinc/glass-octopus
 licenses:
@@ -179,8 +211,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A Kafka consumer framework. Like Rack but for Kafka.

data/.env

@@ -1,3 +0,0 @@
-KAFKA_0_8_PORT=9092
-KAFKA_0_10_PORT=9093
-ZOOKEEPER_EXTERNAL_PORT=2181

data/example/advanced.rb

@@ -1,35 +0,0 @@
-require "bundler/setup"
-require "glass_octopus"
-
-app = GlassOctopus.build do
-  use GlassOctopus::Middleware::CommonLogger
-
-  run Proc.new { |ctx|
-    puts "Got message: #{ctx.message.key} => #{ctx.message.value}"
-  }
-end
-
-def array_from_env(key, default:)
-  return default unless ENV.key?(key)
-  ENV.fetch(key).split(",").map(&:strip)
-end
-
-GlassOctopus.run(app) do |config|
-  config.logger = Logger.new("glass_octopus.log")
-
-  config.adapter :poseidon do |kafka_config|
-    kafka_config.broker_list    = array_from_env("KAFKA_BROKER_LIST", default: %w[localhost:9092])
-    kafka_config.zookeeper_list = array_from_env("ZOOKEEPER_LIST", default: %w[localhost:2181])
-    kafka_config.topic          = ENV.fetch("KAFKA_TOPIC", "mytopic")
-    kafka_config.group          = ENV.fetch("KAFKA_GROUP", "mygroup")
-    kafka_config.logger         = config.logger
-  end
-
-  config.executor = Concurrent::ThreadPoolExecutor.new(
-    max_threads: 25,
-    min_threads: 7
-  )
-
-  config.shutdown_timeout = 30
-end
-

data/lib/glass_octopus/bounded_executor.rb

@@ -1,53 +0,0 @@
-require "delegate"
-require "concurrent"
-
-module GlassOctopus
-  # BoundedExecutor wraps an existing executor implementation and provides
-  # throttling for job submission. It delegates every method to the wrapped
-  # executor.
-  #
-  # Implementation is based on the Java Concurrency In Practice book. See:
-  # http://jcip.net/listings/BoundedExecutor.java
-  #
-  # @example
-  #   pool = BoundedExecutor.new(Concurrent::FixedThreadPool.new(2), 2)
-  #
-  #   pool.post { puts "something time consuming" }
-  #   pool.post { puts "something time consuming" }
-  #
-  #   # This will block until the other submitted jobs are done.
-  #   pool.post { puts "something time consuming" }
-  #
-  class BoundedExecutor < SimpleDelegator
-    # @param executor the executor implementation to wrap
-    # @param limit [Integer] maximum number of jobs that can be submitted
-    def initialize(executor, limit:)
-      super(executor)
-      @semaphore = Concurrent::Semaphore.new(limit)
-    end
-
-    # Submit a task to the executor for asynchronous processing. If the
-    # submission limit is reached {#post} will block until there is a free
-    # worker to accept the new task.
-    #
-    # @param args [Array] arguments to pass to the task
-    # @return [Boolean] +true+ if the task was accepted, false otherwise
-    def post(*args, &block)
-      return false unless running?
-
-      @semaphore.acquire
-      begin
-        __getobj__.post(args, block) do |args, block|
-          begin
-            block.call(*args)
-          ensure
-            @semaphore.release
-          end
-        end
-      rescue Concurrent::RejectedExecutionError
-        @semaphore.release
-        raise
-      end
-    end
-  end
-end

data/lib/glass_octopus/connection/poseidon_adapter.rb

@@ -1,113 +0,0 @@
-require "ostruct"
-require "poseidon_cluster"
-require "glass_octopus/message"
-require "glass_octopus/connection/options_invalid"
-
-module GlassOctopus
-  # Connection adapter that uses the {https://github.com/bpot/poseidon poseidon
-  # gem} to talk to Kafka 0.8.x. Tested with Kafka 0.8.2.
-  #
-  # @example
-  #   adapter = GlassOctopus::PoseidonAdapter.new do |config|
-  #     config.broker_list    = %w[localhost:9092]
-  #     config.zookeeper_list = %w[localhost:2181]
-  #     config.topic          = "mytopic"
-  #     config.group          = "mygroup"
-  #
-  #     require "logger"
-  #     config.logger = Logger.new(STDOUT)
-  #   end
-  #
-  #   adapter.connect.fetch_message do |message|
-  #     p message
-  #   end
-  class PoseidonAdapter
-    # @yield configure poseidon in the yielded block
-    #   The following configuration values are required:
-    #
-    #   * +broker_list+: list of Kafka broker addresses
-    #   * +zookeeper_list+: list of Zookeeper addresses
-    #   * +topic+: name of the topic to subscribe to
-    #   * +group+: name of the consumer group
-    #
-    #   Any other configuration value is passed to
-    #   {http://www.rubydoc.info/github/bsm/poseidon_cluster/Poseidon/ConsumerGroup Poseidon::ConsumerGroup}.
-    #
-    # @raise [OptionsInvalid]
-    def initialize
-      @poseidon_consumer = nil
-      @closed = false
-
-      config = OpenStruct.new
-      yield config
-
-      @options = config.to_h
-      validate_options
-    end
-
-    # Connect to Kafka and Zookeeper, register the consumer group.
-    # This also initiates a rebalance in the consumer group.
-    def connect
-      @closed = false
-      @poseidon_consumer = create_consumer_group
-      self
-    end
-
-    # Fetch messages from kafka in a loop.
-    #
-    # @yield messages read from Kafka
-    # @yieldparam message [Message] a Kafka message
-    def fetch_message
-      @poseidon_consumer.fetch_loop do |partition, messages|
-        break if closed?
-
-        messages.each do |message|
-          yield build_message(partition, message)
-        end
-
-        # Return true to auto-commit offset to Zookeeper
-        true
-      end
-    end
-
-    # Close the connection and stop the {#fetch_message} loop.
-    def close
-      @closed = true
-      @poseidon_consumer.close if @poseidon_consumer
-      @poseidon_cluster = nil
-    end
-
-    # @api private
-    def closed?
-      @closed
-    end
-
-    # @api private
-    def create_consumer_group
-      options = @options.dup
-
-      Poseidon::ConsumerGroup.new(
-        options.delete(:group),
-        options.delete(:broker_list),
-        options.delete(:zookeeper_list),
-        options.delete(:topic),
-        { :max_wait_ms => 1000 }.merge(options)
-      )
-    end
-
-    # @api private
-    def build_message(partition, message)
-      GlassOctopus::Message.new(message.topic, partition, message.offset, message.key, message.value)
-    end
-
-    # @api private
-    def validate_options
-      errors = []
-      [:group, :broker_list, :zookeeper_list, :topic].each do |key|
-        errors << "Missing key: #{key}" unless @options.key?(key)
-      end
-
-      raise OptionsInvalid.new(errors) if errors.any?
-    end
-  end
-end

data/lib/glass_octopus/unit_of_work.rb

@@ -1,24 +0,0 @@
-require "glass_octopus/context"
-
-module GlassOctopus
-  # Unit of work. Builds a context for a message and runs it through the
-  # middleware stack. It catches and logs all application level exceptions.
-  #
-  # @api private
-  class UnitOfWork
-    attr_reader :message, :processor, :logger
-
-    def initialize(message, processor, logger)
-      @message   = message
-      @processor = processor
-      @logger    = logger
-    end
-
-    def perform
-      processor.call(Context.new(message, logger))
-    rescue => ex
-      logger.logger.error("#{ex.class} - #{ex.message}:")
-      logger.logger.error(ex.backtrace.join("\n")) if ex.backtrace
-    end
-  end
-end