glass_octopus 1.0.0

data/lib/glass_octopus/application.rb

@@ -0,0 +1,37 @@
+require "glass_octopus/consumer"
+require "glass_octopus/configuration"
+
+module GlassOctopus
+  # @api private
+  class Application
+    attr_reader :config, :processor
+
+    def initialize(processor)
+      @processor = processor
+      @config    = Configuration.new
+      @consumer  = nil
+
+      yield @config
+    end
+
+    def run
+      @consumer = Consumer.new(connection, processor, config.executor, logger)
+      @consumer.run
+    end
+
+    def shutdown(timeout=nil)
+      timeout ||= config.shutdown_timeout
+      @consumer.shutdown(timeout) if @consumer
+
+      nil
+    end
+
+    def logger
+      config.logger
+    end
+
+    def connection
+      config.connection_adapter.connect
+    end
+  end
+end

data/lib/glass_octopus/bounded_executor.rb

@@ -0,0 +1,53 @@
+require "delegate"
+require "concurrent"
+
+module GlassOctopus
+  # BoundedExecutor wraps an existing executor implementation and provides
+  # throttling for job submission. It delegates every method to the wrapped
+  # executor.
+  #
+  # Implementation is based on the Java Concurrency In Practice book. See:
+  # http://jcip.net/listings/BoundedExecutor.java
+  #
+  # @example
+  #   pool = BoundedExecutor.new(Concurrent::FixedThreadPool.new(2), 2)
+  #
+  #   pool.post { puts "something time consuming" }
+  #   pool.post { puts "something time consuming" }
+  #
+  #   # This will block until the other submitted jobs are done.
+  #   pool.post { puts "something time consuming" }
+  #
+  class BoundedExecutor < SimpleDelegator
+    # @param executor the executor implementation to wrap
+    # @param limit [Integer] maximum number of jobs that can be submitted
+    def initialize(executor, limit:)
+      super(executor)
+      @semaphore = Concurrent::Semaphore.new(limit)
+    end
+
+    # Submit a task to the executor for asynchronous processing. If the
+    # submission limit is reached {#post} will block until there is a free
+    # worker to accept the new task.
+    #
+    # @param args [Array] arguments to pass to the task
+    # @return [Boolean] +true+ if the task was accepted, false otherwise
+    def post(*args, &block)
+      return false unless running?
+
+      @semaphore.acquire
+      begin
+        __getobj__.post(args, block) do |args, block|
+          begin
+            block.call(*args)
+          ensure
+            @semaphore.release
+          end
+        end
+      rescue Concurrent::RejectedExecutionError
+        @semaphore.release
+        raise
+      end
+    end
+  end
+end

data/lib/glass_octopus/builder.rb

@@ -0,0 +1,115 @@
+require "glass_octopus/middleware/common_logger"
+
+module GlassOctopus
+  # GlassOctopus::Builder is a small DLS to build processing pipelines. It is
+  # very similar to Rack::Builder.
+  #
+  # Middleware can be a class with a similar signature to Rack middleware. The
+  # constructor needs to take an +app+ object which is basically the next
+  # middleware in the stack and the instance of the class has to respond to
+  # +#call(ctx)+.
+  #
+  # Lambdas/procs can also be used as middleware. In this case the middleware
+  # does not have control over the execution of the next middleware: the next
+  # one is *always* called.
+  #
+  # @example
+  #   require "glass_octopus"
+  #
+  #   app = GlassOctopus::Builder.new do
+  #     use GlassOctopus::Middleware::CommonLogger
+  #     use GlassOctopus::Middleware::JsonParser
+  #
+  #     run Proc.new { |ctx|
+  #       puts "Hello, #{ctx.params['name']}"
+  #     }
+  #   end.to_app
+  #
+  #   GlassOctopus.run(app) do |config|
+  #     # set config here
+  #   end
+  #
+  # @example Using lambdas
+  #
+  #   require "glass_octopus"
+  #
+  #   logger = Logger.new("log/example.log")
+  #
+  #   app = GlassOctopus::Builder.new do
+  #     use lambda { |ctx| ctx.logger = logger }
+  #
+  #     run Proc.new { |ctx|
+  #       ctx.logger.info "Hello, #{ctx.params['name']}"
+  #     }
+  #   end.to_app
+  #
+  #   GlassOctopus.run(app) do |config|
+  #     # set config here
+  #   end
+  #
+  class Builder
+    def initialize(&block)
+      @entries = []
+      @app = ->(ctx) {}
+
+      instance_eval(&block) if block_given?
+    end
+
+    # Append a middleware to the stack.
+    #
+    # @param klass [Class, #call] a middleware class or a callable object
+    # @param args [Array] arguments to be passed to the klass constructor
+    # @param block a block to be passed to klass constructor
+    # @return [Builder] returns self so calls are chainable
+    def use(klass, *args, &block)
+      @entries << Entry.new(klass, args, block)
+      self
+    end
+
+    # Sets the final step in the middleware pipeline, essentially the
+    # application itself. Takes a parameter that responds to +#call(ctx)+.
+    #
+    # @param app [#call] the application to process messages
+    # @return [Builder] returns self so calls are chainable
+    def run(app)
+      @app = app
+      self
+    end
+
+    # Generate a new middleware stack from the registered entries.
+    #
+    # @return [#call] the entry point of the middleware stack which is callable
+    #   and when called runs the whole stack.
+    def to_app
+      @entries.reverse_each.reduce(@app) do |app, current_entry|
+        current_entry.build(app)
+      end
+    end
+
+    # Generate the middleware stack and call it with the passed in context.
+    #
+    # @param context [Context] message context
+    # @return [void]
+    # @note This method instantiates a new middleware stack on every call.
+    def call(context)
+      to_app.call(context)
+    end
+
+    # Represents an entry in the middleware stack.
+    # @api private
+    Entry = Struct.new(:klass, :args, :block) do
+      def build(next_middleware)
+        if klass.is_a?(Class)
+          klass.new(next_middleware, *args, &block)
+        elsif klass.respond_to?(:call)
+          lambda do |context|
+            klass.call(context)
+            next_middleware.call(context)
+          end
+        else
+          raise ArgumentError, "Invalid middleware, it must respond to `call`: #{klass.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/glass_octopus/configuration.rb

@@ -0,0 +1,62 @@
+require "logger"
+require "concurrent"
+
+require "glass_octopus/bounded_executor"
+
+module GlassOctopus
+  # Configuration for the application.
+  #
+  # @!attribute [rw] connection_adapter
+  #   Connection adapter that connects to the Kafka.
+  # @!attribute [rw] executor
+  #   A thread pool executor to process messages concurrently. Defaults to
+  #   a {BoundedExecutor} with 25 threads.
+  # @!attribute [rw] logger
+  #   A standard library compatible logger for the application. By default it
+  #   logs to the STDOUT.
+  # @!attribute [rw] shutdown_timeout
+  #   Number of seconds to wait for the processing to finish before shutting down.
+  class Configuration
+    attr_accessor :connection_adapter,
+                  :executor,
+                  :logger,
+                  :shutdown_timeout
+
+    def initialize
+      self.logger = Logger.new(STDOUT).tap { |l| l.level = Logger::INFO }
+      self.executor = default_executor
+      self.shutdown_timeout = 10
+    end
+
+    # Creates a new adapter
+    #
+    # @param type [:poseidon, :ruby_kafka] type of the adapter to use
+    # @yield a block to conigure the adapter
+    # @yieldparam config configuration object
+    #
+    # @see PoseidonAdapter
+    # @see RubyKafkaAdapter
+    def adapter(type, &block)
+      self.connection_adapter = build_adapter(type, &block)
+    end
+
+    # @api private
+    def default_executor
+      BoundedExecutor.new(Concurrent::FixedThreadPool.new(25), limit: 25)
+    end
+
+    # @api private
+    def build_adapter(type, &block)
+      case type
+      when :poseidon
+        require "glass_octopus/connection/poseidon_adapter"
+        PoseidonAdapter.new(&block)
+      when :ruby_kafka
+        require "glass_octopus/connection/ruby_kafka_adapter"
+        RubyKafkaAdapter.new(&block)
+      else
+        raise ArgumentError, "Unknown adapter: #{type}"
+      end
+    end
+  end
+end

data/lib/glass_octopus/connection/options_invalid.rb

@@ -0,0 +1,10 @@
+module GlassOctopus
+  class OptionsInvalid < StandardError
+    attr_reader :errors
+
+    def initialize(errors)
+      super("Invalid consumer options: #{errors.join(", ")}")
+      @errors = errors
+    end
+  end
+end

data/lib/glass_octopus/connection/poseidon_adapter.rb

@@ -0,0 +1,113 @@
+require "ostruct"
+require "poseidon_cluster"
+require "glass_octopus/message"
+require "glass_octopus/connection/options_invalid"
+
+module GlassOctopus
+  # Connection adapter that uses the {https://github.com/bpot/poseidon poseidon
+  # gem} to talk to Kafka 0.8.x. Tested with Kafka 0.8.2.
+  #
+  # @example
+  #   adapter = GlassOctopus::PoseidonAdapter.new do |config|
+  #     config.broker_list    = %w[localhost:9092]
+  #     config.zookeeper_list = %w[localhost:2181]
+  #     config.topic          = "mytopic"
+  #     config.group          = "mygroup"
+  #
+  #     require "logger"
+  #     config.logger = Logger.new(STDOUT)
+  #   end
+  #
+  #   adapter.connect.fetch_message do |message|
+  #     p message
+  #   end
+  class PoseidonAdapter
+    # @yield configure poseidon in the yielded block
+    #   The following configuration values are required:
+    #
+    #   * +broker_list+: list of Kafka broker addresses
+    #   * +zookeeper_list+: list of Zookeeper addresses
+    #   * +topic+: name of the topic to subscribe to
+    #   * +group+: name of the consumer group
+    #
+    #   Any other configuration value is passed to
+    #   {http://www.rubydoc.info/github/bsm/poseidon_cluster/Poseidon/ConsumerGroup Poseidon::ConsumerGroup}.
+    #
+    # @raise [OptionsInvalid]
+    def initialize
+      @poseidon_consumer = nil
+      @closed = false
+
+      config = OpenStruct.new
+      yield config
+
+      @options = config.to_h
+      validate_options
+    end
+
+    # Connect to Kafka and Zookeeper, register the consumer group.
+    # This also initiates a rebalance in the consumer group.
+    def connect
+      @closed = false
+      @poseidon_consumer = create_consumer_group
+      self
+    end
+
+    # Fetch messages from kafka in a loop.
+    #
+    # @yield messages read from Kafka
+    # @yieldparam message [Message] a Kafka message
+    def fetch_message
+      @poseidon_consumer.fetch_loop do |partition, messages|
+        break if closed?
+
+        messages.each do |message|
+          yield build_message(partition, message)
+        end
+
+        # Return true to auto-commit offset to Zookeeper
+        true
+      end
+    end
+
+    # Close the connection and stop the {#fetch_message} loop.
+    def close
+      @closed = true
+      @poseidon_consumer.close if @poseidon_consumer
+      @poseidon_cluster = nil
+    end
+
+    # @api private
+    def closed?
+      @closed
+    end
+
+    # @api private
+    def create_consumer_group
+      options = @options.dup
+
+      Poseidon::ConsumerGroup.new(
+        options.delete(:group),
+        options.delete(:broker_list),
+        options.delete(:zookeeper_list),
+        options.delete(:topic),
+        { :max_wait_ms => 1000 }.merge(options)
+      )
+    end
+
+    # @api private
+    def build_message(partition, message)
+      GlassOctopus::Message.new(message.topic, partition, message.offset, message.key, message.value)
+    end
+
+    # @api private
+    def validate_options
+      errors = []
+      [:group, :broker_list, :zookeeper_list, :topic].each do |key|
+        errors << "Missing key: #{key}" unless @options.key?(key)
+      end
+
+      raise OptionsInvalid.new(errors) if errors.any?
+    end
+  end
+end

data/lib/glass_octopus/connection/ruby_kafka_adapter.rb

@@ -0,0 +1,116 @@
+require "kafka"
+require "ostruct"
+require "glass_octopus/message"
+require "glass_octopus/connection/options_invalid"
+
+module GlassOctopus
+  # Connection adapter that uses the {https://github.com/zendesk/ruby-kafka ruby-kafka} gem
+  # to talk to Kafka 0.9+.
+  #
+  # @example
+  #   adapter = GlassOctopus::RubyKafkaAdapter.new do |kafka_config|
+  #     kafka_config.broker_list = %w[localhost:9092]
+  #     kafka_config.topic       = "mytopic"
+  #     kafka_config.group       = "mygroup"
+  #     kafka_config.kafka       = { logger: Logger.new(STDOUT) }
+  #   end
+  #
+  #   adapter.connect.fetch_message do |message|
+  #     p message
+  #   end
+  #
+  class RubyKafkaAdapter
+    # A hash that hold the configuration set up in the initializer block.
+    # @return [Hash]
+    attr_reader :options
+
+    # @yield configure ruby-kafka in the yielded block.
+    #
+    #   The following configuration values are required:
+    #
+    #   * +broker_list+: list of Kafka broker addresses
+    #   * +topic+: name of the topic to subscribe to
+    #   * +group+: name of the consumer group
+    #
+    #   Optional configuration:
+    #
+    #   * +client+: a hash passed on to Kafka.new
+    #   * +consumer+: a hash passed on to kafka.consumer
+    #   * +subscription+: a hash passed on to consumer.subscribe
+    #
+    #   Check the ruby-kafka documentation for driver specific configurations.
+    #
+    # @raise [OptionsInvalid]
+    def initialize
+      config = OpenStruct.new
+      yield config
+      @options = config.to_h
+      validate_options
+
+      @kafka = nil
+      @consumer = nil
+    end
+
+    # Connect to Kafka and join the consumer group.
+    # @return [void]
+    def connect
+      @kafka = connect_to_kafka
+      @consumer = create_consumer(@kafka)
+      @consumer.subscribe(
+        options.fetch(:topic),
+        **options.fetch(:subscription, {})
+      )
+
+      self
+    end
+
+    # Fetch messages from kafka in a loop.
+    # @yield messages read from Kafka
+    # @yieldparam message [Message] a Kafka message
+    def fetch_message
+      @consumer.each_message do |fetched_message|
+        message = Message.new(
+          fetched_message.topic,
+          fetched_message.partition,
+          fetched_message.offset,
+          fetched_message.key,
+          fetched_message.value
+        )
+
+        yield message
+      end
+    end
+
+    # @api private
+    def close
+      @consumer.stop
+      @kafka.close
+    end
+
+    # @api private
+    def connect_to_kafka
+      Kafka.new(
+        seed_brokers: options.fetch(:broker_list),
+        **options.fetch(:client, {})
+      )
+    end
+
+    # @api private
+    def create_consumer(kafka)
+      kafka.consumer(
+        group_id: options.fetch(:group),
+        **options.fetch(:consumer, {})
+      )
+    end
+
+    # @api private
+    def validate_options
+      errors = []
+      [:broker_list, :group, :topic].each do |key|
+        errors << "Missing key: #{key}" unless options.key?(key)
+      end
+
+      raise OptionsInvalid.new(errors) if errors.any?
+    end
+  end
+end