phobos_temp_fork 0.0.1

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'irb'
+require 'bundler/setup'
+require 'phobos'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+config_path = ENV['CONFIG_PATH'] ||
+              (File.exist?('config/phobos.yml') ? 'config/phobos.yml' : 'config/phobos.yml.example')
+Phobos.configure(config_path)
+
+IRB.start

data/bin/phobos

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'phobos'
+require 'phobos/cli'
+
+STDOUT.sync = true
+Phobos::CLI::Commands.start(ARGV)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/phobos.yml.example

@@ -0,0 +1,137 @@
+logger:
+  # Optional log file, set to false or remove to disable it
+  file: log/phobos.log
+  # Optional output format for stdout, default is false (human readable).
+  # Set to true to enable json output.
+  stdout_json: false
+  level: info
+  # Comment the block to disable ruby-kafka logs
+  ruby_kafka:
+    level: error
+
+kafka:
+  # identifier for this application
+  client_id: phobos
+  # timeout setting for connecting to brokers
+  connect_timeout:
+  # timeout setting for socket connections
+  socket_timeout:
+  # PEM encoded CA cert to use with an SSL connection (string)
+  ssl_ca_cert:
+  # PEM encoded client cert to use with an SSL connection (string)
+  # Must be used in combination with ssl_client_cert_key
+  ssl_client_cert:
+  # PEM encoded client cert key to use with an SSL connection (string)
+  # Must be used in combination with ssl_client_cert
+  ssl_client_cert_key:
+  # list of brokers used to initialize the client ("port:protocol")
+  seed_brokers:
+    - localhost:9092
+
+producer:
+  # number of seconds a broker can wait for replicas to acknowledge
+  # a write before responding with a timeout
+  ack_timeout: 5
+  # number of replicas that must acknowledge a write, or `:all`
+  # if all in-sync replicas must acknowledge
+  required_acks: :all
+  # number of retries that should be attempted before giving up sending
+  # messages to the cluster. Does not include the original attempt
+  max_retries: 2
+  # number of seconds to wait between retries
+  retry_backoff: 1
+  # number of messages allowed in the buffer before new writes will
+  # raise {BufferOverflow} exceptions
+  max_buffer_size: 1000
+  # maximum size of the buffer in bytes. Attempting to produce messages
+  # when the buffer reaches this size will result in {BufferOverflow} being raised
+  max_buffer_bytesize: 10000000
+  # name of the compression codec to use, or nil if no compression should be performed.
+  # Valid codecs: `:snappy` and `:gzip`
+  compression_codec:
+  # number of messages that needs to be in a message set before it should be compressed.
+  # Note that message sets are per-partition rather than per-topic or per-producer
+  compression_threshold: 1
+  # maximum number of messages allowed in the queue. Only used for async_producer
+  max_queue_size: 1000
+  # if greater than zero, the number of buffered messages that will automatically
+  # trigger a delivery. Only used for async_producer
+  delivery_threshold: 0
+  # if greater than zero, the number of seconds between automatic message
+  # deliveries. Only used for async_producer
+  delivery_interval: 0
+  # Set this to true to keep the producer connection between publish calls.
+  # This can speed up subsequent messages by around 30%, but it does mean
+  # that you need to manually call sync_producer_shutdown before exiting,
+  # similar to async_producer_shutdown.
+  persistent_connections: false
+  # kafka here supports the same parameters as the top-level, allowing custom connection
+  # configuration details for producers
+  kafka:
+    connect_timeout: 120
+
+consumer:
+  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
+  # it will be kicked out of the group
+  session_timeout: 30
+  # interval between offset commits, in seconds
+  offset_commit_interval: 10
+  # number of messages that can be processed before their offsets are committed.
+  # If zero, offset commits are not triggered by message processing
+  offset_commit_threshold: 0
+  # the time period that committed offsets will be retained, in seconds. Defaults to the broker setting.
+  offset_retention_time:
+  # interval between heartbeats; must be less than the session window
+  heartbeat_interval: 10
+  # kafka here supports the same parameters as the top-level, allowing custom connection
+  # configuration details for consumers
+  kafka:
+    connect_timeout: 130
+
+backoff:
+  min_ms: 1000
+  max_ms: 60000
+
+listeners:
+  - handler: Phobos::EchoHandler
+    topic: test
+    # id of the group that the consumer should join
+    group_id: test-1
+    # Number of threads created for this listener, each thread will behave as an independent consumer.
+    # They don't share any state
+    max_concurrency: 1
+    # Once the consumer group has checkpointed its progress in the topic's partitions,
+    # the consumers will always start from the checkpointed offsets, regardless of config
+    # As such, this setting only applies when the consumer initially starts consuming from a topic
+    start_from_beginning: true
+    # maximum amount of data fetched from a single partition at a time
+    max_bytes_per_partition: 524288 # 512 KB
+    # Minimum number of bytes to read before returning messages from the server; if `max_wait_time` is reached, this is ignored.
+    min_bytes: 1
+    # Maximum duration of time to wait before returning messages from the server, in seconds
+    max_wait_time: 5
+    # Apply this encoding to the message payload, if blank it uses the original encoding. This property accepts values
+    # defined by the ruby Encoding class (https://ruby-doc.org/core-2.3.0/Encoding.html). Ex: UTF_8, ASCII_8BIT, etc
+    force_encoding:
+    # Specify the delivery method for a listener.
+    # Possible values: [`message`, `batch` (default)]
+    #  - `message` will yield individual messages from Ruby Kafka using `each_message` and will commit/heartbeat at every consumed message.
+    #     This is overall a bit slower than using batch, but easier to configure.
+    #  - `batch` will yield batches from Ruby Kafka using `each_batch`, and commit at every consumed batch. It will
+    #     still heartbeat after every message if necessary (using the heartbeat_interval, below).
+    #   - `inline_batch` also uses `each_batch`, but will pass the entire batch to your handler instead
+    #     of one message at a time. To use this method, you should include Phobos::BatchHandler
+    #     instead of Phobos::Handler so that you can make use of the `consume_batch` etc. methods.
+    # Note: Ultimately commit/heartbeart will depend on the offset commit options and the heartbeat interval.
+    delivery: batch
+    # Use this if custom backoff is required for a listener
+    backoff:
+      min_ms: 500
+      max_ms: 10000
+    # session_timeout, offset_commit_interval, offset_commit_threshold, offset_retention_time, and heartbeat_interval
+    # can be customized per listener if desired
+    session_timeout: 30
+    offset_commit_interval: 15
+    offset_commit_threshold: 5
+    offset_retention_time: 172800
+    heartbeat_interval: 20

data/docker-compose.yml

@@ -0,0 +1,28 @@
+version: '2'
+services:
+  test:
+    depends_on:
+      - kafka
+    build:
+      context: .
+    network_mode: service:kafka
+    environment:
+      - "DEFAULT_TIMEOUT=${DEFAULT_TIMEOUT}"
+    command: rspec
+    volumes:
+      - ./coverage:/opt/phobos/coverage
+
+  zookeeper:
+    image: wurstmeister/zookeeper:latest
+    ports:
+     - 2181:2181
+
+  kafka:
+    depends_on:
+      - zookeeper
+    image: wurstmeister/kafka:0.11.0.1
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_BROKER_ID: 0

data/examples/handler_saving_events_database.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+#
+# This example assumes that you want to save all events in your database for
+# recovery purposes. The consumer will process the message and perform other
+# operations, this implementation assumes a generic way to save the events.
+#
+# Setup your database connection using `phobos_boot.rb`. Remember to Setup
+# a hook to disconnect, e.g: `at_exit { Database.disconnect! }`
+#
+class HandlerSavingEventsDatabase
+  include Phobos::Handler
+  include Phobos::Producer
+
+  def self.around_consume(payload, _metadata)
+    #
+    # Let's assume `::from_message` will initialize our object with `payload`
+    #
+    event = Model::Event.from_message(payload)
+
+    #
+    # If event already exists in the database, skip this message
+    #
+    return if event.exists?
+
+    Model::Event.transaction do
+      #
+      # Executes `#consume` method
+      #
+      new_values = yield
+
+      #
+      # `#consume` method can return additional data (up to your code)
+      #
+      event.update_with_new_attributes(new_values)
+
+      #
+      # Let's assume the event is just initialized and now is the time to save it
+      #
+      event.save!
+    end
+  end
+
+  def consume(payload, _metadata)
+    #
+    # Process the event, it might index it to elasticsearch or notify other
+    # system, you should process your message inside this method.
+    #
+    { new_vale: payload.length % 3 }
+  end
+end

data/examples/handler_using_async_producer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+#
+# This example assumes you want to process the event and publish another
+# one to kafka. A new event is always published thus we want to use the async producer
+# to better use our resources and to speed up the process
+#
+class HandlerUsingAsyncProducer
+  include Phobos::Handler
+  include Phobos::Producer
+
+  PUBLISH_TO 'another-topic'
+
+  def consume(payload, _metadata)
+    producer.async_publish(PUBLISH_TO, "#{payload}-#{rand}")
+  end
+end

data/examples/publishing_messages_without_consumer.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+#
+# This example assumes you want to create a threaded kafka generator which
+# publish a stream of kafka messages without consuming them. It also shows
+# what happens when you produce more messages than the producer can handle.
+#
+require 'bundler/setup'
+require 'json'
+require 'phobos'
+
+TOPIC = 'test-partitions'
+
+Phobos.configure('config/phobos.yml')
+
+class MyProducer
+  include Phobos::Producer
+end
+
+#
+# Trapping signals to properly stop this generator
+#
+@stop = false
+[:INT, :TERM, :QUIT].each do |signal|
+  Signal.trap(signal) do
+    puts 'Stopping'
+    @stop = true
+  end
+end
+
+Thread.new do
+  begin
+    total = 1
+
+    loop do
+      break if @stop
+
+      key = SecureRandom.uuid
+      payload = Time.now.utc.to_json
+
+      begin
+        # Producer will use phobos configuration to create a kafka client and
+        # a producer and it will bind both to the current thread, so it's safe
+        # to call class methods here
+        #
+        MyProducer
+          .producer
+          .async_publish(topic: TOPIC, payload: payload, key: key)
+
+        puts "produced #{key}, total: #{total}"
+
+      # Since this is very simplistic code, we are going to generate more messages than
+      # the producer can write to Kafka. Eventually we'll get some buffer overflows
+      #
+      rescue Kafka::BufferOverflow => e
+        puts '| waiting'
+        sleep(1)
+        retry
+      end
+
+      total += 1
+    end
+  ensure
+    #
+    # Before we stop we must shutdown the async producer to ensure that all messages
+    # are delivered
+    #
+    MyProducer
+      .producer
+      .async_producer_shutdown
+
+    #
+    # Since no client was configured (we can do this with
+    #   `MyProducer.producer.configure_kafka_client`)
+    # we must get the auto generated one and close it properly
+    #
+    MyProducer
+      .producer
+      .kafka_client
+      .close
+  end
+end.join

data/lib/phobos/actions/process_batch.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Phobos
+  module Actions
+    class ProcessBatch
+      include Phobos::Instrumentation
+
+      attr_reader :metadata
+
+      def initialize(listener:, batch:, listener_metadata:)
+        @listener = listener
+        @batch = batch
+        @listener_metadata = listener_metadata
+        @metadata = listener_metadata.merge(
+          batch_size: batch.messages.count,
+          partition: batch.partition,
+          offset_lag: batch.offset_lag
+        )
+      end
+
+      def execute
+        instrument('listener.process_batch', @metadata) do |_metadata|
+          @batch.messages.each do |message|
+            Phobos::Actions::ProcessMessage.new(
+              listener: @listener,
+              message: message,
+              listener_metadata: @listener_metadata
+            ).execute
+            @listener.consumer.trigger_heartbeat
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phobos/actions/process_batch_inline.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'phobos/batch_message'
+require 'phobos/processor'
+
+module Phobos
+  module Actions
+    class ProcessBatchInline
+      include Phobos::Processor
+
+      attr_reader :metadata
+
+      def initialize(listener:, batch:, metadata:)
+        @listener = listener
+        @batch = batch
+        @listener = listener
+        @batch = batch
+        @metadata = metadata.merge(
+          batch_size: batch.messages.count,
+          partition: batch.partition,
+          offset_lag: batch.offset_lag,
+          retry_count: 0
+        )
+      end
+
+      def execute
+        batch = @batch.messages.map { |message| instantiate_batch_message(message) }
+
+        begin
+          process_batch(batch)
+        rescue StandardError => e
+          handle_error(e, 'listener.retry_handler_error_batch',
+                       "error processing inline batch, waiting #{backoff_interval}s")
+          retry
+        end
+      end
+
+      private
+
+      def instantiate_batch_message(message)
+        Phobos::BatchMessage.new(
+          key: message.key,
+          partition: message.partition,
+          offset: message.offset,
+          payload: force_encoding(message.value),
+          headers: message.headers
+        )
+      end
+
+      def process_batch(batch)
+        instrument('listener.process_batch_inline', @metadata) do |_metadata|
+          handler = @listener.handler_class.new
+
+          handler.around_consume_batch(batch, @metadata) do |around_batch, around_metadata|
+            handler.consume_batch(around_batch, around_metadata)
+          end
+        end
+      end
+    end
+  end
+end